BUMP 1.8.1

post 1.8.1-rc1
BUMP 1.8.1-rc1
2011-04-06 00:40:13 -04:00 · 2011-04-01 01:14:09 -04:00 · 2011-03-31 18:39:07 -04:00 · 2011-03-31 03:42:07 -04:00 · 2011-03-31 03:12:43 -04:00 · 2011-03-31 03:12:40 -04:00
80 changed files with 800 additions and 537 deletions
--- a/client/connpool.cpp
+++ b/client/connpool.cpp
@@ -37,7 +37,12 @@ namespace mongo {
    }

    void PoolForHost::done( DBClientBase * c ) {
-        _pool.push(c);
+        if ( _pool.size() >= _maxPerHost ) {
+            delete c;
+        }
+        else {
+            _pool.push(c);
+        }
    }

    DBClientBase * PoolForHost::get() {
@@ -86,6 +91,8 @@ namespace mongo {
        _created++;
    }

+    unsigned PoolForHost::_maxPerHost = 50;
+
    // ------ DBConnectionPool ------

    DBConnectionPool pool;
--- a/client/connpool.h
+++ b/client/connpool.h
@@ -57,6 +57,9 @@ namespace mongo {
        void done( DBClientBase * c );

        void flush();
+
+        static void setMaxPerHost( unsigned max ) { _maxPerHost = max; }
+        static unsigned getMaxPerHost() { return _maxPerHost; }
    private:

        struct StoredConnection {
@@ -71,6 +74,8 @@ namespace mongo {
        std::stack<StoredConnection> _pool;
        long long _created;
        ConnectionString::ConnectionType _type;
+
+        static unsigned _maxPerHost;
    };

    class DBConnectionHook {
--- a/client/dbclient.cpp
+++ b/client/dbclient.cpp
@@ -915,13 +915,14 @@ namespace mongo {

    void DBClientConnection::checkResponse( const char *data, int nReturned ) {
        /* check for errors.  the only one we really care about at
-         this stage is "not master" */
+         * this stage is "not master" 
+        */
+        
        if ( clientSet && nReturned ) {
            assert(data);
            BSONObj o(data);
-            BSONElement e = o.firstElement();
-            if ( strcmp(e.fieldName(), "$err") == 0 &&
-                    e.type() == String && strncmp(e.valuestr(), "not master", 10) == 0 ) {
+            BSONElement e = o["$err"];
+            if ( e.type() == String && str::contains( e.valuestr() , "not master" ) ) {
                clientSet->isntMaster();
            }
        }
--- a/client/dbclient_rs.cpp
+++ b/client/dbclient_rs.cpp
@@ -52,13 +52,17 @@ namespace mongo {
        }
    protected:
        void run() {
+            log() << "starting" << endl;
            while ( ! inShutdown() ) {
                sleepsecs( 20 );
                try {
                    ReplicaSetMonitor::checkAll();
                }
                catch ( std::exception& e ) {
-                    error() << "ReplicaSetMonitorWatcher: check failed: " << e.what() << endl;
+                    error() << "check failed: " << e.what() << endl;
+                }
+                catch ( ... ) {
+                    error() << "unkown error" << endl;
                }
            }
        }
@@ -121,6 +125,7 @@ namespace mongo {
        while ( true ) {
            ReplicaSetMonitorPtr m;
            {
+                scoped_lock lk( _setsLock );
                for ( map<string,ReplicaSetMonitorPtr>::iterator i=_sets.begin(); i!=_sets.end(); ++i ) {
                    string name = i->first;
                    if ( seen.count( name ) )
@@ -175,26 +180,26 @@ namespace mongo {
    void ReplicaSetMonitor::notifyFailure( const HostAndPort& server ) {
        scoped_lock lk( _lock );
        if ( _master >= 0 && _master < (int)_nodes.size() ) {
-            if ( server == _nodes[_master].addr )
+            if ( server == _nodes[_master].addr ) {
+                _nodes[_master].ok = false; 
                _master = -1;
+            }
        }
    }



    HostAndPort ReplicaSetMonitor::getMaster() {
-        bool good = false;
        {
            scoped_lock lk( _lock );
-            good = _master >= 0 && _nodes[_master].ok;
+            if ( _master >= 0 && _nodes[_master].ok )
+                return _nodes[_master].addr;
        }
-
-        if ( ! good )
-            _check();
-
-        uassert( 10009 , str::stream() << "ReplicaSetMonitor no master found for set: " << _name , _master >= 0 );
+        
+        _check();

        scoped_lock lk( _lock );
+        uassert( 10009 , str::stream() << "ReplicaSetMonitor no master found for set: " << _name , _master >= 0 );
        return _nodes[_master].addr;
    }
    
@@ -429,7 +434,7 @@ namespace mongo {
        }

        _masterHost = _monitor->getMaster();
-        _master.reset( new DBClientConnection( true ) );
+        _master.reset( new DBClientConnection( true , this ) );
        string errmsg;
        if ( ! _master->connect( _masterHost , errmsg ) ) {
            _monitor->notifyFailure( _masterHost );
@@ -449,7 +454,7 @@ namespace mongo {
        }
        
        _slaveHost = _monitor->getSlave();
-        _slave.reset( new DBClientConnection( true ) );
+        _slave.reset( new DBClientConnection( true , this ) );
        _slave->connect( _slaveHost );
        _auth( _slave.get() );
        return _slave.get();
@@ -564,6 +569,11 @@ namespace mongo {
        assert(0);
    }

+    void DBClientReplicaSet::isntMaster() { 
+        log() << "got not master for: " << _masterHost << endl;
+        _monitor->notifyFailure( _masterHost );
+        _master.reset(); 
+    }

    bool DBClientReplicaSet::call( Message &toSend, Message &response, bool assertOk , string * actualServer ) {
        if ( toSend.operation() == dbQuery ) {
--- a/client/dbclient_rs.h
+++ b/client/dbclient_rs.h
@@ -213,7 +213,7 @@ namespace mongo {

        /* this is the callback from our underlying connections to notify us that we got a "not master" error.
         */
-        void isntMaster() { _master.reset(); }
+        void isntMaster();

        // ----- status ------

--- a/db/btree.cpp
+++ b/db/btree.cpp
@@ -1114,16 +1114,17 @@ namespace mongo {

    /** remove a key from the index */
    bool BtreeBucket::unindex(const DiskLoc thisLoc, IndexDetails& id, const BSONObj& key, const DiskLoc recordLoc ) const {
-        if ( key.objsize() > KeyMax ) {
-            OCCASIONALLY problem() << "unindex: key too large to index, skipping " << id.indexNamespace() << /* ' ' << key.toString() << */ endl;
-            return false;
-        }
-
        int pos;
        bool found;
        DiskLoc loc = locate(id, thisLoc, key, Ordering::make(id.keyPattern()), pos, found, recordLoc, 1);
        if ( found ) {
+
+            if ( key.objsize() > KeyMax ) {
+                OCCASIONALLY problem() << "unindex: key too large to index but was found for " << id.indexNamespace() << " reIndex suggested" << endl;
+            }
+            
            loc.btreemod()->delKeyAtPos(loc, id, pos, Ordering::make(id.keyPattern()));
+            
            return true;
        }
        return false;
@@ -1740,13 +1741,18 @@ namespace mongo {
    }

    void BtreeBuilder::mayCommitProgressDurably() {
-        RARELY {
-            getDur().commitIfNeeded();
+        if ( getDur().commitIfNeeded() ) {
            b = cur.btreemod();
        }
    }

    void BtreeBuilder::addKey(BSONObj& key, DiskLoc loc) {
+        if ( key.objsize() > KeyMax ) {
+            problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace() 
+                      << ' ' << key.objsize() << ' ' << key.toString() << endl;
+            return;
+        }
+
        if( !dupsAllowed ) {
            if( n > 0 ) {
                int cmp = keyLast.woCompare(key, order);
@@ -1760,15 +1766,9 @@ namespace mongo {
        }

        if ( ! b->_pushBack(loc, key, ordering, DiskLoc()) ) {
-            // no room
-            if ( key.objsize() > KeyMax ) {
-                problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace() << ' ' << key.objsize() << ' ' << key.toString() << endl;
-            }
-            else {
-                // bucket was full
-                newBucket();
-                b->pushBack(loc, key, ordering, DiskLoc());
-            }
+            // bucket was full
+            newBucket();
+            b->pushBack(loc, key, ordering, DiskLoc());
        }
        n++;
        mayCommitProgressDurably();
@@ -1790,6 +1790,11 @@ namespace mongo {

            DiskLoc xloc = loc;
            while( !xloc.isNull() ) {
+                if ( getDur().commitIfNeeded() ) {
+                    b = cur.btreemod();
+                    up = upLoc.btreemod();
+                }
+
                BtreeBucket *x = xloc.btreemod();
                BSONObj k;
                DiskLoc r;
@@ -1833,18 +1838,21 @@ namespace mongo {
    }

    BtreeBuilder::~BtreeBuilder() {
-        if( !committed ) {
-            log(2) << "Rolling back partially built index space" << endl;
-            DiskLoc x = first;
-            while( !x.isNull() ) {
-                DiskLoc next = x.btree()->tempNext();
-                string ns = idx.indexNamespace();
-                theDataFileMgr._deleteRecord(nsdetails(ns.c_str()), ns.c_str(), x.rec(), x);
-                x = next;
+        DESTRUCTOR_GUARD(
+            if( !committed ) {
+                log(2) << "Rolling back partially built index space" << endl;
+                DiskLoc x = first;
+                while( !x.isNull() ) {
+                    DiskLoc next = x.btree()->tempNext();
+                    string ns = idx.indexNamespace();
+                    theDataFileMgr._deleteRecord(nsdetails(ns.c_str()), ns.c_str(), x.rec(), x);
+                    x = next;
+                    getDur().commitIfNeeded();
+                }
+                assert( idx.head.isNull() );
+                log(2) << "done rollback" << endl;
            }
-            assert( idx.head.isNull() );
-            log(2) << "done rollback" << endl;
-        }
+        )
    }

 }
--- a/db/cap.cpp
+++ b/db/cap.cpp
@@ -319,6 +319,7 @@ namespace mongo {
                // 'end' has been found and removed, so break.
                break;
            }
+            getDur().commitIfNeeded();
            // 'curr' will point to the newest document in the collection.
            DiskLoc curr = theCapExtent()->lastRecord;
            assert( !curr.isNull() );
--- a/db/client.cpp
+++ b/db/client.cpp
@@ -169,11 +169,6 @@ namespace mongo {
            uassert( 13005 , "can't create db, keeps getting closed" , _db );
        }

-        _client->_context = this;
-        _client->_curOp->enter( this );
-        if ( doauth )
-            _auth( lockState );
-
        switch ( _client->_curOp->getOp() ) {
        case dbGetMore: // getMore's are special and should be handled else where
        case dbUpdate: // update & delete check shard version in instance.cpp, so don't check here as well
@@ -188,6 +183,11 @@ namespace mongo {
            }
        }
        }
+
+        _client->_context = this;
+        _client->_curOp->enter( this );
+        if ( doauth )
+            _auth( lockState );
    }

    void Client::Context::_auth( int lockState ) {
--- a/db/clientcursor.h
+++ b/db/clientcursor.h
@@ -91,6 +91,13 @@ namespace mongo {
                    _c = 0;
                }
            }
+            /**
+             * call this if during a yield, the cursor got deleted
+             * if so, we don't want to use the point address
+             */
+            void deleted() {
+                _c = 0;
+            }
            ~Pointer() { release(); }
            Pointer(long long cursorid) {
                recursive_scoped_lock lock(ccmutex);
--- a/db/cloner.cpp
+++ b/db/cloner.cpp
@@ -244,6 +244,7 @@ namespace mongo {
            string temp = ctx.db()->name + ".system.indexes";
            copy( temp.c_str() , temp.c_str() , /*isindex*/true , logForRepl , false , true , BSON( "ns" << ns ) );
        }
+        getDur().commitIfNeeded();
        return true;
    }

--- a/db/db.cpp
+++ b/db/db.cpp
@@ -635,14 +635,12 @@ int main(int argc, char* argv[]) {
    ("dbpath", po::value<string>() , "directory for datafiles")
    ("diaglog", po::value<int>(), "0=off 1=W 2=R 3=both 7=W+some reads")
    ("directoryperdb", "each database will be stored in a separate directory")
-    ("dur", "enable journaling")
-    ("durOptions", po::value<int>(), "durability diagnostic options")
+    ("journal", "enable journaling")
+    ("journalOptions", po::value<int>(), "journal diagnostic options")
    ("ipv6", "enable IPv6 support (disabled by default)")
    ("jsonp","allow JSONP access via http (has security implications)")
    ("maxConns",po::value<int>(), "max number of simultaneous connections")
    ("noauth", "run without security")
-    ("nocursors", "diagnostic/debugging option")
-    ("nohints", "ignore query hints")
    ("nohttpinterface", "disable http interface")
    ("noprealloc", "disable data file preallocation - will often hurt performance")
    ("noscripting", "disable scripting engine")
@@ -703,6 +701,10 @@ int main(int argc, char* argv[]) {
    ("arbiter", po::value<string>(), "address of replica pair arbiter server DEPRECATED")
    ("nodur", "disable journaling (currently the default)")
    ("appsrvpath", po::value<string>(), "root directory for the babble app server")
+    ("nocursors", "diagnostic/debugging option that turns off cursors DO NOT USE IN PRODUCTION")
+    ("nohints", "ignore query hints")
+    ("dur", "enable journaling") // deprecated version
+    ("durOptions", po::value<int>(), "durability diagnostic options") // deprecated version
    ;


@@ -723,7 +725,11 @@ int main(int argc, char* argv[]) {
    dbExecCommand = argv[0];

    srand(curTimeMicros());
+#if( BOOST_VERSION >= 104500 )
+    boost::filesystem::path::default_name_check( boost::filesystem2::no_check );
+#else
    boost::filesystem::path::default_name_check( boost::filesystem::no_check );
+#endif

    {
        unsigned x = 0x12345678;
@@ -795,12 +801,15 @@ int main(int argc, char* argv[]) {
        if( params.count("nodur") ) {
            cmdLine.dur = false;
        }
-        if( params.count("dur") ) {
+        if( params.count("dur") || params.count( "journal" ) ) {
            cmdLine.dur = true;
        }
        if (params.count("durOptions")) {
            cmdLine.durOptions = params["durOptions"].as<int>();
        }
+        if (params.count("journalOptions")) {
+            cmdLine.durOptions = params["journalOptions"].as<int>();
+        }
        if (params.count("objcheck")) {
            objcheck = true;
        }
@@ -1099,9 +1108,27 @@ namespace mongo {
        oss << "Backtrace:" << endl;
        printStackTrace( oss );
        rawOut( oss.str() );
+
+        if( cmdLine.dur ) { 
+            ::exit(EXIT_ABRUPT);
+        }
+
        dbexit( EXIT_ABRUPT );
    }

+    void abruptQuitWithAddrSignal( int signal, siginfo_t *siginfo, void * ) {
+        ostringstream oss;
+        oss << "Invalid";
+        if ( signal == SIGSEGV || signal == SIGBUS ) {
+            oss << " access";
+        } else {
+            oss << " operation";   
+        }
+        oss << " at address: " << siginfo->si_addr << endl;
+        rawOut( oss.str() );
+        abruptQuit( signal );   
+    }
+        
    sigset_t asyncSignals;
    // The above signals will be processed by this thread only, in order to
    // ensure the db and log mutexes aren't held.
@@ -1124,10 +1151,18 @@ namespace mongo {
    void setupSignals_ignoreHelper( int signal ) {}

    void setupSignals( bool inFork ) {
-        assert( signal(SIGSEGV, abruptQuit) != SIG_ERR );
-        assert( signal(SIGFPE, abruptQuit) != SIG_ERR );
+        struct sigaction addrSignals;
+        memset( &addrSignals, 0, sizeof( struct sigaction ) );
+        addrSignals.sa_sigaction = abruptQuitWithAddrSignal;
+        sigemptyset( &addrSignals.sa_mask );
+        addrSignals.sa_flags = SA_SIGINFO;
+       
+        assert( sigaction(SIGSEGV, &addrSignals, 0) == 0 );
+        assert( sigaction(SIGBUS, &addrSignals, 0) == 0 );
+        assert( sigaction(SIGILL, &addrSignals, 0) == 0 );
+        assert( sigaction(SIGFPE, &addrSignals, 0) == 0 );
+        
        assert( signal(SIGABRT, abruptQuit) != SIG_ERR );
-        assert( signal(SIGBUS, abruptQuit) != SIG_ERR );
        assert( signal(SIGQUIT, abruptQuit) != SIG_ERR );
        assert( signal(SIGPIPE, pipeSigHandler) != SIG_ERR );

--- a/db/dbcommands.cpp
+++ b/db/dbcommands.cpp
@@ -406,6 +406,8 @@ namespace mongo {
                    t.appendBool( "supported" , false );
                }

+                timeBuilder.appendNumber( "middle of mem" , Listener::getElapsedTimeMillis() - start );
+
                t.appendNumber( "mapped" , MemoryMappedFile::totalMappedLength() / ( 1024 * 1024 ) );

                t.done();
--- a/db/dbhelpers.cpp
+++ b/db/dbhelpers.cpp
@@ -85,6 +85,7 @@ namespace mongo {
            }
        }
        virtual long long nscanned() {
+            // We don't support yielding, so will always have c_.
            assert( c_.get() );
            return c_->nscanned();
        }
--- a/db/dur.cpp
+++ b/db/dur.cpp
@@ -211,14 +211,14 @@ namespace mongo {
            return p;
        }

-        void DurableImpl::commitIfNeeded() {
-#if defined(_DEBUG)
-            commitJob._nSinceCommitIfNeededCall = 0;
-#endif
+        bool DurableImpl::commitIfNeeded() {
+            DEV commitJob._nSinceCommitIfNeededCall = 0;
            if (commitJob.bytes() > UncommittedBytesLimit) { // should this also fire if CmdLine::DurAlwaysCommit?
                stats.curr->_earlyCommits++;
                groupCommit();
+                return true;
            }
+            return false;
        }

        /** Used in _DEBUG builds to check that we didn't overwrite the last intent
@@ -369,7 +369,7 @@ namespace mongo {
            // faults after remapping, so doing a little bit at a time will avoid big load spikes on
            // remapping.
            unsigned long long now = curTimeMicros64();
-            double fraction = (now-lastRemap)/20000000.0;
+            double fraction = (now-lastRemap)/2000000.0;
            lastRemap = now;

            rwlock lk(MongoFile::mmmutex, false);
@@ -522,7 +522,9 @@ namespace mongo {
            }

            // starvation on read locks could occur.  so if read lock acquisition is slow, try to get a
-            // write lock instead.  otherwise writes could use too much RAM.
+            // write lock instead.  otherwise journaling could be delayed too long (too much data will 
+            // not accumulate though, as commitIfNeeded logic will have executed in the meantime if there 
+            // has been writes)
            writelock lk;
            groupCommit();
        }
@@ -587,18 +589,11 @@ namespace mongo {
        void recover();

        void releasingWriteLock() {
-            try {
-#if defined(_DEBUG)
-                commitJob._nSinceCommitIfNeededCall = 0; // implicit commit if needed
-#endif
-                if (commitJob.bytes() > UncommittedBytesLimit || cmdLine.durOptions & CmdLine::DurAlwaysCommit) {
-                    stats.curr->_earlyCommits++;
-                    groupCommit();
-                }
-            }
-            catch(std::exception& e) {
-                log() << "exception in dur::releasingWriteLock causing immediate shutdown: " << e.what() << endl;
-                abort(); // based on myTerminate()
+            // implicit commitIfNeeded check on each write unlock
+            DEV commitJob._nSinceCommitIfNeededCall = 0; // implicit commit if needed
+            if( commitJob.bytes() > UncommittedBytesLimit || cmdLine.durOptions & CmdLine::DurAlwaysCommit ) {
+                stats.curr->_earlyCommits++;
+                groupCommit();
            }
        }

--- a/db/dur.h
+++ b/db/dur.h
@@ -96,8 +96,9 @@ namespace mongo {
                (like creating an index or update with $atomic) can call this
                whenever the db is in a sane state and it will prevent commits
                from growing too large.
+                @return true if commited
            */
-            virtual void commitIfNeeded() = 0;
+            virtual bool commitIfNeeded() = 0;

            /** Declare write intent for a DiskLoc.  @see DiskLoc::writing() */
            inline DiskLoc& writingDiskLoc(DiskLoc& d) { return *((DiskLoc*) writingPtr(&d, sizeof(d))); }
@@ -172,7 +173,7 @@ namespace mongo {
            void createdFile(string filename, unsigned long long len) { }
            bool awaitCommit() { return false; }
            bool commitNow() { return false; }
-            void commitIfNeeded() { }
+            bool commitIfNeeded() { return false; }
            void setNoJournal(void *dst, void *src, unsigned len);
            void syncDataAndTruncateJournal() {}
        };
@@ -185,7 +186,7 @@ namespace mongo {
            void createdFile(string filename, unsigned long long len);
            bool awaitCommit();
            bool commitNow();
-            void commitIfNeeded();
+            bool commitIfNeeded();
            void setNoJournal(void *dst, void *src, unsigned len);
            void syncDataAndTruncateJournal();
        };
--- a/db/dur_commitjob.cpp
+++ b/db/dur_commitjob.cpp
@@ -183,7 +183,7 @@ namespace mongo {
                // remember intent. we will journal it in a bit
                _wi.insertWriteIntent(p, len);
                wassert( _wi._writes.size() <  2000000 );
-                assert(  _wi._writes.size() < 20000000 );
+                //assert(  _wi._writes.size() < 20000000 );

                {
                    // a bit over conservative in counting pagebytes used
@@ -200,7 +200,9 @@ namespace mongo {
                                log() << "debug nsincecommitifneeded:" << _nSinceCommitIfNeededCall << " bytes:" << _bytes << endl;
                        }
 #endif
-                        uassert(13623, "DR102 too much data written uncommitted", _bytes < UncommittedBytesLimit * 3);
+                        if ( _bytes >= UncommittedBytesLimit * 3 ) {
+                            wassert( ! "DR102 too much data written uncommitted" );
+                        }
                    }
                }
            }
--- a/db/dur_journal.cpp
+++ b/db/dur_journal.cpp
@@ -262,6 +262,7 @@ namespace mongo {
                        log() << "warning caught exception in preallocateFiles, continuing" << endl;
                    }
            }
+            j.open();
        }

        void removeOldJournalFile(path p) { 
@@ -515,7 +516,7 @@ namespace mongo {

            scoped_lock lk(_curLogFileMutex);

-            if ( inShutdown() )
+            if ( inShutdown() || !_curLogFile )
                return;

            j.updateLSNFile();
--- a/db/dur_journalimpl.h
+++ b/db/dur_journalimpl.h
@@ -63,10 +63,10 @@ namespace mongo {
                    _open();
            }

-        private:
            /** open a journal file to journal operations to. */
            void open();

+        private:
            void _open();
            void closeCurrentJournalFile();
            void removeUnneededJournalFiles();
--- a/db/dur_recover.cpp
+++ b/db/dur_recover.cpp
@@ -73,16 +73,17 @@ namespace mongo {
                    if( m.count(u) ) {
                        uasserted(13531, str::stream() << "unexpected files in journal directory " << dir.string() << " : " << fileName);
                    }
-                    if( !m.empty() && !m.count(u-1) ) {
-                        uasserted(13532,
-                                  str::stream() << "unexpected file in journal directory " << dir.string()
-                                  << " : " << fileName << " : can't find its preceeding file");
-                    }
                    m.insert( pair<unsigned,path>(u,filepath) );
                }
            }
-            for( map<unsigned,path>::iterator i = m.begin(); i != m.end(); ++i )
+            for( map<unsigned,path>::iterator i = m.begin(); i != m.end(); ++i ) {
+                if( i != m.begin() && m.count(i->first - 1) == 0 ) {
+                    uasserted(13532,
+                    str::stream() << "unexpected file in journal directory " << dir.string()
+                      << " : " << filesystem::path(i->second).leaf() << " : can't find its preceeding file");
+                }
                files.push_back(i->second);
+            }
        }

        /** read through the memory mapped data of a journal file (journal/j._<n> file)
@@ -186,8 +187,10 @@ namespace mongo {
        }

        RecoveryJob::~RecoveryJob() {
-            if( !_mmfs.empty() )
-                close();
+            DESTRUCTOR_GUARD(
+                if( !_mmfs.empty() )
+                    close();
+            )
        }

        void RecoveryJob::close() {
@@ -445,8 +448,8 @@ namespace mongo {
            }
        } brunittest;

-
-        RecoveryJob RecoveryJob::_instance;
+        // can't free at termination because order of destruction of global vars is arbitrary
+        RecoveryJob &RecoveryJob::_instance = *(new RecoveryJob());

    } // namespace dur

--- a/db/dur_recover.h
+++ b/db/dur_recover.h
@@ -15,7 +15,7 @@ namespace mongo {
         */
        class RecoveryJob : boost::noncopyable {
        public:
-            RecoveryJob() :_lastDataSyncedFromLastRun(0), _mx("recovery") { _lastSeqMentionedInConsoleLog = 1; }
+            RecoveryJob() :_lastDataSyncedFromLastRun(0), _mx("recovery"), _recovering(false) { _lastSeqMentionedInConsoleLog = 1; }
            void go(vector<path>& files);
            ~RecoveryJob();
            void processSection(const void *, unsigned len);
@@ -39,7 +39,7 @@ namespace mongo {

            bool _recovering; // are we in recovery or WRITETODATAFILES

-            static RecoveryJob _instance;
+            static RecoveryJob &_instance;
        };
    }
 }
--- a/db/lasterror.cpp
+++ b/db/lasterror.cpp
@@ -28,7 +28,6 @@ namespace mongo {

    LastError LastError::noError;
    LastErrorHolder lastError;
-    mongo::mutex LastErrorHolder::_idsmutex("LastErrorHolder");

    bool isShell = false;
    void raiseError(int code , const char *msg) {
@@ -79,22 +78,9 @@ namespace mongo {
    }

    LastErrorHolder::~LastErrorHolder() {
-        for ( IDMap::iterator i = _ids.begin(); i != _ids.end(); ++i ) {
-            delete i->second.lerr;
-            i->second.lerr = 0;
-        }
-        _ids.clear();
    }


-    void LastErrorHolder::setID( int id ) {
-        _id.set( id );
-    }
-
-    int LastErrorHolder::getID() {
-        return _id.get();
-    }
-
    LastError * LastErrorHolder::disableForCommand() {
        LastError *le = _get();
        assert( le );
@@ -111,77 +97,31 @@ namespace mongo {
    }

    LastError * LastErrorHolder::_get( bool create ) {
-        int id = _id.get();
-        if ( id == 0 ) {
-            LastError * le = _tl.get();
-            if ( ! le && create ) {
-                le = new LastError();
-                _tl.reset( le );
-            }
-            return le;
+        LastError * le = _tl.get();
+        if ( ! le && create ) {
+            le = new LastError();
+            _tl.reset( le );
        }
-
-        scoped_lock lock(_idsmutex);
-        map<int,Status>::iterator i = _ids.find( id );
-        if ( i == _ids.end() ) {
-            if ( ! create )
-                return 0;
-
-            LastError * le = new LastError();
-            Status s;
-            s.time = time(0);
-            s.lerr = le;
-            _ids[id] = s;
-            return le;
-        }
-
-        Status &status = i->second;
-        status.time = time(0);
-        return status.lerr;
-    }
-
-    void LastErrorHolder::remove( int id ) {
-        scoped_lock lock(_idsmutex);
-        map<int,Status>::iterator i = _ids.find( id );
-        if ( i == _ids.end() )
-            return;
-
-        delete i->second.lerr;
-        _ids.erase( i );
+        return le;
    }

    void LastErrorHolder::release() {
-        int id = _id.get();
-        if ( id == 0 ) {
-            _tl.release();
-            return;
-        }
-
-        remove( id );
+        _tl.release();
    }

    /** ok to call more than once. */
    void LastErrorHolder::initThread() {
-        if( _tl.get() ) return;
-        assert( _id.get() == 0 );
-        _tl.reset( new LastError() );
+        if( ! _tl.get() ) 
+            _tl.reset( new LastError() );
    }

    void LastErrorHolder::reset( LastError * le ) {
-        int id = _id.get();
-        if ( id == 0 ) {
-            _tl.reset( le );
-            return;
-        }
-
-        scoped_lock lock(_idsmutex);
-        Status & status = _ids[id];
-        status.time = time(0);
-        status.lerr = le;
+        _tl.reset( le );
    }

    void prepareErrForNewRequest( Message &m, LastError * err ) {
        // a killCursors message shouldn't affect last error
+        assert( err );
        if ( m.operation() == dbKillCursors ) {
            err->disabled = true;
        }
@@ -191,60 +131,10 @@ namespace mongo {
        }
    }

-    LastError * LastErrorHolder::startRequest( Message& m , int clientId ) {
-        assert( clientId );
-        setID( clientId );
-
-        LastError * le = _get( true );
+    LastError * LastErrorHolder::startRequest( Message& m , LastError * le ) {
+        assert( le );
        prepareErrForNewRequest( m, le );
        return le;
    }

-    void LastErrorHolder::startRequest( Message& m , LastError * connectionOwned ) {
-        prepareErrForNewRequest( m, connectionOwned );
-    }
-
-    void LastErrorHolder::disconnect( int clientId ) {
-        if ( clientId )
-            remove(clientId);
-    }
-
-    struct LastErrorHolderTest : public UnitTest {
-    public:
-
-        void test( int i ) {
-            _tl.set( i );
-            assert( _tl.get() == i );
-        }
-
-        void tlmaptest() {
-            test( 1 );
-            test( 12123123 );
-            test( -123123 );
-            test( numeric_limits<int>::min() );
-            test( numeric_limits<int>::max() );
-        }
-
-        void run() {
-            tlmaptest();
-
-            LastError * a = new LastError();
-            LastError * b = new LastError();
-
-            LastErrorHolder holder;
-            holder.reset( a );
-            assert( a == holder.get() );
-            holder.setID( 1 );
-            assert( 0 == holder.get() );
-            holder.reset( b );
-            assert( b == holder.get() );
-            holder.setID( 0 );
-            assert( a == holder.get() );
-
-            holder.remove( 1 );
-        }
-
-        ThreadLocalValue<int> _tl;
-    } lastErrorHolderTest;
-
 } // namespace mongo
--- a/db/lasterror.h
+++ b/db/lasterror.h
@@ -100,14 +100,14 @@ namespace mongo {

    extern class LastErrorHolder {
    public:
-        LastErrorHolder() : _id( 0 ) {}
+        LastErrorHolder(){}
        ~LastErrorHolder();

        LastError * get( bool create = false );
        LastError * getSafe() {
            LastError * le = get(false);
            if ( ! le ) {
-                log( LL_ERROR ) << " no LastError!  id: " << getID() << endl;
+                error() << " no LastError!" << endl;
                assert( le );
            }
            return le;
@@ -120,18 +120,12 @@ namespace mongo {
        /** ok to call more than once. */
        void initThread();

-        /**
-         * id of 0 means should use thread local management
-         */
-        void setID( int id );
        int getID();
-
-        void remove( int id );
+        
        void release();

        /** when db receives a message/request, call this */
-        void startRequest( Message& m , LastError * connectionOwned );
-        LastError * startRequest( Message& m , int clientId );
+        LastError * startRequest( Message& m , LastError * connectionOwned );

        void disconnect( int clientId );

@@ -139,17 +133,12 @@ namespace mongo {
        // disable causes get() to return 0.
        LastError *disableForCommand(); // only call once per command invocation!
    private:
-        ThreadLocalValue<int> _id;
        boost::thread_specific_ptr<LastError> _tl;

        struct Status {
            time_t time;
            LastError *lerr;
        };
-        typedef map<int,Status> IDMap;
-
-        static mongo::mutex _idsmutex;
-        IDMap _ids;
    } lastError;

    void raiseError(int code , const char *msg);
--- a/db/mongommf.cpp
+++ b/db/mongommf.cpp
@@ -342,11 +342,7 @@ namespace mongo {

    bool MongoMMF::create(string fname, unsigned long long& len, bool sequentialHint) {
        setPath(fname);
-        bool preExisting = MemoryMappedFile::exists(fname.c_str());
        _view_write = map(fname.c_str(), len, sequentialHint ? SEQUENTIAL : 0);
-        if( cmdLine.dur && _view_write && !preExisting ) {
-            getDur().createdFile(fname, len);
-        }
        return finishOpening();
    }

--- a/db/namespace.cpp
+++ b/db/namespace.cpp
@@ -182,6 +182,7 @@ namespace mongo {
            maybeMkdir();
            unsigned long long l = lenForNewNsFiles;
            if( f.create(pathString, l, true) ) {
+                getDur().createdFile(pathString, l); // always a new file
                len = l;
                assert( len == lenForNewNsFiles );
                p = f.getView();
@@ -194,6 +195,7 @@ namespace mongo {
            dbexit( EXIT_FS );
        }

+
        assert( len <= 0x7fffffff );
        ht = new HashTable<Namespace,NamespaceDetails>(p, (int) len, "namespace index");
        if( checkNsFilesOnLoad )
--- a/db/pdfile.cpp
+++ b/db/pdfile.cpp
@@ -252,6 +252,10 @@ namespace mongo {
            while ( size > 0 ) {
                int max = MongoDataFile::maxSize() - DataFileHeader::HeaderSize;
                int desiredExtentSize = (int) (size > max ? max : size);
+                if ( desiredExtentSize < Extent::minSize() ) {
+                    desiredExtentSize = Extent::minSize();
+                }
+                desiredExtentSize &= 0xffffff00;
                Extent *e = database->allocExtent( ns, desiredExtentSize, newCapped );
                size -= e->length;
            }
@@ -403,7 +407,7 @@ namespace mongo {
            uassert( 10084 , "can't map file memory - mongo requires 64 bit build for larger datasets", _mb != 0);
        else
            uassert( 10085 , "can't map file memory", _mb != 0);
-        header()->init(fileNo, size);
+        header()->init(fileNo, size, filename);
    }

    void MongoDataFile::flush( bool sync ) {
@@ -435,11 +439,11 @@ namespace mongo {

    Extent* MongoDataFile::createExtent(const char *ns, int approxSize, bool newCapped, int loops) {
        massert( 10357 ,  "shutdown in progress", ! inShutdown() );
-        massert( 10358 ,  "bad new extent size", approxSize >= 0 && approxSize <= Extent::maxSize() );
+        massert( 10358 ,  "bad new extent size", approxSize >= Extent::minSize() && approxSize <= Extent::maxSize() );
        massert( 10359 ,  "header==0 on new extent: 32 bit mmap space exceeded?", header() ); // null if file open failed
        int ExtentSize = approxSize <= header()->unusedLength ? approxSize : header()->unusedLength;
        DiskLoc loc;
-        if ( ExtentSize <= 0 ) {
+        if ( ExtentSize < Extent::minSize() ) {
            /* not there could be a lot of looping here is db just started and
               no files are open yet.  we might want to do something about that. */
            if ( loops > 8 ) {
@@ -1187,13 +1191,17 @@ namespace mongo {
            op->setMessage( "index: (3/3) btree-middle" );
            log(t.seconds() > 10 ? 0 : 1 ) << "\t done building bottom layer, going to commit" << endl;
            btBuilder.commit();
-            wassert( btBuilder.getn() == nkeys || dropDups );
+            if ( btBuilder.getn() != nkeys && ! dropDups ) {
+                warning() << "not all entries were added to the index, probably some keys were too large" << endl;
+            }
        }

        log(1) << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl;

-        for( list<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ )
+        for( list<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ ){
            theDataFileMgr.deleteRecord( ns, i->rec(), *i, false, true );
+            getDur().commitIfNeeded();
+        }

        return n;
    }
@@ -1248,6 +1256,8 @@ namespace mongo {
                n++;
                progress.hit();

+                getDur().commitIfNeeded();
+
                if ( n % 128 == 0 && !cc->yield() ) {
                    cc.release();
                    uasserted(12584, "cursor gone during bg index");
@@ -1281,7 +1291,7 @@ namespace mongo {
            prep(ns.c_str(), d);
            assert( idxNo == d->nIndexes );
            try {
-                idx.head = BtreeBucket::addBucket(idx);
+                idx.head.writing() = BtreeBucket::addBucket(idx);
                n = addExistingToIndex(ns.c_str(), d, idx, idxNo);
            }
            catch(...) {
--- a/db/pdfile.h
+++ b/db/pdfile.h
@@ -272,6 +272,7 @@ namespace mongo {
        Extent* getPrevExtent() { return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev); }

        static int maxSize();
+        static int minSize() { return 0x100; }
        /**
         * @param len lengt of record we need
         * @param lastRecord size of last extent which is a factor in next extent size
@@ -325,11 +326,12 @@ namespace mongo {

        bool uninitialized() const { return version == 0; }

-        void init(int fileno, int filelength) {
+        void init(int fileno, int filelength, const char* filename) {
            if ( uninitialized() ) {
                if( !(filelength > 32768 ) ) { 
                    massert(13640, str::stream() << "DataFileHeader looks corrupt at file open filelength:" << filelength << " fileno:" << fileno, false);
                }
+                getDur().createdFile(filename, filelength);
                assert( HeaderSize == 8192 );
                DataFileHeader *h = getDur().writing(this);
                h->fileLength = filelength;
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -76,8 +76,7 @@ namespace mongo {
            }
        }
        virtual long long nscanned() {
-            assert( c_.get() );
-            return c_->nscanned();
+            return c_.get() ? c_->nscanned() : _nscanned;
        }
        virtual void next() {
            if ( !c_->ok() ) {
@@ -380,8 +379,16 @@ namespace mongo {
                    }
                }
                c->advance();
-            }

+                if ( ! cc->yieldSometimes() ) {
+                    ClientCursor::erase(cursorid);
+                    cursorid = 0;
+                    cc = 0;
+                    p.deleted();
+                    break;
+                }
+            }
+            
            if ( cc ) {
                cc->updateLocation();
                cc->mayUpgradeStorage();
@@ -408,6 +415,7 @@ namespace mongo {
            _ns(ns), _capped(false), _count(), _myCount(),
            _skip( spec["skip"].numberLong() ),
            _limit( spec["limit"].numberLong() ),
+            _nscanned(),
            _bc() {
        }

@@ -422,19 +430,22 @@ namespace mongo {
        }

        virtual long long nscanned() {
-            assert( _c.get() );
-            return _c->nscanned();
+            return _c.get() ? _c->nscanned() : _nscanned;
        }

        virtual bool prepareToYield() {
-            if ( ! _cc ) {
+            if ( _c && !_cc ) {
                _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , _ns.c_str() ) );
            }
-            return _cc->prepareToYield( _yieldData );
+            if ( _cc ) {
+	            return _cc->prepareToYield( _yieldData );
+            }
+            // no active cursor - ok to yield
+            return true;
        }

        virtual void recoverFromYield() {
-            if ( !ClientCursor::recoverFromYield( _yieldData ) ) {
+            if ( _cc && !ClientCursor::recoverFromYield( _yieldData ) ) {
                _c.reset();
                _cc.reset();

@@ -453,6 +464,7 @@ namespace mongo {
                return;
            }

+            _nscanned = _c->nscanned();
            if ( _bc ) {
                if ( _firstMatch.isEmpty() ) {
                    _firstMatch = _bc->currKeyNode().key.copy();
@@ -515,6 +527,7 @@ namespace mongo {
        long long _myCount;
        long long _skip;
        long long _limit;
+        long long _nscanned;
        shared_ptr<Cursor> _c;
        BSONObj _query;
        BtreeCursor * _bc;
@@ -690,11 +703,15 @@ namespace mongo {
                return _findingStartCursor->prepareToYield();
            }
            else {
-                if ( ! _cc ) {
+                if ( _c && !_cc ) {
                    _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , _pq.ns() ) );
                }
-                return _cc->prepareToYield( _yieldData );
+                if ( _cc ) {
+	                return _cc->prepareToYield( _yieldData );
+                }
            }
+            // no active cursor - ok to yield
+            return true;
        }

        virtual void recoverFromYield() {
@@ -703,7 +720,7 @@ namespace mongo {
            if ( _findingStartCursor.get() ) {
                _findingStartCursor->recoverFromYield();
            }
-            else if ( ! ClientCursor::recoverFromYield( _yieldData ) ) {
+            else if ( _cc && !ClientCursor::recoverFromYield( _yieldData ) ) {
                _c.reset();
                _cc.reset();
                _so.reset();
@@ -724,8 +741,7 @@ namespace mongo {
            if ( _findingStartCursor.get() ) {
                return 0; // should only be one query plan, so value doesn't really matter.
            }
-            assert( _c.get() );
-            return _c->nscanned();
+            return _c.get() ? _c->nscanned() : _nscanned;
        }

        virtual void next() {
@@ -842,6 +858,7 @@ namespace mongo {

        // this plan won, so set data for response broadly
        void finish( bool stop ) {
+            massert( 13638, "client cursor dropped during explain query yield", !_pq.isExplain() || _c.get() );

            if ( _pq.isExplain() ) {
                _n = _inMemSort ? _so->size() : _n;
@@ -863,7 +880,6 @@ namespace mongo {
            }

            if ( _pq.isExplain() ) {
-                massert( 13638, "client cursor dropped during explain query yield", _c.get() );
                _eb.noteScan( _c.get(), _nscanned, _nscannedObjects, _n, scanAndOrderRequired(),
                              _curop.elapsedMillis(), useHints && !_pq.getHint().eoo(), _nYields ,
                              _nChunkSkips, _keyFieldsOnly.get() > 0 );
--- a/db/repl.cpp
+++ b/db/repl.cpp
@@ -1426,6 +1426,8 @@ namespace mongo {
                        break;
                    }
                    op = oplogReader.next();
+
+                    getDur().commitIfNeeded();
                }
            }
        }
--- a/db/repl/rs.cpp
+++ b/db/repl/rs.cpp
@@ -23,6 +23,7 @@
 #include "../../s/d_logic.h"
 #include "rs.h"
 #include "connections.h"
+#include "../repl.h"

 namespace mongo {

@@ -380,6 +381,10 @@ namespace mongo {
                conn.setTimeout(c.ho.heartbeatTimeoutMillis/1000.0);
            }
            if( me == 0 ) {
+                // initial startup with fastsync
+                if (!reconf && replSettings.fastsync) {
+                    return false;
+                }
                // log() << "replSet config : " << _cfg->toString() << rsLog;
                log() << "replSet error self not present in the repl set configuration:" << rsLog;
                log() << c.toString() << rsLog;
--- a/db/repl/rs_config.cpp
+++ b/db/repl/rs_config.cpp
@@ -314,12 +314,12 @@ namespace mongo {
                }
                if( m.h.isLocalHost() )
                    localhosts++;
-                m.arbiterOnly = mobj.getBoolField("arbiterOnly");
+                m.arbiterOnly = mobj["arbiterOnly"].trueValue();
                m.slaveDelay = mobj["slaveDelay"].numberInt();
                if( mobj.hasElement("hidden") )
-                    m.hidden = mobj.getBoolField("hidden");
+                    m.hidden = mobj["hidden"].trueValue();
                if( mobj.hasElement("buildIndexes") )
-                    m.buildIndexes = mobj.getBoolField("buildIndexes");
+                    m.buildIndexes = mobj["buildIndexes"].trueValue();
                if( mobj.hasElement("priority") )
                    m.priority = mobj["priority"].Number();
                if( mobj.hasElement("votes") )
--- a/db/repl/rs_rollback.cpp
+++ b/db/repl/rs_rollback.cpp
@@ -473,6 +473,8 @@ namespace mongo {
                    continue;
                }

+                getDur().commitIfNeeded();
+
                /* keep an archive of items rolled back */
                shared_ptr<RemoveSaver>& rs = removeSavers[d.ns];
                if ( ! rs )
--- a/db/repl/rs_sync.cpp
+++ b/db/repl/rs_sync.cpp
@@ -143,6 +143,8 @@ namespace mongo {
                    // simple progress metering
                    log() << "replSet initialSyncOplogApplication " << n << rsLog;
                }
+                
+                getDur().commitIfNeeded();
            }
            catch (DBException& e) {
                if( e.getCode() == 11000 || e.getCode() == 11001 ) {
--- a/db/stats/top.cpp
+++ b/db/stats/top.cpp
@@ -44,6 +44,9 @@ namespace mongo {
    }

    void Top::record( const string& ns , int op , int lockType , long long micros , bool command ) {
+        if ( ns[0] == '?' )
+            return;
+
        //cout << "record: " << ns << "\t" << op << "\t" << command << endl;
        scoped_lock lk(_lock);

--- a/db/update.cpp
+++ b/db/update.cpp
@@ -913,15 +913,14 @@ namespace mongo {
            }
        }
        virtual long long nscanned() {
-            assert( _c.get() );
-            return _c->nscanned();
+            return _c.get() ? _c->nscanned() : _nscanned;
        }
        virtual void next() {
            if ( ! _c->ok() ) {
                setComplete();
                return;
            }
-            _nscanned++;
+            _nscanned = _c->nscanned();
            if ( matcher()->matches(_c->currKey(), _c->currLoc(), &_details ) ) {
                setComplete();
                return;
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,33 @@
+mongodb (1.8.1) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10260
+
+ -- Richard Kreuter <richard@10gen.com>  Thu, 6 Apr 2011 16:56:28 -0500
+
+mongodb (1.8.1-rc1) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10260
+
+ -- Richard Kreuter <richard@10gen.com>  Thu, 31 Mar 2011 16:56:28 -0500
+
+mongodb (1.8.1-rc0) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10260
+
+ -- Richard Kreuter <richard@10gen.com>  Thu, 24 Mar 2011 16:56:28 -0500
+
+mongodb (1.8.0) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10256
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 16 Mar 2011 16:56:28 -0500
+
+mongodb (1.8.0-rc1) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10246
+
+ -- Richard Kreuter <richard@10gen.com>  Tue, 1 Mar 2011 16:56:28 -0500
+
 mongodb (1.8.0-rc0) unstable; urgency=low

  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10245
--- a/2
+++ b/2
@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 1.8.0-rc0
+PROJECT_NUMBER         = 1.8.1
 OUTPUT_DIRECTORY       = docs/doxygen
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English
--- a/jstests/check_shard_index.js
+++ b/jstests/check_shard_index.js
@@ -32,9 +32,14 @@ f.ensureIndex( { x: 1 , y: 1 } );
 assert.eq( 0 , f.count() , "2. initial count should be zero" );

 f.save( { x: 1 , y : 1 } );
-f.save( { y: 2 } );
-assert.eq( 2 , f.count() , "2. count after initial insert should be 2" );
+f.save( { x: null , y : 1 } );
+
 res = db.runCommand( { checkShardingIndex: "test.jstests_shardingindex" , keyPattern: {x:1, y:1} , force: true });
-assert.eq( false , res.ok , "2a" );
+assert.eq( true , res.ok , "2a " + tojson(res) );
+
+f.save( { y: 2 } );
+assert.eq( 3 , f.count() , "2. count after initial insert should be 3" );
+res = db.runCommand( { checkShardingIndex: "test.jstests_shardingindex" , keyPattern: {x:1, y:1} , force: true });
+assert.eq( false , res.ok , "2b " + tojson(res) );

 print("PASSED");
--- a/jstests/index11.js
+++ b/jstests/index11.js
@@ -5,12 +5,9 @@ coll.drop();

 coll.ensureIndex({"k": 1, "v": 1});
 coll.insert({k: "x", v: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"});
-/* TODO inconsistent behavior w/ too large to index - this fails but the next assert works.
- * SERVER-1716
- * assert.eq(1, coll.find({"k": "x"}).count());
- */
+assert.eq(0, coll.find({"k": "x"}).count()); // SERVER-1716

 coll.dropIndexes();
 coll.ensureIndex({"k": 1, "v": 1});

-assert.eq(1, coll.find({"k": "x"}).count());
+assert.eq(0, coll.find({"k": "x"}).count());
--- a/jstests/replsets/fastsync.js
+++ b/jstests/replsets/fastsync.js
@@ -73,21 +73,7 @@ print("total in foo: "+foo.bar.count());

 print("2");
 admin.runCommand( {fsync:1,lock:1} );
-
 copyDbpath( basePath + "-p", basePath + "-s" );
-
-print("remove local files or slave will get confused");
-var files = listFiles(basePath+"-s");
-
-for (var i in files) {
-  var filename = files[i].name;
-  if (filename.match("/local")) {
-    print("removing "+filename);
-    removeFile(filename);
-  }
-}
-//run("rm", basePath+"-s/local.*");
-    
 admin.$cmd.sys.unlock.findOne();


--- a/jstests/replsets/replset5.js
+++ b/jstests/replsets/replset5.js
@@ -49,6 +49,23 @@ doTest = function (signal) {
    var master1count = master.getDB(testDB).foo.count();
    assert( master1count == docNum, "Master has " + master1count + " of " + docNum + " documents!");

+    print("reconfigure with hidden=1");
+    config = master.getDB("local").system.replset.findOne();
+    config.version++;
+    config.members[2].priority = 0;
+    config.members[2].hidden = 1;
+
+    try {
+        master.adminCommand({replSetReconfig : config});
+    }
+    catch(e) {
+        print(e);
+    }
+
+    config = master.getDB("local").system.replset.findOne();
+    printjson(config);
+    assert.eq(config.members[2].hidden, true);
+
    replTest.stopSet(signal);
 }

--- a/jstests/sharding/features1.js
+++ b/jstests/sharding/features1.js
@@ -175,7 +175,7 @@ assert( ! s.admin.runCommand( { shardcollection : "test.foo9" , key : { a : 1 }
 // --- listDatabases ---

 r = db.getMongo().getDBs()
-assert.eq( 4 , r.databases.length , "listDatabases 1 : " + tojson( r ) )
+assert.eq( 3 , r.databases.length , "listDatabases 1 : " + tojson( r ) )
 assert.lt( 10000 , r.totalSize , "listDatabases 2 : " + tojson( r ) );

 s.stop()
--- a/jstests/slowNightly/explain1.js
+++ b/jstests/slowNightly/explain1.js
@@ -0,0 +1,17 @@
+// SERVER-2662 - drop client cursor in a context where query will yield frequently
+
+t = db.jstests_slowNightly_explain1;
+t.drop();
+
+// Periodically drops the collection, invalidating client cursors for s2's operations.
+s1 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 80; ++i ) { t.drop(); t.ensureIndex({x:1}); for( var j = 0; j < 1000; ++j ) { t.save( {x:j,y:1} ) }; sleep( 100 ); }" );
+
+// Query repeatedly.
+s2 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 500; ++i ) { try { z = t.find( {x:{$gt:0},y:1} ).explain(); t.count( {x:{$gt:0},y:1} ); } catch( e ) {} }" );
+
+// Put pressure on s2 to yield more often.
+s3 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 200; ++i ) { t.validate({scandata:true}); }" );
+
+s1();
+s2();
+s3();
--- a/jstests/slowNightly/explain2.js
+++ b/jstests/slowNightly/explain2.js
@@ -0,0 +1,18 @@
+// Test for race condition SERVER-2807.  One cursor is dropped and another is not.
+
+collName = 'jstests_slowNightly_explain2';
+
+t = db[ collName ];
+t.drop();
+
+db.createCollection( collName, {capped:true,size:100000} );
+t = db[ collName ];
+t.ensureIndex( {x:1} );
+
+a = startParallelShell( 'for( i = 0; i < 50000; ++i ) { db.' + collName + '.insert( {x:i,y:1} ); }' );
+
+for( i = 0; i < 800; ++i ) {
+    t.find( {x:{$gt:-1},y:1} ).sort({x:-1}).explain();
+}
+
+a();
--- a/jstests/slowNightly/explain3.js
+++ b/jstests/slowNightly/explain3.js
@@ -0,0 +1,17 @@
+// SERVER-2810 - similar to explain1 test, but with a scan and order find
+
+t = db.jstests_slowNightly_explain3;
+t.drop();
+
+// Periodically drops the collection, invalidating client cursors for s2's operations.
+s1 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 80; ++i ) { t.drop(); t.ensureIndex({x:1}); for( var j = 0; j < 1000; ++j ) { t.save( {x:j,y:1} ) }; sleep( 100 ); }" );
+
+// Query repeatedly.
+s2 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 500; ++i ) { try { z = t.find( {x:{$gt:0},y:1} ).sort({x:1}).explain(); } catch( e ) {} }" );
+
+// Put pressure on s2 to yield more often.
+s3 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 200; ++i ) { t.validate({scandata:true}); }" );
+
+s1();
+s2();
+s3();
--- a/jstests/slowNightly/newcollection2.js
+++ b/jstests/slowNightly/newcollection2.js
@@ -0,0 +1,11 @@
+// Alocate collection forcing just a small size remainder in 2nd extent
+
+port = allocatePorts( 1 )[ 0 ]
+var baseName = "jstests_disk_newcollection2";
+var m = startMongod( "--noprealloc", "--smallfiles", "--port", port, "--dbpath", "/data/db/" + baseName );
+db = m.getDB( "test" );
+
+db.createCollection( baseName, {size:0x1FFC0000-0x10-8192} );
+var v = db[ baseName ].validate();
+printjson( v );
+assert( v.valid );
--- a/jstests/slowNightly/sharding_rs1.js
+++ b/jstests/slowNightly/sharding_rs1.js
@@ -43,6 +43,15 @@ function diff(){
 assert.lt( 20 , diff() , "big differential here" );
 print( diff() )

+{
+    // quick test for SERVER-2686
+    var mydbs = db.getMongo().getDBs().databases;
+    for ( var i=0; i<mydbs.length; i++ ) {
+        assert( mydbs[i].name != "local" , "mongos listDatabases can't return local" );
+    }
+}
+
+
 assert.soon( function(){
    var d = diff();
    return d < 5;
--- a/jstests/slowWeekly/rollback4.js
+++ b/jstests/slowWeekly/rollback4.js
@@ -0,0 +1,50 @@
+// Test a large rollback SERVER-2737
+
+var replTest = new ReplSetTest({ name: 'unicomplex', nodes: 3, oplogSize: 2000 });
+var nodes = replTest.nodeList();
+
+var conns = replTest.startSet();
+var r = replTest.initiate({ "_id": "unicomplex",
+                          "members": [
+                                      { "_id": 0, "host": nodes[0] },
+                                      { "_id": 1, "host": nodes[1] },
+                                      { "_id": 2, "host": nodes[2], arbiterOnly: true}]
+                          });
+
+// Make sure we have a master
+var master = replTest.getMaster();
+b_conn = conns[1];
+b_conn.setSlaveOk();
+B = b_conn.getDB("admin");
+
+// Make sure we have an arbiter
+assert.soon(function () {
+            res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 });
+            return res.myState == 7;
+            }, "Arbiter failed to initialize.");
+
+// Wait for initial replication
+replTest.awaitReplication();
+
+// Insert into master
+var big = { b:new Array( 1000 ).toString() };
+for( var i = 0; i < 1000000; ++i ) {
+    if ( i % 10000 == 0 ) {
+        print( i );
+    }
+	master.getDB( 'db' ).c.insert( big );
+}
+
+// Stop master
+replTest.stop( 0 );
+
+// Wait for slave to take over
+assert.soon(function () { return B.isMaster().ismaster; });
+master = replTest.getMaster();
+
+// Save to new master, forcing rollback of old master
+master.getDB( 'db' ).c.save( big );
+
+// Restart old master
+replTest.restart( 0 );
+replTest.awaitReplication();
--- a/pch.h
+++ b/pch.h
@@ -72,6 +72,7 @@
 #include <boost/any.hpp>
 #include "boost/thread/once.hpp"
 #include <boost/archive/iterators/transform_width.hpp>
+#define BOOST_FILESYSTEM_VERSION 2
 #include <boost/filesystem/convenience.hpp>
 #include <boost/filesystem/exception.hpp>
 #include <boost/filesystem/operations.hpp>
--- a/rpm/mongo.spec
+++ b/rpm/mongo.spec
@@ -1,5 +1,5 @@
 Name: mongo
-Version: 1.8.0-rc0
+Version: 1.8.1
 Release: mongodb_1%{?dist}
 Summary: mongo client shell and tools
 License: AGPL 3.0
--- a/s/balance.cpp
+++ b/s/balance.cpp
@@ -147,7 +147,7 @@ namespace mongo {
        auto_ptr<DBClientCursor> cursor = conn.query( ShardNS::collection , BSONObj() );
        vector< string > collections;
        while ( cursor->more() ) {
-            BSONObj col = cursor->next();
+            BSONObj col = cursor->nextSafe();

            // sharded collections will have a shard "key".
            if ( ! col["key"].eoo() )
@@ -199,7 +199,7 @@ namespace mongo {
            map< string,vector<BSONObj> > shardToChunksMap;
            cursor = conn.query( ShardNS::chunk , QUERY( "ns" << ns ).sort( "min" ) );
            while ( cursor->more() ) {
-                BSONObj chunk = cursor->next();
+                BSONObj chunk = cursor->nextSafe();
                vector<BSONObj>& chunks = shardToChunksMap[chunk["shard"].String()];
                chunks.push_back( chunk.getOwned() );
            }
@@ -275,6 +275,15 @@ namespace mongo {
        while ( ! inShutdown() ) {

            try {
+                
+                // first make sure we should even be running
+                if ( ! grid.shouldBalance() ) {
+                    log(1) << "skipping balancing round because balancing is disabled" << endl;
+                    sleepsecs( 30 );
+                    continue;
+                }
+                
+
                ScopedDbConnection conn( config );

                _ping( conn.conn() );
@@ -294,14 +303,6 @@ namespace mongo {
                    continue;
                }

-                if ( ! grid.shouldBalance() ) {
-                    log(1) << "skipping balancing round because balancing is disabled" << endl;;
-                    conn.done();
-
-                    sleepsecs( 30 );
-                    continue;
-                }
-
                log(1) << "*** start balancing round" << endl;

                vector<CandidateChunkPtr> candidateChunks;
--- a/s/balancer_policy.cpp
+++ b/s/balancer_policy.cpp
@@ -155,7 +155,7 @@ namespace mongo {

    bool BalancerPolicy::isDraining( BSONObj limits ) {
        BSONElement draining = limits[ ShardFields::draining.name() ];
-        if ( draining.eoo() || ! draining.Bool() ) {
+        if ( draining.eoo() || ! draining.trueValue() ) {
            return false;
        }

@@ -164,7 +164,7 @@ namespace mongo {

    bool BalancerPolicy::hasOpsQueued( BSONObj limits ) {
        BSONElement opsQueued = limits[ LimitsFields::hasOpsQueued.name() ];
-        if ( opsQueued.eoo() || ! opsQueued.Bool() ) {
+        if ( opsQueued.eoo() || ! opsQueued.trueValue() ) {
            return false;
        }
        return true;
--- a/s/chunk.cpp
+++ b/s/chunk.cpp
@@ -45,6 +45,8 @@ namespace mongo {
    string Chunk::chunkMetadataNS = "config.chunks";

    int Chunk::MaxChunkSize = 1024 * 1024 * 64;
+    int Chunk::MaxObjectPerChunk = 250000;
+    

    Chunk::Chunk( ChunkManager * manager ) : _manager(manager), _lastmod(0) {
        _setDataWritten();
@@ -182,8 +184,7 @@ namespace mongo {
        if ( ! force ) {
            vector<BSONObj> candidates;
            const int maxPoints = 2;
-            const int maxObjs = 250000;
-            pickSplitVector( candidates , getManager()->getCurrentDesiredChunkSize() , maxPoints , maxObjs );
+            pickSplitVector( candidates , getManager()->getCurrentDesiredChunkSize() , maxPoints , MaxObjectPerChunk );
            if ( candidates.size() <= 1 ) {
                // no split points means there isn't enough data to split on
                // 1 split point means we have between half the chunk size to full chunk size
--- a/s/chunk.h
+++ b/s/chunk.h
@@ -174,7 +174,7 @@ namespace mongo {

        static string chunkMetadataNS;
        static int MaxChunkSize;
-
+        static int MaxObjectPerChunk;
        //
        // accessors and helpers
        //
@@ -189,10 +189,11 @@ namespace mongo {
        const char * getNS() { return "config.chunks"; }
        Shard getShard() const { return _shard; }
        const ChunkManager* getManager() const { return _manager; }
+        

    private:
        // main shard info
-
+        
        ChunkManager * _manager;

        BSONObj _min;
--- a/s/client.cpp
+++ b/s/client.cpp
@@ -36,7 +36,7 @@

 namespace mongo {

-    ClientInfo::ClientInfo( int clientId ) : _id( clientId ) {
+    ClientInfo::ClientInfo() {
        _cur = &_a;
        _prev = &_b;
        _autoSplitOk = true;
@@ -44,13 +44,6 @@ namespace mongo {
    }

    ClientInfo::~ClientInfo() {
-        if ( _lastAccess ) {
-            scoped_lock lk( _clientsLock );
-            Cache::iterator i = _clients.find( _id );
-            if ( i != _clients.end() ) {
-                _clients.erase( i );
-            }
-        }
    }

    void ClientInfo::addShard( const string& shard ) {
@@ -79,49 +72,19 @@ namespace mongo {
        _cur->clear();
    }

-    void ClientInfo::disconnect() {
-        _lastAccess = 0;
-    }
-
-    ClientInfo * ClientInfo::get( int clientId , bool create ) {
-
-        if ( ! clientId )
-            clientId = getClientId();
-
-        if ( ! clientId ) {
-            ClientInfo * info = _tlInfo.get();
-            if ( ! info ) {
-                info = new ClientInfo( 0 );
-                _tlInfo.reset( info );
-            }
+    ClientInfo * ClientInfo::get() {
+        ClientInfo * info = _tlInfo.get();
+        if ( ! info ) {
+            info = new ClientInfo();
+            _tlInfo.reset( info );
            info->newRequest();
-            return info;
        }
-
-        scoped_lock lk( _clientsLock );
-        Cache::iterator i = _clients.find( clientId );
-        if ( i != _clients.end() )
-            return i->second;
-        if ( ! create )
-            return 0;
-        ClientInfo * info = new ClientInfo( clientId );
-        _clients[clientId] = info;
        return info;
    }

-    void ClientInfo::disconnect( int clientId ) {
-        if ( ! clientId )
-            return;
-
-        scoped_lock lk( _clientsLock );
-        Cache::iterator i = _clients.find( clientId );
-        if ( i == _clients.end() )
-            return;
-
-        ClientInfo* ci = i->second;
-        ci->disconnect();
-        delete ci;
-        _clients.erase( i );
+    void ClientInfo::disconnect() {
+        // should be handled by TL cleanup
+        _lastAccess = 0;
    }

    void ClientInfo::_addWriteBack( vector<WBInfo>& all , const BSONObj& gle ) {
@@ -142,14 +105,14 @@ namespace mongo {

    vector<BSONObj> ClientInfo::_handleWriteBacks( vector<WBInfo>& all , bool fromWriteBackListener ) {
        vector<BSONObj> res;
+
+        if ( all.size() == 0 )
+            return res;
        
        if ( fromWriteBackListener ) {
            LOG(1) << "not doing recusrive writeback" << endl;
            return res;
        }
-
-        if ( all.size() == 0 )
-            return res;
        
        for ( unsigned i=0; i<all.size(); i++ ) {
            res.push_back( WriteBackListener::waitFor( all[i].connectionId , all[i].id ) );
@@ -177,7 +140,21 @@ namespace mongo {
            ShardConnection conn( theShard , "" );
            
            BSONObj res;
-            bool ok = conn->runCommand( "admin" , options , res );
+            bool ok = false;
+            try{
+            	ok = conn->runCommand( "admin" , options , res );
+            }
+            catch( std::exception &e ){
+                
+                warning() << "Could not get last error." << e.what() << endl;
+                
+                // Catch everything that happens here, since we need to ensure we return our connection when we're
+            	// finished.
+            	conn.done();
+                
+            	return false;
+            }
+            
            res = res.getOwned();
            conn.done();
            
@@ -205,6 +182,7 @@ namespace mongo {
                    assert( v.size() == 1 );
                    result.appendElements( v[0] );
                    result.appendElementsUnique( res );
+                    result.append( "writebackGLE" , v[0] );
                    result.append( "initialGLEHost" , theShard );
                }
            }
@@ -217,8 +195,11 @@ namespace mongo {
        }

        BSONArrayBuilder bbb( result.subarrayStart( "shards" ) );
+        BSONObjBuilder shardRawGLE;

        long long n = 0;
+        
+        int updatedExistingStat = 0; // 0 is none, -1 has but false, 1 has true

        // hit each shard
        vector<string> errors;
@@ -228,7 +209,22 @@ namespace mongo {
            bbb.append( theShard );
            ShardConnection conn( theShard , "" );
            BSONObj res;
-            bool ok = conn->runCommand( "admin" , options , res );
+            bool ok = false;
+            try {
+                ok = conn->runCommand( "admin" , options , res );
+                shardRawGLE.append( theShard , res );
+            }
+            catch( std::exception &e ){
+
+        	    // Safe to return here, since we haven't started any extra processing yet, just collecting
+        	    // responses.
+                
+        	    warning() << "Could not get last error." << e.what() << endl;
+                conn.done();
+                
+                return false;
+            }
+            
            _addWriteBack( writebacks, res );
            
            string temp = DBClientWithCommands::getLastErrorString( res );
@@ -236,13 +232,24 @@ namespace mongo {
                errors.push_back( temp );
                errorObjects.push_back( res );
            }
+
            n += res["n"].numberLong();
+            if ( res["updatedExisting"].type() ) {
+                if ( res["updatedExisting"].trueValue() )
+                    updatedExistingStat = 1;
+                else if ( updatedExistingStat == 0 )
+                    updatedExistingStat = -1;
+            }
+
            conn.done();
        }

        bbb.done();
+        result.append( "shardRawGLE" , shardRawGLE.obj() );

        result.appendNumber( "n" , n );
+        if ( updatedExistingStat )
+            result.appendBool( "updatedExisting" , updatedExistingStat > 0 );

        // hit other machines just to block
        for ( set<string>::const_iterator i=sinceLastGetError().begin(); i!=sinceLastGetError().end(); ++i ) {
@@ -285,8 +292,6 @@ namespace mongo {
        return true;
    }

-    ClientInfo::Cache& ClientInfo::_clients = *(new ClientInfo::Cache());
-    mongo::mutex ClientInfo::_clientsLock("_clientsLock");
    boost::thread_specific_ptr<ClientInfo> ClientInfo::_tlInfo;

 } // namespace mongo
--- a/s/client.h
+++ b/s/client.h
@@ -26,11 +26,8 @@ namespace mongo {
     * currently implemented with a thread local
     */
    class ClientInfo {
-
-        typedef map<int,ClientInfo*> Cache;
-
    public:
-        ClientInfo( int clientId );
+        ClientInfo();
        ~ClientInfo();

        /** new request from client, adjusts internal state */
@@ -54,7 +51,7 @@ namespace mongo {
         * gets shards used on the previous request
         */
        set<string> * getPrev() const { return _prev; };
-
+        
        /**
         * gets all shards we've accessed since the last time we called clearSinceLastGetError
         */
@@ -65,6 +62,12 @@ namespace mongo {
         */
        void clearSinceLastGetError() { _sinceLastGetError.clear(); }

+
+        /**
+         * resets the list of shards using to process the current request
+         */
+        void clearCurrentShards(){ _cur->clear(); }
+
        /**
         * calls getLastError
         * resets shards since get last error
@@ -77,8 +80,7 @@ namespace mongo {
        
        void noAutoSplit() { _autoSplitOk = false; }

-        static ClientInfo * get( int clientId = 0 , bool create = true );
-        static void disconnect( int clientId );
+        static ClientInfo * get();

    private:

@@ -111,8 +113,6 @@ namespace mongo {
        int _lastAccess;
        bool _autoSplitOk; 

-        static mongo::mutex _clientsLock;
-        static Cache& _clients;
        static boost::thread_specific_ptr<ClientInfo> _tlInfo;
    };

--- a/s/commands_admin.cpp
+++ b/s/commands_admin.cpp
@@ -534,7 +534,7 @@ namespace mongo {
                    result.append( "cause" , res );
                    return false;
                }
-
+                config->getChunkManager( ns , true );
                return true;
            }
        } splitCollectionCmd;
@@ -959,6 +959,13 @@ namespace mongo {
            BSONArrayBuilder bb( result.subarrayStart( "databases" ) );
            for ( map<string,long long>::iterator i=sizes.begin(); i!=sizes.end(); ++i ) {
                string name = i->first;
+
+                if ( name == "local" ) {
+                    // we don't return local
+                    // since all shards have their own independant local
+                    continue;
+                }
+
                long long size = i->second;
                totalSize += size;

--- a/s/commands_public.cpp
+++ b/s/commands_public.cpp
@@ -545,12 +545,8 @@ namespace mongo {
                bool ok = conn->runCommand( conf->getName() , cmdObj , res );
                conn.done();

-                if (ok || (strcmp(res["errmsg"].valuestrsafe(), "No matching object found") != 0)) {
-                    result.appendElements(res);
-                    return ok;
-                }
-
-                return true;
+                result.appendElements(res);
+                return ok;
            }

        } findAndModifyCmd;
--- a/s/d_migrate.cpp
+++ b/s/d_migrate.cpp
@@ -506,7 +506,6 @@ namespace mongo {
                // use the builder size instead of accumulating 'o's size so that we take into consideration
                // the overhead of BSONArray indices
                if ( a.len() + o.objsize() + 1024 > BSONObjMaxUserSize ) {
-                    i--;
                    break;
                }
                a.append( o );
@@ -1162,7 +1161,7 @@ namespace mongo {
                string system_indexes = cc().database()->name + ".system.indexes";
                for ( unsigned i=0; i<all.size(); i++ ) {
                    BSONObj idx = all[i];
-                    theDataFileMgr.insert( system_indexes.c_str() , idx.objdata() , idx.objsize() );
+                    theDataFileMgr.insertAndLog( system_indexes.c_str() , idx );
                }

                timing.done(1);
--- a/s/d_split.cpp
+++ b/s/d_split.cpp
@@ -178,18 +178,37 @@ namespace mongo {
            }

            // for now, the only check is that all shard keys are filled
+            // null is ok, 
            // TODO if $exist for nulls were picking the index, it could be used instead efficiently
            while ( cc->ok() ) {
                BSONObj currKey = c->currKey();
-                BSONForEach(key, currKey) {
-                    if ( key.type() == jstNULL ) {
-                        ostringstream os;
-                        os << "found null value in key " << bc->prettyKey( currKey );
-                        log() << "checkShardingIndex for '" << ns << "' failed: " << os.str() << endl;
+                
+                BSONObjIterator i( currKey );
+                int n = 0;
+                while ( i.more() ) {
+                    BSONElement key = i.next();
+                    n++;

-                        errmsg = os.str();
-                        return false;
-                    }
+                    if ( key.type() && key.type() != jstNULL )
+                        continue;
+                    
+                    BSONObj obj = c->current();
+                    BSONObjIterator j( keyPattern );
+                    BSONElement real;
+                    for ( int x=0; x<n; x++ )
+                        real = j.next();
+                    
+                    real = obj.getFieldDotted( real.fieldName() );
+
+                    if ( real.type() )
+                        continue;
+                    
+                    ostringstream os;
+                    os << "found null value in key " << bc->prettyKey( currKey ) << " for doc: " << real["_id"];
+                    log() << "checkShardingIndex for '" << ns << "' failed: " << os.str() << endl;
+                    
+                    errmsg = os.str();
+                    return false;
                }
                cc->advance();
            }
@@ -250,7 +269,7 @@ namespace mongo {
                maxSplitPoints = maxSplitPointsElem.numberLong();
            }

-            long long maxChunkObjects = 0;
+            long long maxChunkObjects = Chunk::MaxObjectPerChunk;
            BSONElement MaxChunkObjectsElem = jsobj[ "maxChunkObjects" ];
            if ( MaxChunkObjectsElem.isNumber() ) {
                maxChunkObjects = MaxChunkObjectsElem.numberLong();
@@ -353,48 +372,62 @@ namespace mongo {
                // at the end. If a key appears more times than entries allowed on a chunk, we issue a warning and
                // split on the following key.
                set<BSONObj> tooFrequentKeys;
-                splitKeys.push_back( c->currKey() );
-                while ( cc->ok() ) {
-                    currCount++;
-                    BSONObj currKey = c->currKey();
-                    
-                    DEV assert( currKey.woCompare( max ) <= 0 );
-                    
-                    if ( currCount > keyCount ) {
-                        // Do not use this split key if it is the same used in the previous split point.
-                        if ( currKey.woCompare( splitKeys.back() ) == 0 ) {
-                            tooFrequentKeys.insert( currKey.getOwned() );
+                splitKeys.push_back( c->currKey().getOwned() );
+                while ( 1 ) {
+                    while ( cc->ok() ) {
+                        currCount++;
+                        BSONObj currKey = c->currKey();
+                        
+                        DEV assert( currKey.woCompare( max ) <= 0 );
+                        
+                        if ( currCount > keyCount ) {
+                            // Do not use this split key if it is the same used in the previous split point.
+                            if ( currKey.woCompare( splitKeys.back() ) == 0 ) {
+                                tooFrequentKeys.insert( currKey.getOwned() );
+                                
+                            }
+                            else {
+                                splitKeys.push_back( currKey.getOwned() );
+                                currCount = 0;
+                                numChunks++;
+                                
+                                LOG(4) << "picked a split key: " << bc->prettyKey( currKey ) << endl;
+                            }
                            
                        }
-                        else {
-                            splitKeys.push_back( currKey.getOwned() );
-                            currCount = 0;
-                            numChunks++;
-                            
-                            LOG(4) << "picked a split key: " << bc->prettyKey( currKey ) << endl;
+                        
+                        cc->advance();
+                        
+                        // Stop if we have enough split points.
+                        if ( maxSplitPoints && ( numChunks >= maxSplitPoints ) ) {
+                            log() << "max number of requested split points reached (" << numChunks
+                                  << ") before the end of chunk " << ns << " " << min << " -->> " << max
+                                  << endl;
+                            break;
                        }
                        
-                    }
-
-                    cc->advance();
-                    
-                    // Stop if we have enough split points.
-                    if ( maxSplitPoints && ( numChunks >= maxSplitPoints ) ) {
-                        log() << "max number of requested split points reached (" << numChunks
-                              << ") before the end of chunk " << ns << " " << min << " -->> " << max
-                              << endl;
-                        break;
+                        if ( ! cc->yieldSometimes() ) {
+                            // we were near and and got pushed to the end
+                            // i think returning the splits we've already found is fine
+                            
+                            // don't use the btree cursor pointer to acces keys beyond this point but ok
+                            // to use it for format the keys we've got already
+                            
+                            break;
+                        }
                    }
                    
-                    if ( ! cc->yieldSometimes() ) {
-                        // we were near and and got pushed to the end
-                        // i think returning the splits we've already found is fine
-                        
-                        // don't use the btree cursor pointer to acces keys beyond this point but ok
-                        // to use it for format the keys we've got already
-                        
+                    if ( splitKeys.size() > 1 || ! force )
                        break;
-                    }
+                    
+                    force = false;
+                    keyCount = currCount / 2;
+                    currCount = 0;
+                    log() << "splitVector doing another cycle because of force, keyCount now: " << keyCount << endl;
+                    
+                    bc = new BtreeCursor( d , d->idxNo(*idx) , *idx , min , max , false , 1 );
+                    c.reset( bc );
+                    cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , c , ns ) );
                }
                
                //
@@ -410,6 +443,7 @@ namespace mongo {
                
                // Remove the sentinel at the beginning before returning and add fieldnames.
                splitKeys.erase( splitKeys.begin() );
+                assert( c.get() );
                for ( vector<BSONObj>::iterator it = splitKeys.begin(); it != splitKeys.end() ; ++it ) {
                    *it = bc->prettyKey( *it );
                }
--- a/s/d_state.cpp
+++ b/s/d_state.cpp
@@ -60,6 +60,7 @@ namespace mongo {
    }

    void ShardingState::gotShardName( const string& name ) {
+        scoped_lock lk(_mutex);
        if ( _shardName.size() == 0 ) {
            // TODO SERVER-2299 verify the name is sound w.r.t IPs
            _shardName = name;
@@ -78,7 +79,7 @@ namespace mongo {
    }

    void ShardingState::gotShardHost( string host ) {
-
+        scoped_lock lk(_mutex);
        size_t slash = host.find( '/' );
        if ( slash != string::npos )
            host = host.substr( 0 , slash );
@@ -386,79 +387,128 @@ namespace mongo {
            help << " example: { setShardVersion : 'alleyinsider.foo' , version : 1 , configdb : '' } ";
        }

-        virtual LockType locktype() const { return WRITE; } // TODO: figure out how to make this not need to lock
+        virtual LockType locktype() const { return NONE; }
+        
+        bool checkConfigOrInit( const string& configdb , bool authoritative , string& errmsg , BSONObjBuilder& result , bool locked=false ) const {
+            if ( configdb.size() == 0 ) {
+                errmsg = "no configdb";
+                return false;
+            }
+            
+            if ( shardingState.enabled() ) {
+                if ( configdb == shardingState.getConfigServer() ) 
+                    return true;
+                
+                result.append( "configdb" , BSON( "stored" << shardingState.getConfigServer() << 
+                                                  "given" << configdb ) );
+                errmsg = "specified a different configdb!";
+                return false;
+            }
+            
+            if ( ! authoritative ) {
+                result.appendBool( "need_authoritative" , true );
+                errmsg = "first setShardVersion";
+                return false;
+            }
+            
+            if ( locked ) {
+                shardingState.enable( configdb );
+                configServer.init( configdb );
+                return true;
+            }
+
+            dblock lk;
+            return checkConfigOrInit( configdb , authoritative , errmsg , result , true );
+        }
+        
+        bool checkMongosID( ShardedConnectionInfo* info, const BSONElement& id, string errmsg ) {
+            if ( id.type() != jstOID ) {
+                // TODO: fix this
+                //errmsg = "need serverID to be an OID";
+                //return 0;
+                return true;
+            }
+            
+            OID clientId = id.__oid();
+            if ( ! info->hasID() ) {
+                info->setID( clientId );
+                return true;
+            }
+            
+            if ( clientId != info->getID() ) {
+                errmsg = "server id has changed!";
+                return false;
+            }
+
+            return true;
+        }

        bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {

+            // Steps
+            // 1. check basic config
+            // 2. extract params from command
+            // 3. fast check
+            // 4. slow check (LOCKS)
+            
+            // step 1
+
            lastError.disableForCommand();
            ShardedConnectionInfo* info = ShardedConnectionInfo::get( true );

            bool authoritative = cmdObj.getBoolField( "authoritative" );
+            
+            // check config server is ok or enable sharding
+            if ( ! checkConfigOrInit( cmdObj["configdb"].valuestrsafe() , authoritative , errmsg , result ) )
+                return false;

-            string configdb = cmdObj["configdb"].valuestrsafe();
-            {
-                // configdb checking
-                if ( configdb.size() == 0 ) {
-                    errmsg = "no configdb";
-                    return false;
-                }
-
-                if ( shardingState.enabled() ) {
-                    if ( configdb != shardingState.getConfigServer() ) {
-                        errmsg = "specified a different configdb!";
-                        return false;
-                    }
-                }
-                else {
-                    if ( ! authoritative ) {
-                        result.appendBool( "need_authoritative" , true );
-                        errmsg = "first setShardVersion";
-                        return false;
-                    }
-                    shardingState.enable( configdb );
-                    configServer.init( configdb );
-                }
-            }
-
+            // check shard name/hosts are correct
            if ( cmdObj["shard"].type() == String ) {
                shardingState.gotShardName( cmdObj["shard"].String() );
                shardingState.gotShardHost( cmdObj["shardHost"].String() );
            }
-
-            {
-                // setting up ids
-                if ( cmdObj["serverID"].type() != jstOID ) {
-                    // TODO: fix this
-                    //errmsg = "need serverID to be an OID";
-                    //return 0;
-                }
-                else {
-                    OID clientId = cmdObj["serverID"].__oid();
-                    if ( ! info->hasID() ) {
-                        info->setID( clientId );
-                    }
-                    else if ( clientId != info->getID() ) {
-                        errmsg = "server id has changed!";
-                        return 0;
-                    }
-                }
-            }
-
-            unsigned long long version = extractVersion( cmdObj["version"] , errmsg );
-
-            if ( errmsg.size() ) {
+            
+            // make sure we have the mongos id for writebacks
+            if ( ! checkMongosID( info , cmdObj["serverID"] , errmsg ) )
                return false;
-            }

+            // step 2
+            
            string ns = cmdObj["setShardVersion"].valuestrsafe();
            if ( ns.size() == 0 ) {
-                errmsg = "need to speciy fully namespace";
+                errmsg = "need to speciy namespace";
                return false;
            }

+            const ConfigVersion version = extractVersion( cmdObj["version"] , errmsg );
+            if ( errmsg.size() )
+                return false;
+            
+            // step 3
+
            const ConfigVersion oldVersion = info->getVersion(ns);
            const ConfigVersion globalVersion = shardingState.getVersion(ns);

+            result.appendTimestamp( "oldVersion" , oldVersion );
+            
+            if ( globalVersion > 0 && version > 0 ) {
+                // this means there is no reset going on an either side
+                // so its safe to make some assuptions
+
+                if ( version == globalVersion ) {
+                    // mongos and mongod agree!
+                    if ( oldVersion != version ) {
+                        assert( oldVersion < globalVersion );
+                        info->setVersion( ns , version );
+                    }
+                    return true;
+                }
+                
+            }
+
+            // step 4
+            dblock setShardVersionLock; // TODO: can we get rid of this??
+            
            if ( oldVersion > 0 && globalVersion == 0 ) {
                // this had been reset
                info->setVersion( ns , 0 );
@@ -475,7 +525,6 @@ namespace mongo {
                    result.appendBool( "need_authoritative" , true );
                    result.append( "ns" , ns );
                    result.appendTimestamp( "globalVersion" , globalVersion );
-                    result.appendTimestamp( "oldVersion" , oldVersion );
                    errmsg = "dropping needs to be authoritative";
                    return false;
                }
@@ -491,7 +540,6 @@ namespace mongo {
            if ( version < oldVersion ) {
                errmsg = "you already have a newer version of collection '" + ns + "'";
                result.append( "ns" , ns );
-                result.appendTimestamp( "oldVersion" , oldVersion );
                result.appendTimestamp( "newVersion" , version );
                result.appendTimestamp( "globalVersion" , globalVersion );
                return false;
@@ -510,7 +558,7 @@ namespace mongo {
                return false;
            }

-            if ( globalVersion == 0 && ! cmdObj.getBoolField( "authoritative" ) ) {
+            if ( globalVersion == 0 && ! authoritative ) {
                // need authoritative for first look
                result.append( "ns" , ns );
                result.appendBool( "need_authoritative" , true );
@@ -518,6 +566,7 @@ namespace mongo {
                return false;
            }

+            Timer relockTime;
            {
                dbtemprelease unlock;

@@ -530,11 +579,11 @@ namespace mongo {
                    return false;
                }
            }
-
+            if ( relockTime.millis() >= ( cmdLine.slowMS - 10 ) ) {
+                log() << "setShardVersion - relocking slow: " << relockTime.millis() << endl;
+            }
+            
            info->setVersion( ns , version );
-            result.appendTimestamp( "oldVersion" , oldVersion );
-            result.append( "ok" , 1 );
-
            return true;
        }

@@ -635,10 +684,10 @@ namespace mongo {
            return false;
        }

-        if ( isWriteOp && version.majorVersion() == clientVersion.majorVersion() ) {
+        if ( version.majorVersion() == clientVersion.majorVersion() ) {
            // this means there was just a split
            // since on a split w/o a migrate this server is ok
-            // going to accept write
+            // going to accept 
            return true;
        }

--- a/s/request.cpp
+++ b/s/request.cpp
@@ -41,8 +41,7 @@ namespace mongo {
        assert( _d.getns() );
        _id = _m.header()->id;

-        _clientId = p ? p->getClientId() : 0;
-        _clientInfo = ClientInfo::get( _clientId );
+        _clientInfo = ClientInfo::get();
        _clientInfo->newRequest( p );

    }
@@ -74,7 +73,7 @@ namespace mongo {
        }

        _m.header()->id = _id;
-
+        _clientInfo->clearCurrentShards();
    }

    Shard Request::primaryShard() const {
--- a/s/request.h
+++ b/s/request.h
@@ -66,9 +66,6 @@ namespace mongo {
            return _chunkManager;
        }

-        int getClientId() const {
-            return _clientId;
-        }
        ClientInfo * getClientInfo() const {
            return _clientInfo;
        }
@@ -103,7 +100,6 @@ namespace mongo {
        DBConfigPtr _config;
        ChunkManagerPtr _chunkManager;

-        int _clientId;
        ClientInfo * _clientInfo;

        OpCounters* _counter;
--- a/s/server.cpp
+++ b/s/server.cpp
@@ -77,19 +77,19 @@ namespace mongo {
    public:
        virtual ~ShardedMessageHandler() {}

-        virtual void process( Message& m , AbstractMessagingPort* p ) {
+        virtual void connected( AbstractMessagingPort* p ) {
+            assert( ClientInfo::get() );
+        }
+
+        virtual void process( Message& m , AbstractMessagingPort* p , LastError * le) {
            assert( p );
            Request r( m , p );

-            LastError * le = lastError.startRequest( m , r.getClientId() );
-            assert( le );
+            assert( le );            
+            lastError.startRequest( m , le );

-            if ( logLevel > 5 ) {
-                log(5) << "client id: " << hex << r.getClientId() << "\t" << r.getns() << "\t" << dec << r.op() << endl;
-            }
            try {
                r.init();
-                setClientId( r.getClientId() );
                r.process();
            }
            catch ( AssertionException & e ) {
@@ -119,8 +119,7 @@ namespace mongo {
        }

        virtual void disconnected( AbstractMessagingPort* p ) {
-            ClientInfo::disconnect( p->getClientId() );
-            lastError.disconnect( p->getClientId() );
+            // all things are thread local
        }
    };

--- a/s/shard_version.cpp
+++ b/s/shard_version.cpp
@@ -142,9 +142,10 @@ namespace mongo {
            checkShardVersion( conn , ns , 1 , tryNumber + 1 );
            return true;
        }
-
-        log() << "     setShardVersion failed: " << result << endl;
-        massert( 10429 , (string)"setShardVersion failed! " + result.jsonString() , 0 );
+        
+        string errmsg = str::stream() << "setShardVersion failed host[" << conn.getServerAddress() << "] " << result;
+        log() << "     " << errmsg << endl;
+        massert( 10429 , errmsg , 0 );
        return true;
    }

--- a/s/util.h
+++ b/s/util.h
@@ -105,7 +105,7 @@ namespace mongo {
                _combined = 0;
                break;
            default:
-                assert(0);
+                massert( 13657 , str::stream() << "unknown type for ShardChunkVersion: " << elem , 0 );
            }
            return *this;
        }
--- a/s/writeback_listener.cpp
+++ b/s/writeback_listener.cpp
@@ -159,7 +159,7 @@ namespace mongo {
                    DBConfigPtr db = grid.getDBConfig( ns );
                    ShardChunkVersion needVersion( data["version"] );

-                    log(1) << "connectionId: " << cid << " writebackId: " << wid << " needVersion : " << needVersion.toString()
+                    LOG(1) << "connectionId: " << cid << " writebackId: " << wid << " needVersion : " << needVersion.toString()
                           << " mine : " << db->getChunkManager( ns )->getVersion().toString() << endl;// TODO change to log(3)

                    if ( logLevel ) log(1) << debugString( m ) << endl;
--- a/shell/db.js
+++ b/shell/db.js
@@ -769,8 +769,8 @@ DB.prototype.listCommands = function(){
    }
 }

-DB.prototype.printShardingStatus = function(){
-    printShardingStatus( this.getSiblingDB( "config" ) );
+DB.prototype.printShardingStatus = function( verbose ){
+    printShardingStatus( this.getSiblingDB( "config" ) , verbose );
 }

 DB.autocomplete = function(obj){
--- a/shell/mongo_vstudio.cpp
+++ b/shell/mongo_vstudio.cpp
@@ -2253,8 +2253,8 @@ const StringData _jscode_raw_db =
 "}\n" 
 "}\n" 
 "\n" 
-"DB.prototype.printShardingStatus = function(){\n" 
-"printShardingStatus( this.getSiblingDB( \"config\" ) );\n" 
+"DB.prototype.printShardingStatus = function( verbose ){\n" 
+"printShardingStatus( this.getSiblingDB( \"config\" ) , verbose );\n" 
 "}\n" 
 "\n" 
 "DB.autocomplete = function(obj){\n" 
--- a/shell/servers.js
+++ b/shell/servers.js
@@ -478,7 +478,7 @@ printShardingStatus = function( configDB , verbose ){
                                output( "\t\t\t\t" + z.shard + "\t" + z.nChunks );
                            } )
                            
-                            if ( totalChunks < 1000 || verbose ){
+                            if ( totalChunks < 20 || verbose ){
                                configDB.chunks.find( { "ns" : coll._id } ).sort( { min : 1 } ).forEach( 
                                    function(chunk){
                                        output( "\t\t\t" + tojson( chunk.min ) + " -->> " + tojson( chunk.max ) + 
--- a/tools/dump.cpp
+++ b/tools/dump.cpp
@@ -69,13 +69,13 @@ public:
        Writer writer(out, m);

        // use low-latency "exhaust" mode if going over the network
-        if (typeid(connBase) == typeid(DBClientConnection&)) {
+        if (!_usingMongos && typeid(connBase) == typeid(DBClientConnection&)) {
            DBClientConnection& conn = static_cast<DBClientConnection&>(connBase);
            boost::function<void(const BSONObj&)> castedWriter(writer); // needed for overload resolution
            conn.query( castedWriter, coll.c_str() , q , NULL, queryOptions | QueryOption_Exhaust);
        }
        else {
-            //This branch should only be taken with DBDirectClient which doesn't support exhaust mode
+            //This branch should only be taken with DBDirectClient or mongos which doesn't support exhaust mode
            scoped_ptr<DBClientCursor> cursor(connBase.query( coll.c_str() , q , 0 , 0 , 0 , queryOptions ));
            while ( cursor->more() ) {
                writer(cursor->next());
@@ -303,8 +303,6 @@ public:
            opLogStart = op["ts"]._numberLong();
        }

-
-
        // check if we're outputting to stdout
        string out = getParam("out");
        if ( out == "-" ) {
@@ -318,6 +316,13 @@ public:
            }
        }

+        {
+            // TODO: when mongos supports QueryOption_Exaust add a version check (SERVER-2628)
+            BSONObj isdbgrid;
+            conn("true").simpleCommand("admin", &isdbgrid, "isdbgrid");
+            _usingMongos = isdbgrid["isdbgrid"].trueValue();
+        }
+
        path root( out );
        string db = _db;

@@ -358,6 +363,7 @@ public:
        return 0;
    }

+    bool _usingMongos;
    BSONObj _query;
 };

--- a/tools/tool.cpp
+++ b/tools/tool.cpp
@@ -101,7 +101,11 @@ namespace mongo {
        // we want durability to be disabled.
        cmdLine.dur = false;

-        boost::filesystem::path::default_name_check( boost::filesystem::no_check );
+#if( BOOST_VERSION >= 104500 )
+    boost::filesystem::path::default_name_check( boost::filesystem2::no_check );
+#else
+    boost::filesystem::path::default_name_check( boost::filesystem::no_check );
+#endif

        _name = argv[0];

--- a/util/alignedbuilder.cpp
+++ b/util/alignedbuilder.cpp
@@ -48,7 +48,7 @@ namespace mongo {
        assert( a );
        while( 1 ) {
            a *= 2;
-            wassert( a <= 128*1024*1024 );
+            wassert( a <= 256*1024*1024 );
            assert( a <= 512*1024*1024 );
            if( _len < a )
                break;
--- a/util/logfile.cpp
+++ b/util/logfile.cpp
@@ -136,7 +136,7 @@ namespace mongo {
        }
        ssize_t written = write(_fd, buf, len);
        if( written != (ssize_t) len ) {
-            log() << "write fails written:" << written << " len:" << len << " buf:" << buf << " errno:" << errno << endl;
+            log() << "write fails written:" << written << " len:" << len << " buf:" << buf << ' ' << errnoWithDescription() << endl;
            uasserted(13515, str::stream() << "error appending to file " << _fd  << ' ' << errnoWithDescription());
        }

--- a/util/message.cpp
+++ b/util/message.cpp
@@ -703,7 +703,6 @@ again:


    MSGID NextMsgId;
-    ThreadLocalValue<int> clientId;

    struct MsgStart {
        MsgStart() {
@@ -721,14 +720,6 @@ again:
        return op == dbQuery || op == dbGetMore;
    }

-    void setClientId( int id ) {
-        clientId.set( id );
-    }
-
-    int getClientId() {
-        return clientId.get();
-    }
-
    const int DEFAULT_MAX_CONN = 20000;
    const int MAX_MAX_CONN = 20000;

--- a/util/message.h
+++ b/util/message.h
@@ -85,12 +85,8 @@ namespace mongo {
        virtual HostAndPort remote() const = 0;
        virtual unsigned remotePort() const = 0;

-        virtual int getClientId() {
-            int x = remotePort();
-            x = x << 16;
-            x |= ( ( 0xFF0 & (long long)this ) >> 8 ); // lowest byte in pointer often meaningless
-            return x;
-        }
+    private:
+        int _clientId;
    };

    class MessagingPort : public AbstractMessagingPort {
@@ -471,9 +467,6 @@ namespace mongo {

    MSGID nextMessageId();

-    void setClientId( int id );
-    int getClientId();
-
    extern TicketHolder connTicketHolder;

    class ElapsedTracker {
--- a/util/message_server.h
+++ b/util/message_server.h
@@ -29,7 +29,9 @@ namespace mongo {
    class MessageHandler {
    public:
        virtual ~MessageHandler() {}
-        virtual void process( Message& m , AbstractMessagingPort* p ) = 0;
+
+        virtual void connected( AbstractMessagingPort* p ) = 0;
+        virtual void process( Message& m , AbstractMessagingPort* p , LastError * err ) = 0;
        virtual void disconnected( AbstractMessagingPort* p ) = 0;
    };

--- a/util/message_server_port.cpp
+++ b/util/message_server_port.cpp
@@ -23,6 +23,7 @@
 #include "message_server.h"

 #include "../db/cmdline.h"
+#include "../db/lasterror.h"
 #include "../db/stats/counters.h"

 namespace mongo {
@@ -38,14 +39,19 @@ namespace mongo {

            setThreadName( "conn" );

-            auto_ptr<MessagingPort> p( inPort );
+            scoped_ptr<MessagingPort> p( inPort );

            string otherSide;

            Message m;
            try {
+                LastError * le = new LastError();
+                lastError.reset( le ); // lastError now has ownership
+
                otherSide = p->farEnd.toString();

+                handler->connected( p.get() );
+
                while ( 1 ) {
                    m.reset();
                    p->clearCounters();
@@ -57,7 +63,7 @@ namespace mongo {
                        break;
                    }

-                    handler->process( m , p.get() );
+                    handler->process( m , p.get() , le );
                    networkCounter.hit( p->getBytesIn() , p->getBytesOut() );
                }
            }
--- a/util/mmap_posix.cpp
+++ b/util/mmap_posix.cpp
@@ -51,6 +51,10 @@ namespace mongo {

 #ifndef O_NOATIME
 #define O_NOATIME (0)
+#endif
+
+#ifndef MAP_NORESERVE
+#define MAP_NORESERVE (0)
 #endif

    void* MemoryMappedFile::map(const char *filename, unsigned long long &length, int options) {
@@ -118,7 +122,7 @@ namespace mongo {
    }

    void* MemoryMappedFile::createPrivateMap() {
-        void * x = mmap( /*start*/0 , len , PROT_READ|PROT_WRITE , MAP_PRIVATE , fd , 0 );
+        void * x = mmap( /*start*/0 , len , PROT_READ|PROT_WRITE , MAP_PRIVATE|MAP_NORESERVE , fd , 0 );
        if( x == MAP_FAILED ) {
            if ( errno == ENOMEM ) {
                if( sizeof(void*) == 4 ) {
@@ -140,7 +144,7 @@ namespace mongo {

    void* MemoryMappedFile::remapPrivateView(void *oldPrivateAddr) {
        // don't unmap, just mmap over the old region
-        void * x = mmap( oldPrivateAddr, len , PROT_READ|PROT_WRITE , MAP_PRIVATE|MAP_FIXED , fd , 0 );
+        void * x = mmap( oldPrivateAddr, len , PROT_READ|PROT_WRITE , MAP_PRIVATE|MAP_NORESERVE|MAP_FIXED , fd , 0 );
        if( x == MAP_FAILED ) {
            int err = errno;
            error()  << "13601 Couldn't remap private view: " << errnoWithDescription(err) << endl;
--- a/util/ntservice.cpp
+++ b/util/ntservice.cpp
@@ -312,9 +312,21 @@ namespace mongo {

        SERVICE_STATUS ssStatus;

+        DWORD dwControlsAccepted;
+        switch ( reportState ) {
+        case SERVICE_START_PENDING:
+        case SERVICE_STOP_PENDING:
+        case SERVICE_STOPPED:
+            dwControlsAccepted = 0;
+            break;
+        default:
+            dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN;
+            break;
+        }
+
        ssStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
        ssStatus.dwServiceSpecificExitCode = 0;
-        ssStatus.dwControlsAccepted = reportState == SERVICE_START_PENDING ? 0 : SERVICE_ACCEPT_STOP;
+        ssStatus.dwControlsAccepted = dwControlsAccepted;
        ssStatus.dwCurrentState = reportState;
        ssStatus.dwWin32ExitCode = NO_ERROR;
        ssStatus.dwWaitHint = waitHint;
@@ -340,6 +352,7 @@ namespace mongo {
        switch ( ctrlCode ) {
        case SERVICE_CONTROL_STOP:
        case SERVICE_CONTROL_SHUTDOWN:
+            reportStatus( SERVICE_STOP_PENDING );
            shutdownServer();
            reportStatus( SERVICE_STOPPED );
            return;
--- a/util/version.cpp
+++ b/util/version.cpp
@@ -26,7 +26,7 @@

 namespace mongo {

-    const char versionString[] = "1.8.0-rc0";
+    const char versionString[] = "1.8.1";

    string mongodVersion() {
        stringstream ss;
Author	SHA1	Message	Date
Eliot Horowitz	a429cd4f53	BUMP 1.8.1	2011-04-06 00:40:13 -04:00
Eliot Horowitz	27413a5ca3	post 1.8.1-rc1	2011-04-01 01:14:09 -04:00
Eliot Horowitz	c340b4882b	BUMP 1.8.1-rc1	2011-03-31 18:39:07 -04:00
Eliot Horowitz	5bd0556e0e	fix 1.8 compile backport	2011-03-31 03:42:07 -04:00
Eliot Horowitz	134e8eb0e0	remove all clientId related code and systems, and just use tls SERVER-2872	2011-03-31 03:12:43 -04:00
greg	7c5b035e1b	Cleanup connection when getlasterror fails	2011-03-31 03:12:40 -04:00
Eliot Horowitz	5cc6b7bca4	fix callback for not master errors on replica set connections and make background thread more reliable SERVER-2868	2011-03-31 03:09:07 -04:00
Eliot Horowitz	444950f086	safer not master check	2011-03-31 03:08:59 -04:00
Eliot Horowitz	62cbfb842c	more custom backport of SERVER-2854	2011-03-31 03:06:55 -04:00
Eliot Horowitz	64eeb57f33	custom backport of SERVER-2854	2011-03-30 22:26:34 -04:00
Mathias Stearn	25cabc25ae	missing commitIfNeeded for dropDups SERVER-2815	2011-03-30 22:20:52 -04:00
Eliot Horowitz	843747d201	Missing commitIfNeeded in background index build SERVER-2853	2011-03-30 01:57:42 -04:00
Eliot Horowitz	a75fddc244	better error message	2011-03-30 01:40:59 -04:00
Eliot Horowitz	c480248550	better errmsg for setShardVersion failure	2011-03-30 01:40:45 -04:00
Eliot Horowitz	6545677613	post 1.8.1-rc0	2011-03-25 11:33:07 -04:00
Aaron	674a1b857a	SERVER-2808 try to compile non apple	2011-03-24 15:42:48 -07:00
Aaron	badd1cd93f	SERVER-2808 print address information when available for signals	2011-03-24 15:42:48 -07:00
Eliot Horowitz	8c0cb88da9	BUMP 1.8.1-rc0	2011-03-24 17:42:36 -04:00
Eliot Horowitz	cd7e619868	visaul studio	2011-03-24 17:42:29 -04:00
Aaron	20ca858390	SERVER-2737 commitIfNeeded() in cappedTruncateAfter()	2011-03-24 10:03:35 -07:00
Aaron	2394d382e1	SERVER-2810 check for dropped explain cursor before accessing so	2011-03-24 10:02:29 -07:00
Aaron	6021266abb	SERVER-2809 update _nscanned with proper value, now that it may be returned	2011-03-22 22:13:03 -07:00
Aaron	0d5979c50b	SERVER-2809 don't read cursor's nscanned if failed yield invalidated the cursor	2011-03-22 22:12:54 -07:00
Eliot Horowitz	7e59336595	SERVER-2797 fix BTreeCursor handling and ensure chunks aren't over sized	2011-03-19 15:48:39 -04:00
Kristina	010e692d65	allow 1 or true for rs config options SERVER-2714	2011-03-18 22:07:24 -04:00
Mathias Stearn	6187cea64f	support boost 1.46 SERVER-2324 181?	2011-03-18 22:05:51 -04:00
Aaron	15c6cce90a	SERVER-2662 don't attempt to yield a query after an earlier yield fails and drops the cursor	2011-03-18 22:04:52 -04:00
Aaron	ab41f59e8f	SERVER-2662 test	2011-03-18 22:04:48 -04:00
Aaron	3d7f9d035e	SERVER-2791 use write intent when setting initial bucket	2011-03-17 22:20:21 -04:00
rstam	e4df10c2e0	Fixed SERVER-2719, SERVER-1680 and SERVER-1434 (all had the same underlying cause).	2011-03-16 17:46:50 -04:00
Mathias Stearn	0dcc19701f	Better handling of sharded findAndModify output SERVER-2698 180?	2011-03-16 17:46:50 -04:00
Eliot Horowitz	f1488a594f	fix SERVER-2745	2011-03-16 17:46:50 -04:00
Eliot Horowitz	fade9c4e51	more accurate client id	2011-03-16 15:54:40 -04:00
Eliot Horowitz	3d083de24b	movving logic from .h to .cpp	2011-03-16 15:54:35 -04:00
Eliot Horowitz	9a1ba315af	post 1.8.0	2011-03-16 14:45:01 -04:00
Eliot Horowitz	9c28b1d608	BUMP 1.8.0	2011-03-16 11:33:30 -04:00
Eliot Horowitz	cc5b8c1094	use nextSafe 180	2011-03-16 11:27:52 -04:00
Kristina	2ad98fd1fa	fix fastsync to not be confused on new members SERVER-2717	2011-03-13 01:33:16 -05:00
Eliot Horowitz	742349b018	fix migrate with oversize docs SERVER-2744 180	2011-03-11 16:55:44 -05:00
Eliot Horowitz	e89833c5d2	--journal SERVER-2745 180	2011-03-11 16:55:14 -05:00
Eliot Horowitz	2409249072	make some options hidden	2011-03-11 16:55:02 -05:00
Dwight	d83d060854	dur rollback replsets SERVER-2737	2011-03-11 16:54:51 -05:00
Eliot Horowitz	257b016504	post 1.8.0-rc2	2011-03-09 22:29:44 -05:00
Mathias Stearn	460d32f86b	Open a journal file on startup 180	2011-03-09 17:21:11 -05:00
Mathias Stearn	b2927d9860	Don't rotate and open a journal file if one isn't already opened SERVER-2655 180	2011-03-09 15:27:13 -05:00
Eliot Horowitz	b3a113564f	BUMP 1.8.0-rc2	2011-03-09 13:30:16 -05:00
Mathias Stearn	0e97d8b44b	Add MAP_NORESERVE option to private mmaps 180?	2011-03-09 11:14:12 -05:00
Eliot Horowitz	3bc93cc5dc	fix potential seg fault when Client Context isn't handled correct with exceptions	2011-03-09 10:23:51 -05:00
Eliot Horowitz	2bc1b84422	more safety for getMore yielding 180	2011-03-09 02:46:41 -05:00
Eliot Horowitz	b28116bafe	fix sharding test with local change	2011-03-08 16:22:12 -05:00
Eliot Horowitz	43f5f6f9bc	safer key too large check 180	2011-03-08 14:12:48 -05:00
Eliot Horowitz	ec9dbe2b51	don't return local in listDatabases sharded 180 SERVER-2686	2011-03-08 14:12:44 -05:00
Dwight	085553d8dd	dur typo in constant 180	2011-03-07 12:56:39 -05:00
Eliot Horowitz	5d5b9dc17e	do'nt use RARELY with commitIfNeeded, just check if it did it	2011-03-07 12:18:57 -05:00
Eliot Horowitz	a14508a41e	dur commitIfNeeded returns if it commited	2011-03-07 12:18:43 -05:00
dwight	3754755c30	faster: groupCommit already catches exceptions	2011-03-07 12:18:40 -05:00
Eliot Horowitz	e451c5699d	Revert "fix setShardVersion on a new db" This reverts commit `049436fd5b`.	2011-03-07 12:06:25 -05:00
Eliot Horowitz	eb3d063c95	fix setShardVersion on a new db	2011-03-07 10:36:00 -05:00
Eliot Horowitz	7a20ea7d74	make warning about missing index entries for large keys nicer SERVER-2693	2011-03-07 08:16:06 -05:00
Eliot Horowitz	4d75f1be77	fix boost 1.45	2011-03-06 20:21:08 -05:00
Eliot Horowitz	9d1f1a1fe3	clean setShardVersion to reduce locking 180?	2011-03-06 20:18:53 -05:00
Eliot Horowitz	c3c2a3d41d	don't record info once context finished 180	2011-03-06 20:18:45 -05:00
Eliot Horowitz	e521301d18	make forcing a split work better SERVER-2419 180?	2011-03-06 20:18:32 -05:00
Eliot Horowitz	d2749d18ee	cap connections in connection pool stack SERVER-2687 180	2011-03-05 21:10:26 -05:00
Eliot Horowitz	fa09f824ed	use .trueValue() rather than .Bool() for backwards compat. 180	2011-03-05 21:10:19 -05:00
Eliot Horowitz	66af39a8b3	reload conifg after manual split	2011-03-05 16:24:57 -05:00
Eliot Horowitz	fe13a93925	log why setShardVersion is slow	2011-03-04 17:23:20 -05:00
Eliot Horowitz	9f47ba6ca4	pass verbose to printShardingStatus	2011-03-04 16:16:51 -05:00
Eliot Horowitz	e3b4054f66	when removing keys that are too large from an index, stll remove but warn	2011-03-04 16:16:47 -05:00
Eliot Horowitz	e0e5a4061d	POST 1.8.0-rc1	2011-03-04 16:16:19 -05:00
Mathias Stearn	87dd5720e4	initialize _recovering 180	2011-03-03 21:56:55 -05:00
Eliot Horowitz	b1eb83e61c	fix yield on getMore SERVER-2450	2011-03-03 21:56:42 -05:00
Eliot Horowitz	f53b65645f	yield on getMore SERVER-2450	2011-03-03 21:56:38 -05:00
Eliot Horowitz	bbf18b270c	fix rs client error on failover SERVER-2619	2011-03-03 12:21:23 -05:00
Eliot Horowitz	081b046b72	update index test because of fix SERVER-1716	2011-03-02 10:59:04 -05:00
Eliot Horowitz	00bee15013	fix BtreeBuilder to always check key size	2011-03-02 10:30:58 -05:00
Eliot Horowitz	72f754439a	don't do commitIfNeeded RARELY SERVER-2657 180	2011-03-02 01:28:57 -05:00
Eliot Horowitz	85bff78d00	some missing commitIfNeeded SERVER-2657 180	2011-03-02 01:28:51 -05:00
Eliot Horowitz	dc65bd51ae	make sure that index creation is logged on secondaries during migrate SERVER-2665	2011-03-01 23:59:43 -05:00
dwight	e5839f0a76	cant destroy complex global variables at shutdown 180?	2011-03-01 23:48:38 -05:00
Mathias Stearn	28267f6792	add FileCreatedOp on first use of file, regardless of preexistence 180? Previously FileCreatedOp wouldn't be done for preallocated files since they already existed.	2011-03-01 23:48:22 -05:00
Mathias Stearn	162418c1dd	Better error message. 180?	2011-03-01 23:48:10 -05:00
Eliot Horowitz	b47f248e0f	BUMP 1.8.0-rc1	2011-03-01 15:58:08 -05:00
Aaron	af4f0597fd	SERVER-2650 prevent too small extents	2011-03-01 15:56:24 -05:00
Eliot Horowitz	ad4abbc55b	debugging for slow server status	2011-03-01 13:32:39 -05:00
Eliot Horowitz	a0b0d4eb70	accept only major version max for reads and writes 181?	2011-03-01 13:31:22 -05:00
Eliot Horowitz	13608bb283	null is ok in shard key, blank is not SERVER-2648	2011-03-01 13:21:26 -05:00
Eliot Horowitz	83282b72fa	less verbose chunk info by default	2011-03-01 13:21:22 -05:00
Dwight	caaff8dfbe	don't clean up journal files, or do much of anything, on an abruptQuit() only doing this for --dur scenarios for now to keep the change small. later should probably change on a wider basis. 180?	2011-03-01 10:57:27 -05:00
Dwight	22596b3c06	SERVER-2631 missing commitIfNeeded in ~BtreeBuilder - 180?	2011-03-01 10:57:20 -05:00
Dwight	d328f5b1a5	SERVER-2631 missing destructor guard - 180?	2011-03-01 10:57:14 -05:00
Dwight	09ed6d9098	SERVER-2631 missing commitIfNeeded call - 180?	2011-03-01 10:57:08 -05:00
dwight	b2963ff3b2	SERVER-2631 180? 181?	2011-03-01 10:57:02 -05:00
dwight	bd46051947	fix invalid 'unexpected file in journal directory' error message. 180	2011-03-01 10:51:56 -05:00
Eliot Horowitz	ed6273ba49	make sure not to use memory in a cursor when yielding SERVER-2447	2011-03-01 10:51:48 -05:00
Eliot Horowitz	f6734a4784	fix version #	2011-02-28 20:59:08 -05:00
Eliot Horowitz	d38d0f8f57	don't take balancer lock if balancing is disable SERVER-2632	2011-02-27 13:38:51 -05:00
Mathias Stearn	f6074c8024	mongodump shouldn't use QueryOption_Exhaust when connected to mongos SERVER-2628	2011-02-25 15:00:51 -05:00