newer xulrunner

(cherry picked from commit ae02e61015)

SConstruct

@@ -1178,6 +1178,11 @@ testEnv.Alias( "startMongod", [add_exe("mongod")], [startMongodForTests] );
 testEnv.AlwaysBuild( "startMongod" );
 testEnv.SideEffect( "dummySmokeSideEffect", "startMongod" )
 
+testEnv.Alias( "startMongodSmallOplog", [add_exe("mongod")], [startMongodForTests] );
+testEnv.AlwaysBuild( "startMongodSmallOplog" );
+testEnv.SideEffect( "dummySmokeSideEffect", "startMongodSmallOplog" )
+
+
 def addMongodReqTargets( env, target, source ):
     mongodReqTargets = [ "smokeClient", "smokeJs", "smokeQuota" ]
     for target in mongodReqTargets:

buildscripts/hacks_ubuntu.py

@@ -39,7 +39,9 @@ def foundxulrunner( env , options ):
     env.Prepend( RPATH=[ libroot ] )
 
     env.Prepend( CPPPATH=[ incroot + "stable/" , 
-                           incroot + "unstable/" ] )
+                           incroot + "unstable/" ,
+                           incroot ] )
+    env.Prepend( CPPPATH=[ "/usr/include/nspr/" ] )
     
     env.Append( CPPDEFINES=[ "XULRUNNER" , "OLDJS" ] )
     if best.find( "1.9.0" ) >= 0 or best.endswith("1.9"):

db/db.cpp

@@ -65,6 +65,8 @@ namespace mongo {
     void pairWith(const char *remoteEnd, const char *arb);
     void setRecCacheSize(unsigned MB);
 
+    void exitCleanly( ExitCode code );
+
     const char *ourgetns() { 
         Client *c = currentClient.get();
         return c ? c->ns() : "";
@@ -224,6 +226,9 @@ namespace mongo {
             problem() << "SocketException in connThread, closing client connection" << endl;
             dbMsgPort.shutdown();
         }
+        catch ( const ClockSkewException &e ) {
+            exitCleanly( EXIT_CLOCK_SKEW );
+        }        
         catch ( std::exception &e ) {
             problem() << "Uncaught std::exception: " << e.what() << ", terminating" << endl;
             dbexit( EXIT_UNCAUGHT );
@@ -943,13 +948,13 @@ namespace mongo {
 
 #undef out
 
-    void exitCleanly() {
+    void exitCleanly( ExitCode code ) {
         goingAway = true;
         killCurrentOp = 1;
         {
             dblock lk;
             log() << "now exiting" << endl;
-            dbexit( EXIT_KILL );        
+            dbexit( code );        
         }
     }
 
@@ -995,7 +1000,7 @@ namespace mongo {
         int x;
         sigwait( &asyncSignals, &x );
         log() << "got kill or ctrl c signal " << x << " (" << strsignal( x ) << "), will terminate after current cmd ends" << endl;
-        exitCleanly();
+        exitCleanly( EXIT_KILL );
     }
 
     void setupSignals() {
@@ -1018,7 +1023,7 @@ namespace mongo {
 #else
 void ctrlCTerminate() {
     log() << "got kill or ctrl c signal, will terminate after current cmd ends" << endl;
-    exitCleanly();
+    exitCleanly( EXIT_KILL );
 }
 BOOL CtrlHandler( DWORD fdwCtrlType )
 {

db/dbcommands.cpp

@@ -1281,7 +1281,32 @@ namespace mongo {
 
     } distinctCmd;
 
-
+    /* For testing only, not for general use */
+    class GodInsert : public Command {
+    public:
+        GodInsert() : Command( "godinsert" ) { }
+        virtual bool logTheOp() {
+            return true;
+        }
+        virtual bool slaveOk() {
+            return false;
+        }
+        virtual bool requiresAuth() {
+            return true;
+        }
+        virtual void help( stringstream &help ) const {
+            help << "[for testing only]";
+        }        
+        virtual bool run(const char *dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+            string coll = cmdObj[ "godinsert" ].valuestrsafe();
+            uassert( "godinsert must specify a collection", !coll.empty() );
+            string ns = nsToClient( dbname ) + "." + coll;
+            BSONObj obj = cmdObj[ "obj" ].embeddedObjectUserCheck();
+            DiskLoc loc = theDataFileMgr.insert( ns.c_str(), obj, true );
+            return true;
+        }
+    } cmdGodInsert;
+    
     extern map<string,Command*> *commands;
 
     /* TODO make these all command objects -- legacy stuff here

db/jsobj.cpp

@@ -608,11 +608,15 @@ namespace mongo {
             case DBRef:
             case Code:
             case Symbol:
-            case String:
-                massert( "Invalid dbref/code/string/symbol size",
-                        valuestrsize() > 0 &&
-                        valuestrsize() - 1 == strnlen( valuestr(), valuestrsize() ) );
+            case String: {
+                int x = valuestrsize();
+                if ( x > 0 && valuestr()[x-1] == 0 )
+                    return;
+                stringstream buf;
+                buf <<  "Invalid dbref/code/string/symbol size: " << x;
+                massert( buf.str() , 0 );
                 break;
+            }
             case CodeWScope: {
                 int totalSize = *( int * )( value() );
                 massert( "Invalid CodeWScope size", totalSize >= 8 );

db/repl.cpp

@@ -1644,8 +1644,14 @@ namespace mongo {
         const char * ns = "local.oplog.$main";
         setClient(ns);
         
-        if ( nsdetails( ns ) )
+        if ( nsdetails( ns ) ) {
+            DBDirectClient c;
+            BSONObj lastOp = c.findOne( ns, Query().sort( BSON( "$natural" << -1 ) ) );
+            if ( !lastOp.isEmpty() ) {
+                OpTime::setLast( lastOp[ "ts" ].date() );
+            }
             return;
+        }
         
         /* create an oplog collection, if it doesn't yet exist. */
         BSONObjBuilder b;

dbtests/jsobjtests.cpp

@@ -339,8 +339,9 @@ namespace JsobjTests {
                 }
                 BSONObj invalid() const {
                     BSONObj ret = valid();
-                    set( ret, 0, get( ret, 0 ) + 1 );
-                    set( ret, 7, get( ret, 7 ) + 1 );
+                    ASSERT_EQUALS( ret.firstElement().valuestr()[0] , 'b' );
+                    ASSERT_EQUALS( ret.firstElement().valuestr()[1] , 0 );
+                    ((char*)ret.firstElement().valuestr())[1] = 1;
                     return ret.copy();
                 }
             };
@@ -391,32 +392,6 @@ namespace JsobjTests {
                 };
             };
 
-            class WrongSymbolSize : public Base {
-                BSONObj valid() const {
-                    return fromjson( "{\"a\":\"b\"}" );
-                }
-                BSONObj invalid() const {
-                    BSONObj ret = valid();
-                    set( ret, 4, Symbol );
-                    set( ret, 0, get( ret, 0 ) + 1 );
-                    set( ret, 7, get( ret, 7 ) + 1 );
-                    return ret.copy();
-                }
-            };
-
-            class WrongCodeSize : public Base {
-                BSONObj valid() const {
-                    return fromjson( "{\"a\":\"b\"}" );
-                }
-                BSONObj invalid() const {
-                    BSONObj ret = valid();
-                    set( ret, 4, Code );
-                    set( ret, 0, get( ret, 0 ) + 1 );
-                    set( ret, 7, get( ret, 7 ) + 1 );
-                    return ret.copy();
-                }
-            };
-
             class NoFieldNameEnd : public Base {
                 BSONObj valid() const {
                     return fromjson( "{\"a\":1}" );
@@ -1248,8 +1223,6 @@ namespace JsobjTests {
             add< BSONObjTests::Validation::NegativeStringSize >();
             add< BSONObjTests::Validation::WrongSubobjectSize >();
             add< BSONObjTests::Validation::WrongDbrefNsSize >();
-            add< BSONObjTests::Validation::WrongSymbolSize >();
-            add< BSONObjTests::Validation::WrongCodeSize >();
             add< BSONObjTests::Validation::NoFieldNameEnd >();
             add< BSONObjTests::Validation::BadRegex >();
             add< BSONObjTests::Validation::BadRegexOptions >();

debian/changelog

@@ -1,3 +1,9 @@
+mongodb (1.2.5) unsable; urgency=low
+
+  * replication clock skew fix
+
+ -- Richard Kreuter <richard@10gen.com>  Fri, 05 Feb 2010 16:56:28 -0500
+
 mongodb (1.2.4) unsable; urgency=low
 
   * shell fix

debian/init.d

@@ -68,6 +68,10 @@ if test ! -x $DAEMON; then
     exit 0
 fi
 
+if test "x$ENABLE_MONGODB" != "xyes"; then
+    exit 0
+fi
+
 if test ! -x $DATA; then
     mkdir $DATA || exit 0
 fi

debian/mongodb.upstart

@@ -8,5 +8,8 @@ end script
 start on runlevel [2345]
 stop on runlevel [06]
 
-exec start-stop-daemon --start --quiet --chuid mongodb --exec  /usr/bin/mongod -- --config /etc/mongodb.conf
-
+script
+  ENABLE_MONGODB="yes"
+  if [ -f /etc/default/mongodb ]; then . /etc/default/mongodb; fi
+  if [ "x$ENABLE_MONGODB" = "xyes" ]; then exec start-stop-daemon --start --quiet --chuid mongodb --exec  /usr/bin/mongod -- --config /etc/mongodb.conf; fi
+end script

doxygenConfig

@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 1.2.4
+PROJECT_NUMBER         = 1.2.5
 OUTPUT_DIRECTORY       = docs
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English

jstests/repl/master1.js

@@ -0,0 +1,53 @@
+// Test handling of clock skew and optimes across mongod instances
+
+if ( typeof( BinData ) != "undefined" ) { // don't run in v8
+
+var baseName = "jstests_repl_master1test";
+
+oplog = function() {
+    return m.getDB( "local" ).oplog.$main;
+}
+
+lastop = function() {
+    return oplog().find().sort( {$natural:-1} ).next();
+}
+
+am = function() {
+    return m.getDB( baseName ).a;
+}
+
+rt = new ReplTest( baseName );
+
+m = rt.start( true );
+
+am().save( {} );
+assert.eq( "i", lastop().op );
+
+op = lastop();
+printjson( op );
+op.ts.t = op.ts.t + 600000 // 10 minutes
+m.getDB( "local" ).runCommand( {godinsert:"oplog.$main", obj:op} );
+
+rt.stop( true );
+m = rt.start( true, null, true );
+
+assert.eq( op.ts.t, lastop().ts.t );
+am().save( {} );
+assert.eq( op.ts.t, lastop().ts.t );
+assert.eq( op.ts.i + 1, lastop().ts.i );
+
+op = lastop();
+printjson( op );
+op.ts.i = Math.pow(2,31);
+printjson( op );
+m.getDB( "local" ).runCommand( {godinsert:"oplog.$main", obj:op} );
+
+rt.stop( true );
+m = rt.start( true, null, true );
+assert.eq( op.ts.i, lastop().ts.i );
+am().save( {} );
+sleep( 3000 ); // make sure dies on its own before stop() called
+
+assert.eq( 47 /*EXIT_CLOCK_SKEW*/, rt.stop( true ) );
+
+}

rpm/init.d-mongod

@@ -12,30 +12,37 @@
 
 # things from mongod.conf get there by mongod reading it
 
+
+
 OPTIONS=" -f /etc/mongod.conf"
+SYSCONFIG="/etc/sysconfig/mongod"
 
 mongod=${MONGOD-/usr/bin/mongod}
-pidfile=${PIDFILE-/var/run/mongod.pid}
-lockfile=${LOCKFILE-/var/lock/subsys/mongod}
+
+MONGO_USER=mongod
+MONGO_GROUP=mongod
+
+. "$SYSCONFIG" || true
 
 start()
 {
   echo -n $"Starting mongod: "
-  #daemon --pidfile=${pidfile} $mongod $OPTIONS  > /var/log/mongod
-  $mongod $OPTIONS > /var/log/mongod 2>&1 &
+  daemon --user "$MONGO_USER" $mongod $OPTIONS
   RETVAL=$?
-  [ $RETVAL = 0 ] && touch ${lockfile}
-  echo OK
+  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/mongod && success
 }
 
 stop()
 {
   echo -n $"Stopping mongod: "
-  #killproc -p ${pidfile} -d 10 $mongod
-  #RETVAL=$?
-  killall mongod > /dev/null 2>&1 
-  #[ $RETVAL = 0 ] && rm -f ${lockfile} ${pidfile}
-  echo OK
+  killproc -p /var/lib/mongo/mongod.lock -t30 -TERM /usr/bin/mongod
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/mongod && success
+}
+
+restart () {
+	stop
+	start
 }
 
 ulimit -n 12000
@@ -48,15 +55,17 @@ case "$1" in
   stop)
     stop
     ;;
-  restart)
-    stop
-    start
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f /var/lock/subsys/mongodb ] && restart || :
+    ;;
+  status)
+    status $mongod
     ;;
-#  status)
-#    status -p ${pidfile} $mongod
-#    ;;
   *)
-    echo $"Usage: $0 {start|stop|restart}"
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
     RETVAL=1
 esac
 

rpm/mongo.spec

@@ -1,5 +1,5 @@
 Name: mongo
-Version: 1.2.4
+Version: 1.2.5
 Release: mongodb_1%{?dist}
 Summary: mongo client shell and tools
 License: AGPL 3.0
@@ -23,6 +23,7 @@ client utilities.
 %package server
 Summary: mongo server, sharding server, and support scripts
 Group: Applications/Databases
+Requires: mongo
 
 %description server
 Mongo (from "huMONGOus") is a schema-free document-oriented database.
@@ -56,17 +57,19 @@ cp rpm/init.d-mongod $RPM_BUILD_ROOT/etc/rc.d/init.d/mongod
 chmod a+x $RPM_BUILD_ROOT/etc/rc.d/init.d/mongod
 mkdir -p $RPM_BUILD_ROOT/etc
 cp rpm/mongod.conf $RPM_BUILD_ROOT/etc/mongod.conf
+mkdir -p $RPM_BUILD_ROOT/etc/sysconfig
+cp rpm/mongod.sysconfig $RPM_BUILD_ROOT/etc/sysconfig/mongod
 mkdir -p $RPM_BUILD_ROOT/var/lib/mongo
-mkdir -p $RPM_BUILD_ROOT/var/log
-touch $RPM_BUILD_ROOT/var/log/mongo
+mkdir -p $RPM_BUILD_ROOT/var/log/mongo
+touch $RPM_BUILD_ROOT/var/log/mongo/mongod.log
 
 %clean
 scons -c
 rm -rf $RPM_BUILD_ROOT
 
 %pre server
-#/usr/sbin/useradd -M -o -r -d /var/mongo -s /bin/bash \
-#	-c "mongod" mongod > /dev/null 2>&1 || :
+/usr/sbin/useradd -M -r -U -d /var/lib/mongo -s /bin/false \
+    -c mongod mongod > /dev/null 2>&1
 
 %post server
 if test $1 = 1
@@ -98,10 +101,12 @@ fi
 %{_bindir}/mongorestore
 
 %{_mandir}/man1/mongo.1*
+%{_mandir}/man1/mongod.1*
 %{_mandir}/man1/mongodump.1*
 %{_mandir}/man1/mongoexport.1*
 %{_mandir}/man1/mongofiles.1*
 %{_mandir}/man1/mongoimport.1*
+%{_mandir}/man1/mongosniff.1*
 %{_mandir}/man1/mongorestore.1*
 
 %files server
@@ -114,8 +119,9 @@ fi
 /etc/rc.d/init.d/mongod
 /etc/sysconfig/mongod
 #/etc/rc.d/init.d/mongos
-%attr(0755,root,root) %dir /var/mongo
-%attr(0640,root,root) %config(noreplace) %verify(not md5 size mtime) /var/log/mongo
+%attr(0755,mongod,mongod) %dir /var/lib/mongo
+%attr(0755,mongod,mongod) %dir /var/log/mongo
+%attr(0640,mongod,mongod) %config(noreplace) %verify(not md5 size mtime) /var/log/mongo/mongod.log
 
 %files devel
 /usr/include/mongo
@@ -123,6 +129,8 @@ fi
 #%{_libdir}/libmongotestfiles.a
 
 %changelog
+* Thu Jan 28 2010 Richard M Kreuter <richard@10gen.com>
+- Minor fixes.
+
 * Sat Oct 24 2009 Joe Miklojcik <jmiklojcik@shopwiki.com> - 
 - Wrote mongo.spec.
-

rpm/mongod.conf

@@ -1,7 +1,10 @@
 # mongo.conf
 
 #where to log
-logpath=/var/log/mongod
+logpath=/var/log/mongo/mongod.log
+
+# fork and run in background
+fork = true
 
 #port = 27017
 

rpm/mongod.sysconfig

@@ -0,0 +1 @@
+# TODO: add relevant configuration stuff here.

scripting/engine_spidermonkey.h

@@ -42,6 +42,7 @@
 #endif
 
 #include "jsapi.h"
+#include "jsobj.h"
 #include "jsdate.h"
 #include "jsregexp.h"
 

scripting/v8_wrapper.cpp

@@ -124,7 +124,7 @@ namespace mongo {
             case mongo::Timestamp: {
                 Local<v8::Object> sub = v8::Object::New();            
 
-                sub->Set( v8::String::New( "time" ) , v8::Date::New( f.timestampTime() ) );
+                sub->Set( v8::String::New( "t" ) , v8::Number::New( f.timestampTime() ) );
                 sub->Set( v8::String::New( "i" ) , v8::Number::New( f.timestampInc() ) );
             
                 o->Set( v8::String::New( f.fieldName() ) , sub );
@@ -213,7 +213,7 @@ namespace mongo {
         case mongo::Timestamp: {
             Local<v8::Object> sub = v8::Object::New();            
             
-            sub->Set( v8::String::New( "time" ) , v8::Date::New( f.timestampTime() ) );
+            sub->Set( v8::String::New( "t" ) , v8::Number::New( f.timestampTime() ) );
             sub->Set( v8::String::New( "i" ) , v8::Number::New( f.timestampInc() ) );
             
             return sub;

shell/servers.js

@@ -599,7 +599,7 @@ ReplTest.prototype.stop = function( master , signal ){
         this.stop( false );
         return;
     }
-    stopMongod( this.getPort( master ) , signal || 15 );
+    return stopMongod( this.getPort( master ) , signal || 15 );
 }
 
 allocatePorts = function( n ) {

shell/utils.cpp

@@ -321,12 +321,13 @@ namespace mongo {
             return undefined_;
         }
         
-        void killDb( int port, pid_t _pid, int signal ) {
+        int killDb( int port, pid_t _pid, int signal ) {
             pid_t pid;
+            int exitCode = 0;
             if ( port > 0 ) {
                 if( dbs.count( port ) != 1 ) {
                     cout << "No db started on port: " << port << endl;
-                    return;
+                    return 0;
                 }
                 pid = dbs[ port ].first;
             } else {
@@ -346,6 +347,7 @@ namespace mongo {
                 }        
                 int temp;
                 int ret = waitpid( pid, &temp, WNOHANG );
+                exitCode = WEXITSTATUS( temp );
                 if ( ret == pid )
                     break;
                 sleepms( 1000 );
@@ -368,6 +370,8 @@ namespace mongo {
             if ( i > 4 || signal == SIGKILL ) {
                 sleepms( 4000 ); // allow operating system to reclaim resources
             }
+            
+            return exitCode;
         }
 
         int getSignal( const BSONObj &a ) {
@@ -386,18 +390,18 @@ namespace mongo {
             assert( a.nFields() == 1 || a.nFields() == 2 );
             assert( a.firstElement().isNumber() );
             int port = int( a.firstElement().number() );
-            killDb( port, 0, getSignal( a ) );
+            int code = killDb( port, 0, getSignal( a ) );
             cout << "shell: stopped mongo program on port " << port << endl;
-            return undefined_;
+            return BSON( "" << code );
         }        
         
         BSONObj StopMongoProgramByPid( const BSONObj &a ) {
             assert( a.nFields() == 1 || a.nFields() == 2 );
             assert( a.firstElement().isNumber() );
             int pid = int( a.firstElement().number() );            
-            killDb( 0, pid, getSignal( a ) );
+            int code = killDb( 0, pid, getSignal( a ) );
             cout << "shell: stopped mongo program on pid " << pid << endl;
-            return undefined_;            
+            return BSON( "" << code );
         }
                                                 
         void KillMongoProgramInstances() {

stdafx.cpp

@@ -32,6 +32,6 @@
 
 namespace mongo {
 
-    const char versionString[] = "1.2.4";
+    const char versionString[] = "1.2.6-pre-";
 
 } // namespace mongo

stdafx.h

@@ -53,6 +53,7 @@ namespace mongo {
         EXIT_OOM_MALLOC = 42 , 
         EXIT_OOM_REALLOC = 43 , 
         EXIT_FS = 45 ,
+        EXIT_CLOCK_SKEW = 47 ,
         EXIT_POSSIBLE_CORRUPTION = 60 , // this means we detected a possible corruption situation, like a buf overflow
         EXIT_UNCAUGHT = 100 , // top level exception that wasn't caught
         EXIT_TEST = 101 ,

util/goodies.h

@@ -242,6 +242,15 @@ namespace mongo {
         return (xt.sec & 0xfffff) * 1000 + t;
     }
 
+    struct Date_t {
+        // TODO: make signed (and look for related TODO's)
+        unsigned long long millis;
+        Date_t(): millis(0) {}
+        Date_t(unsigned long long m): millis(m) {}
+        operator unsigned long long&() { return millis; }
+        operator const unsigned long long&() const { return millis; }
+    };    
+    
     inline unsigned long long jsTime() {
         boost::xtime xt;
         boost::xtime_get(&xt, boost::TIME_UTC);

util/mmap_win.cpp

@@ -99,7 +99,21 @@ namespace mongo {
         return view;
     }
 
-    void MemoryMappedFile::flush(bool) {
-    }
+    void MemoryMappedFile::flush(bool sync) {
+        uassert("Async flushing not supported on windows", sync);
 
+        if (!view) return;
+
+        bool success = FlushViewOfFile(view, 0); // 0 means whole mapping
+        if (!success){
+            int err = GetLastError();
+            out() << "FlushViewOfFile failed " << err << endl;
+        }
+
+        success = FlushFileBuffers(fd);
+        if (!success){
+            int err = GetLastError();
+            out() << "FlushFileBuffers failed " << err << endl;
+        }
+    }
 } 

util/optime.h

@@ -20,15 +20,24 @@
 #include "../db/concurrency.h"
 
 namespace mongo {
+    void exitCleanly( int code );
     
     /* Operation sequence #.  A combination of current second plus an ordinal value.
      */
+    struct ClockSkewException : public DBException {
+        virtual const char* what() const throw() { return "clock skew exception"; }
+        virtual int getCode(){ return 20001; }
+    };
+    
 #pragma pack(4)
     class OpTime {
         unsigned i;
         unsigned secs;
         static OpTime last;
     public:
+        static void setLast(const Date_t &date) {
+            last = OpTime(date);
+        }
         unsigned getSecs() const {
             return secs;
         }
@@ -46,6 +55,20 @@ namespace mongo {
         static OpTime now() {
             unsigned t = (unsigned) time(0);
 //            DEV assertInWriteLock();
+            if ( t < last.secs ){
+                bool toLog = false;
+                ONCE toLog = true;
+                RARELY toLog = true;
+                if ( last.i & 0x80000000 )
+                    toLog = true;
+                if ( toLog )
+                    log() << "clock skew detected  prev: " << last.secs << " now: " << t << " trying to handle..." << endl;
+                if ( last.i & 0x80000000 ) {
+                    log() << "ERROR Large clock skew detected, shutting down" << endl;
+                    throw ClockSkewException();
+                }
+                t = last.secs;
+            }
             if ( last.secs == t ) {
                 last.i++;
                 return last;