From a63c019309929b1cfd6ab54a1547e259aebc392a Mon Sep 17 00:00:00 2001 From: Richard Kreuter Date: Tue, 25 May 2010 18:03:14 -0400 Subject: [PATCH 1/6] Make smoke.py know about suites of tests. --- buildscripts/smoke.py | 90 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py index 5dee97a82d6..9893c1584f5 100644 --- a/buildscripts/smoke.py +++ b/buildscripts/smoke.py @@ -8,6 +8,7 @@ import time import socket from optparse import OptionParser import atexit +import glob mongodExecutable = "./mongod" mongodPort = "32000" @@ -85,7 +86,6 @@ class mongod(object): # return startMongodWithArgs("--master", "--oplogSize", "10") def stop(self): - print "FOO" if not self.proc: print >> sys.stderr, "probable bug: self.proc unset in stop()" return @@ -125,10 +125,11 @@ class TestServerFailure(TestFailure): def __str__(self): return 'mongod not ok after test %s' % self.path -def runTest(path): +def runTest(test): + (path, usedb) = test (ignore, ext) = os.path.splitext(path) if ext == ".js": - argv=[shellExecutable, "--port", mongodPort, path] + argv=filter(lambda x:x, [shellExecutable, "--port", mongodPort] + [None if usedb else "--nodb"] + [path]) elif ext in ["", ".exe"]: argv=[path, "--port", mongodPort] else: @@ -182,32 +183,50 @@ def report(): print "%s\t%d" % (loser, losers[loser]) def main(): - parser = OptionParser(usage="usage: smoke.py [OPTIONS] FILE*") + parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*") + parser.add_option('--mode', dest='mode', default='files', + help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests. (default "files")') + parser.add_option('--mongo-repo', dest='mongoRepo', default=None, + help='Top-level directory of mongo checkout to use. (default: script will make a guess)') parser.add_option('--mongod', dest='mongodExecutable', default='./mongod', - help='Path to mongod to run (default "./mongod")') + help='Path to mongod to run (default "./mongod" in the mongo repo)') parser.add_option('--port', dest='mongodPort', default="32000", help='Port the mongod will bind to (default 32000)') parser.add_option('--mongo', dest='shellExecutable', default="./mongo", - help='Path to mongo, for .js test files (default "./mongo")') + help='Path to mongo, for .js test files (default "./mongo" in the mongo repo)') parser.add_option('--continue-on-failure', dest='continueOnFailure', action="store_true", default=False, - help='If supplied, continue testing even after a failure') + help='If supplied, continue testing even after a test fails') parser.add_option('--one-mongod-per-test', dest='oneMongodPerTest', action="store_true", default=False, help='If supplied, run each test in a fresh mongod') parser.add_option('--from-file', dest='File', - help="Run tests named in FILE, one test per line, '-' means stdin") + help="Run tests/suites named in FILE, one test per line, '-' means stdin") + global tests (options, tests) = parser.parse_args() - + + if options.mongoRepo: + mongoRepo = options.mongoRepo + else: + prefix = '' + while True: + if os.stat(prefix+'buildscripts'): + mongoRepo = os.path.normpath(prefix) + break + else: + prefix += '../' + # FIXME: will this be a device's root directory on + # Windows? + if os.path.samefile('/', prefix): + raise Exception("couldn't guess the mongo repository path") global mongodExecutable, mongodPort, shellExecutable, continueOnFailure, oneMongodPerTest - mongodExecutable = options.mongodExecutable if options.mongodExecutable else mongodExecutable + mongodExecutable = options.mongodExecutable if options.mongodExecutable else os.path.join(mongoRepo, mongodExecutable) mongodPort = options.mongodPort if options.mongodPort else mongodPort - shellExecutable = options.shellExecutable if options.shellExecutable else shellExecutable + shellExecutable = options.shellExecutable if options.shellExecutable else os.path.join(mmongoRepo, shellExecutable) continueOnFailure = options.continueOnFailure if options.continueOnFailure else continueOnFailure oneMongodPerTest = options.oneMongodPerTest if options.oneMongodPerTest else oneMongodPerTest - global tests if options.File: if options.File == '-': tests = sys.stdin.readlines() @@ -216,6 +235,53 @@ def main(): tests = f.readlines() tests = [t.rstrip('\n') for t in tests] + # If we're in suite mode, tests is a list of names of sets of tests. + if options.mode == 'suite': + # Suites: smoke, smokePerf, smokeJs, smokeQuota, smokeJsPerf, + # smokeJsSlow, smokeParalell, smokeClone, smokeRepl, smokeDisk + suites = tests + tests = [] + for suite in suites: + if suite == 'smoke': + (globstr, usedb) = ('test', False) + elif suite == 'smokePerf': + (globstr, usedb) = ('perftest', False) + elif suite == 'smokeClient': + (globstr, usedb) = ('*Test', False) + # (globstr, usedb) = ('*Example', False) + elif suite == 'smokeJs': + # FIXME: _runner.js seems equivalent to "[!_]*.js". + #(globstr, usedb) = ('_runner.js', True) + (globstr, usedb) = ('[!_]*.js', True) + elif suite == 'smokeQuota': + (globstr, usedb) = ('quota/*.js', True) + elif suite == 'smokeJsPerf': + (globstr, usedb) = ('perf/*.js', True) + elif suite == 'smokeJsSlow': + (globstr, usedb) = ('slow/*.js', True) + elif suite == 'smokeParallel': + (globstr, usedb) = ('parallel/*', True) + elif suite == 'smokeClone': + (globstr, usedb) = ('clone/*.js', False) + elif suite == 'smokeRepl': + (globstr, usedb) = ('repl/*.js', False) + elif suite == 'smokeAuth': + (globstr, usedb) = ('auth/*.js', False) + elif suite == 'smokeSharding': + (globstr, usedb) = ('sharding/*.js', False) + elif suite == 'smokeTool': + (globstr, usedb) = ('tool/*.js', False) + else: + raise Exception('unknown test suite %s' % suite) + + globstr = mongoRepo+('/jstests/' if globstr.endswith('.js') else '/')+globstr + print globstr + print glob.glob(globstr) + tests += [(path, usedb) for path in glob.glob(globstr)] + elif options.mode == 'files': + tests = [(os.path.abspath(test), True) for test in tests] + + print tests return runTests(tests) atexit.register(report) From 6199f2db33044be45e4a04df4cd8c557a6dbc479 Mon Sep 17 00:00:00 2001 From: Richard Kreuter Date: Mon, 7 Jun 2010 10:03:07 -0400 Subject: [PATCH 2/6] Rejigger suite support in smoke.py; cleanup children in jstests/shellspawn.js. SERVER-253 --- buildscripts/smoke.py | 126 +++++++++++++++++++++++++----------------- jstests/shellspawn.js | 6 +- 2 files changed, 80 insertions(+), 52 deletions(-) mode change 100644 => 100755 buildscripts/smoke.py diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py old mode 100644 new mode 100755 index 9893c1584f5..d7fa8461783 --- a/buildscripts/smoke.py +++ b/buildscripts/smoke.py @@ -1,5 +1,6 @@ #!/usr/bin/python +from __future__ import with_statement from subprocess import Popen, PIPE, call import os import sys @@ -10,6 +11,9 @@ from optparse import OptionParser import atexit import glob + +mongoRepo = None + mongodExecutable = "./mongod" mongodPort = "32000" shellExecutable = "./mongo" @@ -19,7 +23,9 @@ oneMongodPerTest = False tests = [] winners = [] losers = {} -# grumble + +# This class just implements the with statement API, for a sneaky +# purpose below. class nothing(object): def __enter__(self): return self @@ -37,6 +43,7 @@ class mongod(object): def __exit__(self, type, value, traceback): try: + pass self.stop() except Exception, e: print >> sys.stderr, "error shutting down mongod" @@ -82,6 +89,7 @@ class mongod(object): if not self.didMongodStart( mongodPort ): raise Exception( "Failed to start mongod" ) + # FIXME: need to support this. # def startMongodSmallOplog(env, target, source): # return startMongodWithArgs("--master", "--oplogSize", "10") @@ -89,9 +97,6 @@ class mongod(object): if not self.proc: print >> sys.stderr, "probable bug: self.proc unset in stop()" return - # ??? - #if self.proc.poll() is not None: - # raise Exception( "Failed to start mongod" ) try: # This function not available in Python 2.5 self.proc.terminate() @@ -131,7 +136,11 @@ def runTest(test): if ext == ".js": argv=filter(lambda x:x, [shellExecutable, "--port", mongodPort] + [None if usedb else "--nodb"] + [path]) elif ext in ["", ".exe"]: - argv=[path, "--port", mongodPort] + # Blech. + if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]: + argv=[path] + else: + argv=[path, "--port", mongodPort] else: raise Bug("fell off in extenstion case: %s" % path) print " *******************************************" @@ -151,19 +160,26 @@ def runTest(test): print "" def runTests(tests): - with nothing() if oneMongodPerTest else mongod() as nevermind: + # If we're in one-mongo-per-test mode, we instantiate a nothing + # around the loop, and a mongod inside the loop. We don't + # actually the instance directly. + + # FIXME: some suites of tests start their own mongod, so don't + # need this. (So long as there are no conflicts with port, + # dbpath, etc., and so long as we shut ours down properly, + # starting this mongod shouldn't break anything, though.) + with nothing() if oneMongodPerTest else mongod() as _: for test in tests: try: - with mongod() if oneMongodPerTest else nothing() as nevermind: + with mongod() if oneMongodPerTest else nothing() as _: runTest(test) winners.append(test) except TestFailure, f: try: print f + # Record the failing test and re-raise. losers[f.path] = f.status raise f - # If the server's hosed and we're not in oneMongodPerTest - # mode, there's nothing else we can do. except TestServerFailure, f: if not oneMongodPerTest: return 2 @@ -182,9 +198,53 @@ def report(): for loser in losers: print "%s\t%d" % (loser, losers[loser]) +def expandSuites(suites): + globstr = None + global mongoRepo, tests + for suite in suites: + if suite == 'smokeAll': + tests = [] + expandSuites(['smoke', 'smokePerf', 'smokeClient', 'smokeJs', 'smokeJsPerf', 'smokeJsSlow', 'smokeParallel', 'smokeClone', 'smokeParallel', 'smokeRepl', 'smokeAuth', 'smokeSharding', 'smokeTool']) + break + if suite == 'smoke': + (globstr, usedb) = ('test', False) + elif suite == 'smokePerf': + (globstr, usedb) = ('perftest', False) + elif suite == 'smokeClient': + tests += [(os.path.join(mongoRepo, path), False) for path in ["firstExample", "secondExample", "whereExample", "authTest", "clientTest", "httpClientTest"]] + elif suite == 'smokeJs': + # FIXME: _runner.js seems equivalent to "[!_]*.js". + #(globstr, usedb) = ('_runner.js', True) + (globstr, usedb) = ('[!_]*.js', True) + elif suite == 'smokeQuota': + (globstr, usedb) = ('quota/*.js', True) + elif suite == 'smokeJsPerf': + (globstr, usedb) = ('perf/*.js', True) + elif suite == 'smokeJsSlow': + (globstr, usedb) = ('slow/*.js', True) + elif suite == 'smokeParallel': + (globstr, usedb) = ('parallel/*', True) + elif suite == 'smokeClone': + (globstr, usedb) = ('clone/*.js', False) + elif suite == 'smokeRepl': + (globstr, usedb) = ('repl/*.js', False) + elif suite == 'smokeAuth': + (globstr, usedb) = ('auth/*.js', False) + elif suite == 'smokeSharding': + (globstr, usedb) = ('sharding/*.js', False) + elif suite == 'smokeTool': + (globstr, usedb) = ('tool/*.js', False) + else: + raise Exception('unknown test suite %s' % suite) + + if globstr: + globstr = mongoRepo+('/jstests/' if globstr.endswith('.js') else '/')+globstr + tests += [(path, usedb) for path in glob.glob(globstr)] + return tests + def main(): parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*") - parser.add_option('--mode', dest='mode', default='files', + parser.add_option('--mode', dest='mode', default='suite', help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests. (default "files")') parser.add_option('--mongo-repo', dest='mongoRepo', default=None, help='Top-level directory of mongo checkout to use. (default: script will make a guess)') @@ -205,12 +265,13 @@ def main(): global tests (options, tests) = parser.parse_args() + global mongoRepo if options.mongoRepo: mongoRepo = options.mongoRepo else: prefix = '' while True: - if os.stat(prefix+'buildscripts'): + if os.path.exists(prefix+'buildscripts'): mongoRepo = os.path.normpath(prefix) break else: @@ -220,10 +281,10 @@ def main(): if os.path.samefile('/', prefix): raise Exception("couldn't guess the mongo repository path") - global mongodExecutable, mongodPort, shellExecutable, continueOnFailure, oneMongodPerTest + global mongoRepo, mongodExecutable, mongodPort, shellExecutable, continueOnFailure, oneMongodPerTest mongodExecutable = options.mongodExecutable if options.mongodExecutable else os.path.join(mongoRepo, mongodExecutable) mongodPort = options.mongodPort if options.mongodPort else mongodPort - shellExecutable = options.shellExecutable if options.shellExecutable else os.path.join(mmongoRepo, shellExecutable) + shellExecutable = options.shellExecutable if options.shellExecutable else os.path.join(mongoRepo, shellExecutable) continueOnFailure = options.continueOnFailure if options.continueOnFailure else continueOnFailure oneMongodPerTest = options.oneMongodPerTest if options.oneMongodPerTest else oneMongodPerTest @@ -241,47 +302,10 @@ def main(): # smokeJsSlow, smokeParalell, smokeClone, smokeRepl, smokeDisk suites = tests tests = [] - for suite in suites: - if suite == 'smoke': - (globstr, usedb) = ('test', False) - elif suite == 'smokePerf': - (globstr, usedb) = ('perftest', False) - elif suite == 'smokeClient': - (globstr, usedb) = ('*Test', False) - # (globstr, usedb) = ('*Example', False) - elif suite == 'smokeJs': - # FIXME: _runner.js seems equivalent to "[!_]*.js". - #(globstr, usedb) = ('_runner.js', True) - (globstr, usedb) = ('[!_]*.js', True) - elif suite == 'smokeQuota': - (globstr, usedb) = ('quota/*.js', True) - elif suite == 'smokeJsPerf': - (globstr, usedb) = ('perf/*.js', True) - elif suite == 'smokeJsSlow': - (globstr, usedb) = ('slow/*.js', True) - elif suite == 'smokeParallel': - (globstr, usedb) = ('parallel/*', True) - elif suite == 'smokeClone': - (globstr, usedb) = ('clone/*.js', False) - elif suite == 'smokeRepl': - (globstr, usedb) = ('repl/*.js', False) - elif suite == 'smokeAuth': - (globstr, usedb) = ('auth/*.js', False) - elif suite == 'smokeSharding': - (globstr, usedb) = ('sharding/*.js', False) - elif suite == 'smokeTool': - (globstr, usedb) = ('tool/*.js', False) - else: - raise Exception('unknown test suite %s' % suite) - - globstr = mongoRepo+('/jstests/' if globstr.endswith('.js') else '/')+globstr - print globstr - print glob.glob(globstr) - tests += [(path, usedb) for path in glob.glob(globstr)] + expandSuites(suites) elif options.mode == 'files': tests = [(os.path.abspath(test), True) for test in tests] - print tests return runTests(tests) atexit.register(report) diff --git a/jstests/shellspawn.js b/jstests/shellspawn.js index 5b0de6b6740..7df3c04a929 100644 --- a/jstests/shellspawn.js +++ b/jstests/shellspawn.js @@ -14,11 +14,15 @@ else { spawn = startMongoProgramNoConnect( "mongo", "--port", myPort(), "--eval", "print( 'I am a shell' );" ); + stopMongoProgramByPid( spawn ); + spawn = startMongoProgramNoConnect( "mongo", "--port", myPort() ); + + stopMongoProgramByPid( spawn ); spawn = startMongoProgramNoConnect( "mongo", "--port", myPort() ); stopMongoProgramByPid( spawn ); - + // all these shells should be killed } From a4c850a06de9a14f99d1c93c236beab983573f6b Mon Sep 17 00:00:00 2001 From: Eliot Horowitz Date: Mon, 7 Jun 2010 10:55:43 -0400 Subject: [PATCH 3/6] nothing yet --- docs/building.opensolaris.ec2.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/building.opensolaris.ec2.md diff --git a/docs/building.opensolaris.ec2.md b/docs/building.opensolaris.ec2.md new file mode 100644 index 00000000000..c8278b83a27 --- /dev/null +++ b/docs/building.opensolaris.ec2.md @@ -0,0 +1,14 @@ + +ami-4133d528 + + +pkg install SUNWgcc +pkg install SUNWgit +pkg install SUNWpython-setuptools + +easy_install-2.4 scons + + +git clone git://github.com/mongodb/mongo.git +cd mongo +scons From 8c6a00675f53749ea0ebea1e7027c199e24df7e5 Mon Sep 17 00:00:00 2001 From: Eliot Horowitz Date: Mon, 7 Jun 2010 10:56:05 -0400 Subject: [PATCH 4/6] perf temp file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d43a15dfd87..03424e335ab 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .dbshell .sconsign.dblite .sconf_temp +perf.data *~ *.o From 11dd486bdb40d950d5b716bbd44befc8e365570f Mon Sep 17 00:00:00 2001 From: Richard Kreuter Date: Mon, 7 Jun 2010 11:01:41 -0400 Subject: [PATCH 5/6] Comment out the --mongo-repo option in smoke.py for now. SERVER-253 --- buildscripts/smoke.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py index d7fa8461783..6e27292bcc5 100755 --- a/buildscripts/smoke.py +++ b/buildscripts/smoke.py @@ -245,15 +245,17 @@ def expandSuites(suites): def main(): parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*") parser.add_option('--mode', dest='mode', default='suite', - help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests. (default "files")') - parser.add_option('--mongo-repo', dest='mongoRepo', default=None, - help='Top-level directory of mongo checkout to use. (default: script will make a guess)') - parser.add_option('--mongod', dest='mongodExecutable', default='./mongod', - help='Path to mongod to run (default "./mongod" in the mongo repo)') + help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests. (default "suite")') + # Some of our tests hard-code pathnames e.g., to execute, so until + # th we don't have the freedom to run from anyplace. +# parser.add_option('--mongo-repo', dest='mongoRepo', default=None, +# help='Top-level directory of mongo checkout to use. (default: script will make a guess)') + parser.add_option('--mongod', dest='mongodExecutable', #default='./mongod', + help='Path to mongod to run (default "./mongod")') parser.add_option('--port', dest='mongodPort', default="32000", help='Port the mongod will bind to (default 32000)') - parser.add_option('--mongo', dest='shellExecutable', default="./mongo", - help='Path to mongo, for .js test files (default "./mongo" in the mongo repo)') + parser.add_option('--mongo', dest='shellExecutable', #default="./mongo", + help='Path to mongo, for .js test files (default "./mongo")') parser.add_option('--continue-on-failure', dest='continueOnFailure', action="store_true", default=False, help='If supplied, continue testing even after a test fails') @@ -266,8 +268,9 @@ def main(): (options, tests) = parser.parse_args() global mongoRepo - if options.mongoRepo: - mongoRepo = options.mongoRepo + if False: #options.mongoRepo: + pass + #mongoRepo = options.mongoRepo else: prefix = '' while True: @@ -282,9 +285,9 @@ def main(): raise Exception("couldn't guess the mongo repository path") global mongoRepo, mongodExecutable, mongodPort, shellExecutable, continueOnFailure, oneMongodPerTest - mongodExecutable = options.mongodExecutable if options.mongodExecutable else os.path.join(mongoRepo, mongodExecutable) + mongodExecutable = options.mongodExecutable if options.mongodExecutable else os.path.join(mongoRepo, 'mongod') mongodPort = options.mongodPort if options.mongodPort else mongodPort - shellExecutable = options.shellExecutable if options.shellExecutable else os.path.join(mongoRepo, shellExecutable) + shellExecutable = options.shellExecutable if options.shellExecutable else os.path.join(mongoRepo, 'mongo') continueOnFailure = options.continueOnFailure if options.continueOnFailure else continueOnFailure oneMongodPerTest = options.oneMongodPerTest if options.oneMongodPerTest else oneMongodPerTest From b9c427ec29891dd62a3a0d6ccbbb46a785f244a6 Mon Sep 17 00:00:00 2001 From: Eliot Horowitz Date: Mon, 7 Jun 2010 11:06:02 -0400 Subject: [PATCH 6/6] plumbing for shell auto-complete --- shell/dbshell.cpp | 70 +++++++++++++++++++++++++++++++++++++++++++---- shell/utils.js | 7 +++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/shell/dbshell.cpp b/shell/dbshell.cpp index 3f9449f6867..bbec715729c 100644 --- a/shell/dbshell.cpp +++ b/shell/dbshell.cpp @@ -37,6 +37,11 @@ jmp_buf jbuf; using namespace std; using namespace boost::filesystem; +using mongo::BSONObj; +using mongo::BSONObjBuilder; +using mongo::BSONObjIterator; +using mongo::BSONElement; + string historyFile; bool gotInterrupted = 0; bool inMultiLine = 0; @@ -45,9 +50,60 @@ bool inMultiLine = 0; #define CTRLC_HANDLE #endif -static char** my_completion(const char* text , int start ,int end ){ - cout << "YO [" << text << "] " << start << " " << end << endl; - return 0; +mongo::Scope * shellMainScope; + +void generateCompletions( const string& prefix , vector& all ){ + if ( prefix.find( '"' ) != string::npos ) + return; + shellMainScope->exec( "shellAutocomplete( \"" + prefix + "\" );" , "autocomplete help" , false , true , false ); + + BSONObjBuilder b; + shellMainScope->append( b , "" , "__autocomplete__" ); + BSONObj res = b.obj(); + BSONObj arr = res.firstElement().Obj(); + + BSONObjIterator i(arr); + while ( i.more() ){ + BSONElement e = i.next(); + all.push_back( e.String() ); + } + +} + +static char** completionHook(const char* text , int start ,int end ){ + static map m; + + vector all; + + if ( start == 0 ){ + generateCompletions( string(text,end) , all ); + } + + if ( all.size() == 0 ){ + rl_bind_key('\t',rl_abort); + return 0; + } + + string longest = all[0]; + for ( vector::iterator i=all.begin(); i!=all.end(); ++i ){ + string s = *i; + for ( unsigned j=0; jsetScopeInitCallback( mongo::shellUtils::initScope ); auto_ptr< mongo::Scope > scope( mongo::globalScriptEngine->newScope() ); + shellMainScope = scope.get(); if ( !script.empty() ) { mongo::shellUtils::MongoProgramScope s; diff --git a/shell/utils.js b/shell/utils.js index 67c3fc50b2a..c42f31c910c 100644 --- a/shell/utils.js +++ b/shell/utils.js @@ -750,6 +750,13 @@ shellPrintHelper = function( x ){ print( tojson( x ) ); } +shellAutocomplete = function( prefix ){ + var a = []; + //a.push( prefix + "z" ) + //a.push( prefix + "y" ) + __autocomplete__ = a; +} + shellHelper = function( command , rest , shouldPrint ){ command = command.trim(); var args = rest.trim().replace(/;$/,"").split( "\s+" );