From dd79445bbbdce5b24b3cd51113b4f35d264f9f40 Mon Sep 17 00:00:00 2001 From: Mathias Stearn Date: Thu, 22 Jul 2010 21:00:30 -0400 Subject: [PATCH] make more commands work with mongos SERVER-1462 --- jstests/sharding/count1.js | 1 - .../slowNightly/run_sharding_passthrough.js | 16 +- s/chunk.cpp | 15 ++ s/chunk.h | 2 +- s/commands_public.cpp | 184 +++++++++++++++++- s/config.cpp | 9 + s/config.h | 2 + shell/collection.js | 6 +- shell/mongo_vstudio.cpp | 24 +-- 9 files changed, 233 insertions(+), 26 deletions(-) diff --git a/jstests/sharding/count1.js b/jstests/sharding/count1.js index 4fb6d747f49..94aec2acf56 100644 --- a/jstests/sharding/count1.js +++ b/jstests/sharding/count1.js @@ -36,7 +36,6 @@ assert.eq( 6 , db.foo.find().sort( { name : 1 } ).count() , "basic count after s s.adminCommand( { movechunk : "test.foo" , find : { name : "joe" } , to : secondary.getMongo().name } ); - assert.eq( 3 , primary.foo.find().toArray().length , "primary count" ); assert.eq( 3 , secondary.foo.find().toArray().length , "secondary count" ); assert.eq( 3 , primary.foo.find().sort( { name : 1 } ).toArray().length , "primary count sorted" ); diff --git a/jstests/slowNightly/run_sharding_passthrough.js b/jstests/slowNightly/run_sharding_passthrough.js index 66ce8ee4e7f..5f8392e9f94 100644 --- a/jstests/slowNightly/run_sharding_passthrough.js +++ b/jstests/slowNightly/run_sharding_passthrough.js @@ -52,9 +52,19 @@ files.forEach( // getnonce logout medianKey profile reIndex repairDatabase // reseterror splitVector validate - // These all fail due to missing commands: - if (/[\/\\](apitest_db|apitest_dbcollection|auth1|auth2|basic1|basic2|capped1|capped3|capped4|copydb-auth|cursor6|datasize|datasize2|dbadmin|dbhash|drop|dropIndex|error3|evalb|find1|find3|in5|index1|index10|index2|index.*|jni1|jni2|jni3|jni4|jni8|median|multi2|profile1|recstore|remove|remove2|repair|sort1|sort2|sort4|sort5|sort_numeric|splitvector|unique2|update)\.js$/.test(x.name)) { - print(" !!!!!!!!!!!!!!! skipping test that fails under sharding (missing command) " + x.name) + /* missing commands : + * forceerror and switchtoclienterrors + * cloneCollectionAsCapped + * splitvector + * profile (apitest_db, cursor6, evalb) + * copydbgetnonce + * dbhash + * medianKey + * clean (apitest_dbcollection) + * logout and getnonce + */ + if (/[\/\\](error3|capped3|splitvector|apitest_db|cursor6|copydb-auth|profile1|dbhash|median|apitest_dbcollection|evalb|auth1|auth2)\.js$/.test(x.name)) { + print(" !!!!!!!!!!!!!!! skipping test that has failed under sharding but might not anymore " + x.name) return; } // These are bugs (some might be fixed now): diff --git a/s/chunk.cpp b/s/chunk.cpp index fe6bbbc97fb..b823c030e7d 100644 --- a/s/chunk.cpp +++ b/s/chunk.cpp @@ -744,6 +744,21 @@ namespace mongo { } } + void ChunkManager::getShardsForRange(set& shards, const BSONObj& min, const BSONObj& max){ + uassert(13405, "min must have shard key", hasShardKey(min)); + uassert(13406, "max must have shard key", hasShardKey(max)); + + ChunkRangeMap::const_iterator it = _chunkRanges.upper_bound(min); + ChunkRangeMap::const_iterator end = _chunkRanges.lower_bound(max); + + for (; it!=end; ++ it){ + shards.insert(it->second->getShard()); + + // once we know we need to visit all shards no need to keep looping + if (shards.size() == _shards.size()) + break; + } + } void ChunkManager::getAllShards( set& all ){ rwlock lk( _lock , false ); diff --git a/s/chunk.h b/s/chunk.h index e7081b11da8..9369943c680 100644 --- a/s/chunk.h +++ b/s/chunk.h @@ -267,8 +267,8 @@ namespace mongo { void maybeChunkCollection(); void getShardsForQuery( set& shards , const BSONObj& query ); - void getAllShards( set& all ); + void getShardsForRange(set& shards, const BSONObj& min, const BSONObj& max); // [min, max) void save(); diff --git a/s/commands_public.cpp b/s/commands_public.cpp index ad442408cf2..f116e9111e0 100644 --- a/s/commands_public.cpp +++ b/s/commands_public.cpp @@ -35,7 +35,7 @@ namespace mongo { class PublicGridCommand : public Command { public: - PublicGridCommand( const char * n ) : Command( n ){ + PublicGridCommand( const char* n, const char* oldname=NULL ) : Command( n, false, oldname ){ } virtual bool slaveOk() const { return true; @@ -65,6 +65,79 @@ namespace mongo { return ok; } }; + + class RunOnAllShardsCommand : public Command { + public: + RunOnAllShardsCommand(const char* n, const char* oldname=NULL) : Command(n, false, oldname) {} + + virtual bool slaveOk() const { return true; } + virtual bool adminOnly() const { return false; } + + // all grid commands are designed not to lock + virtual LockType locktype() const { return NONE; } + + + // default impl uses all shards for DB + virtual void getShards(const string& dbName , BSONObj& cmdObj, set& shards){ + DBConfigPtr conf = grid.getDBConfig( dbName , false ); + conf->getAllShards(shards); + } + + virtual void aggregateResults(const vector& results, BSONObjBuilder& output) {} + + // don't override + virtual bool run(const string& dbName , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& output, bool){ + set shards; + getShards(dbName, cmdObj, shards); + + list< shared_ptr > futures; + for ( set::const_iterator i=shards.begin(), end=shards.end() ; i != end ; i++ ){ + futures.push_back( Future::spawnCommand( i->getConnString() , dbName , cmdObj ) ); + } + + vector results; + BSONObjBuilder subobj (output.subobjStart("raw")); + BSONObjBuilder errors; + for ( list< shared_ptr >::iterator i=futures.begin(); i!=futures.end(); i++ ){ + shared_ptr res = *i; + if ( ! res->join() ){ + errors.appendAs(res->result()["errmsg"], res->getServer()); + } + results.push_back( res->result() ); + subobj.append( res->getServer() , res->result() ); + } + + subobj.done(); + + BSONObj errobj = errors.done(); + if (! errobj.isEmpty()){ + errmsg = errobj.toString(false, true); + return false; + } + + aggregateResults(results, output); + return true; + } + + }; + + class AllShardsCollectionCommand : public RunOnAllShardsCommand { + public: + AllShardsCollectionCommand(const char* n, const char* oldname=NULL) : RunOnAllShardsCommand(n, oldname) {} + + virtual void getShards(const string& dbName , BSONObj& cmdObj, set& shards){ + string fullns = dbName + '.' + cmdObj.firstElement().valuestrsafe(); + + DBConfigPtr conf = grid.getDBConfig( dbName , false ); + + if ( ! conf || ! conf->isShardingEnabled() || ! conf->isSharded( fullns ) ){ + shards.insert(conf->getShard(fullns)); + } else { + conf->getChunkManager(fullns)->getAllShards(shards); + } + } + }; + class NotAllowedOnShardedCollectionCmd : public PublicGridCommand { public: @@ -87,6 +160,62 @@ namespace mongo { // ---- + class DropIndexesCmd : public AllShardsCollectionCommand { + public: + DropIndexesCmd() : AllShardsCollectionCommand("dropIndexes", "deleteIndexes") {} + } dropIndexesCmd; + + class ReIndexCmd : public AllShardsCollectionCommand { + public: + ReIndexCmd() : AllShardsCollectionCommand("reIndex") {} + } reIndexCmd; + + class ValidateCmd : public AllShardsCollectionCommand { + public: + ValidateCmd() : AllShardsCollectionCommand("validate") {} + } validateCmd; + + class RepairDatabaseCmd : public RunOnAllShardsCommand { + public: + RepairDatabaseCmd() : RunOnAllShardsCommand("repairDatabase") {} + } repairDatabaseCmd; + + class DBStatsCmd : public RunOnAllShardsCommand { + public: + DBStatsCmd() : RunOnAllShardsCommand("dbstats") {} + + virtual void aggregateResults(const vector& results, BSONObjBuilder& output) { + long long objects = 0; + long long dataSize = 0; + long long storageSize = 0; + long long numExtents = 0; + long long indexes = 0; + long long indexSize = 0; + long long fileSize = 0; + + for (vector::const_iterator it(results.begin()), end(results.end()); it != end; ++it){ + const BSONObj& b = *it; + objects += b["objects"].numberLong(); + dataSize += b["dataSize"].numberLong(); + storageSize += b["storageSize"].numberLong(); + numExtents += b["numExtents"].numberLong(); + indexes += b["indexes"].numberLong(); + indexSize += b["indexSize"].numberLong(); + fileSize += b["fileSize"].numberLong(); + } + + //result.appendNumber( "collections" , ncollections ); //TODO: need to find a good way to get this + output.appendNumber( "objects" , objects ); + output.append ( "avgObjSize" , double(dataSize) / double(objects) ); + output.appendNumber( "dataSize" , dataSize ); + output.appendNumber( "storageSize" , storageSize); + output.appendNumber( "numExtents" , numExtents ); + output.appendNumber( "indexes" , indexes ); + output.appendNumber( "indexSize" , indexSize ); + output.appendNumber( "fileSize" , fileSize ); + } + } DBStatsCmdObj; + class DropCmd : public PublicGridCommand { public: DropCmd() : PublicGridCommand( "drop" ){} @@ -420,6 +549,59 @@ namespace mongo { } findAndModifyCmd; + class DataSizeCmd : public PublicGridCommand { + public: + DataSizeCmd() : PublicGridCommand("dataSize", "datasize") { } + bool run(const string& dbName, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){ + string fullns = cmdObj.firstElement().String(); + + DBConfigPtr conf = grid.getDBConfig( dbName , false ); + + if ( ! conf || ! conf->isShardingEnabled() || ! conf->isSharded( fullns ) ){ + return passthrough( conf , cmdObj , result); + } + + ChunkManagerPtr cm = conf->getChunkManager( fullns ); + massert( 13407 , "how could chunk manager be null!" , cm ); + + BSONObj min = cmdObj.getObjectField( "min" ); + BSONObj max = cmdObj.getObjectField( "max" ); + BSONObj keyPattern = cmdObj.getObjectField( "keyPattern" ); + + uassert(13408, "keyPattern must equal shard key", cm->getShardKey().key() == keyPattern); + + // yes these are doubles... + double size = 0; + double numObjects = 0; + int millis = 0; + + set shards; + cm->getShardsForRange(shards, min, max); + for ( set::iterator i=shards.begin(), end=shards.end() ; i != end; ++i ){ + ShardConnection conn( *i , fullns ); + BSONObj res; + bool ok = conn->runCommand( conf->getName() , cmdObj , res ); + conn.done(); + + if ( ! ok ){ + result.appendElements( res ); + return false; + } + + size += res["size"].number(); + numObjects += res["numObjects"].number(); + millis += res["millis"].numberInt(); + + } + + result.append( "size", size ); + result.append( "numObjects" , numObjects ); + result.append( "millis" , millis ); + return true; + } + + } DataSizeCmd; + class ConvertToCappedCmd : public NotAllowedOnShardedCollectionCmd { public: ConvertToCappedCmd() : NotAllowedOnShardedCollectionCmd("convertToCapped"){} diff --git a/s/config.cpp b/s/config.cpp index f2128354db5..cce007939e3 100644 --- a/s/config.cpp +++ b/s/config.cpp @@ -375,6 +375,15 @@ namespace mongo { return true; } + void DBConfig::getAllShards(set& shards) const{ + shards.insert(getPrimary()); + for (Collections::const_iterator it(_collections.begin()), end(_collections.end()); it != end; ++it){ + if (it->second.isSharded()){ + it->second.getCM()->getAllShards(shards); + } // TODO: handle collections on non-primary shard + } + } + /* --- Grid --- */ DBConfigPtr Grid::getDBConfig( string database , bool create ){ diff --git a/s/config.h b/s/config.h index a8410890e28..aba0f3fd180 100644 --- a/s/config.h +++ b/s/config.h @@ -160,6 +160,8 @@ namespace mongo { */ bool unserialize(const BSONObj& from); + void getAllShards(set& shards) const; + protected: /** diff --git a/shell/collection.js b/shell/collection.js index 1f06a132856..68ee03de29d 100644 --- a/shell/collection.js +++ b/shell/collection.js @@ -355,8 +355,10 @@ DBCollection.prototype.validate = function() { res.valid = false; - if ( res.result ){ - var str = "-" + tojson( res.result ); + var raw = res.result || res.raw; + + if ( raw ){ + var str = "-" + tojson( raw ); res.valid = ! ( str.match( /exception/ ) || str.match( /corrupt/ ) ); var p = /lastExtentSize:(\d+)/; diff --git a/shell/mongo_vstudio.cpp b/shell/mongo_vstudio.cpp index ba90c3cd164..8d512693281 100644 --- a/shell/mongo_vstudio.cpp +++ b/shell/mongo_vstudio.cpp @@ -649,40 +649,34 @@ const char * jsconcatcode = "by = b[key];}\n" "return Math.sqrt( Math.pow( by - ay , 2 ) +\n" "Math.pow( bx - ax , 2 ) );}\n" - "rs = function () { return \"try rs.help()\"; }\n" + "rs = function () { \"try rs.help()\" }\n" "rs.help = function () {\n" "print(\"\\trs.status() { replSetGetStatus : 1 } checks repl set status\");\n" "print(\"\\trs.initiate() { replSetInitiate : null } initiates set with default settings\");\n" "print(\"\\trs.initiate(cfg) { replSetInitiate : cfg } initiates set with configuration cfg\");\n" "print(\"\\trs.add(hostportstr) add a new member to the set with default attributes\");\n" "print(\"\\trs.add(membercfgobj) add a new member to the set with extra attributes\");\n" - "print(\"\\trs.addArb(hostportstr) add a new member which is arbiterOnly:true\");\n" - "print(\"\\trs.stepDown() step down as primary (momentarily)\");\n" "print(\"\\trs.conf() return configuration from local.system.replset\");\n" "print();\n" "print(\"\\tdb.isMaster() check who is primary\");\n" "print();\n" - "print(\"\\tsee also http://:28017/_replSet for additional diagnostic info\");}\n" + "print(\"\\tsee also http://:28017/_replSet for additional diagnostic info\");}\n" "rs.status = function () { return db._adminCommand(\"replSetGetStatus\"); }\n" "rs.initiate = function (c) { return db._adminCommand({ replSetInitiate: c }); }\n" - "rs.add = function (hostport, arb) {\n" + "rs.add = function (hostport) {\n" "var cfg = hostport;\n" "var local = db.getSisterDB(\"local\");\n" - "assert(local.system.replset.count() <= 1, \"error: local.system.replset has unexpected contents\");\n" + "assert(local.system.replset.count() == 1, \"error: local.system.replset unexpected (or empty) contents\");\n" "var c = local.system.replset.findOne();\n" "assert(c, \"no config object retrievable from local.system.replset\");\n" "c.version++;\n" "var max = 0;\n" "for (var i in c.members)\n" "if (c.members[i]._id > max) max = c.members[i]._id;\n" - "if (isString(hostport)) {\n" + "if (isString(hostport))\n" "cfg = { _id: max + 1, host: hostport };\n" - "if (arb)\n" - "cfg.arbiterOnly = true;}\n" "c.members.push(cfg);\n" "return db._adminCommand({ replSetReconfig: c });}\n" - "rs.stepDown = function () { return db._adminCommand({ replSetStepDown: 1 }); }\n" - "rs.addArb = function (hn) { return this.add(hn, true); }\n" "rs.conf = function () { return db.getSisterDB(\"local\").system.replset.findOne(); }\n" "help = shellHelper.help = function (x) {\n" "if (x == \"connect\") {\n" @@ -725,10 +719,10 @@ const char * jsconcatcode = "return;}\n" "print(\"\\t\" + \"db.help() help on db methods\");\n" "print(\"\\t\" + \"db.mycoll.help() help on collection methods\");\n" - "print(\"\\t\" + \"rs.help() help on replica set methods\");\n" "print(\"\\t\" + \"help connect connecting to a db help\");\n" "print(\"\\t\" + \"help admin administrative help\");\n" "print(\"\\t\" + \"help misc misc things to know\");\n" + "print(\"\\t\" + \"rs.help() help on replica set methods\");\n" "print();\n" "print(\"\\t\" + \"show dbs show database names\");\n" "print(\"\\t\" + \"show collections show collections in current database\");\n" @@ -1288,12 +1282,6 @@ const char * jsconcatcode = "throw \"error: \" + tojson( ret );\n" "this._numReturned++;\n" "return ret;}\n" - "DBQuery.prototype.objsLeftInBatch = function(){\n" - "this._exec();\n" - "var ret = this._cursor.objsLeftInBatch();\n" - "if ( ret.$err )\n" - "throw \"error: \" + tojson( ret );\n" - "return ret;}\n" "DBQuery.prototype.toArray = function(){\n" "if ( this._arr )\n" "return this._arr;\n"