From 62aad0ee18bc8a9430a8aa7d81ed0ae75b82fd2f Mon Sep 17 00:00:00 2001 From: Eliot Horowitz Date: Mon, 15 Feb 2010 22:33:27 -0500 Subject: [PATCH] refactoring runQuery / UserQueryOp part 1 --- client/parallel.cpp | 5 +- db/clientcursor.h | 2 +- db/dbmessage.h | 8 +- db/instance.cpp | 3 - db/query.cpp | 137 +++++++++------------------------ db/query.h | 168 +++++++++++++++++++++++++++++++++++++++++ dbtests/querytests.cpp | 30 ++++++++ 7 files changed, 239 insertions(+), 114 deletions(-) diff --git a/client/parallel.cpp b/client/parallel.cpp index fb3993c05c3..bd29013e1d2 100644 --- a/client/parallel.cpp +++ b/client/parallel.cpp @@ -26,13 +26,12 @@ namespace mongo { // -------- ClusteredCursor ----------- - + ClusteredCursor::ClusteredCursor( QueryMessage& q ){ _ns = q.ns; _query = q.query.copy(); _options = q.queryOptions; - if ( q.fields.get() ) - _fields = q.fields->getSpec(); + _fields = q.fields; _done = false; } diff --git a/db/clientcursor.h b/db/clientcursor.h index 260952b055f..04296c0236a 100644 --- a/db/clientcursor.h +++ b/db/clientcursor.h @@ -123,7 +123,7 @@ namespace mongo { return _lastLoc; } - auto_ptr< FieldMatcher > filter; // which fields query wants returned + shared_ptr< FieldMatcher > fields; // which fields query wants returned Message originalMessage; // this is effectively an auto ptr for data the matcher points to /* Get rid of cursors for namespaces that begin with nsprefix. diff --git a/db/dbmessage.h b/db/dbmessage.h index 4afda0d2711..96170383471 100644 --- a/db/dbmessage.h +++ b/db/dbmessage.h @@ -180,7 +180,7 @@ namespace mongo { int ntoreturn; int queryOptions; BSONObj query; - auto_ptr< FieldMatcher > fields; + BSONObj fields; /* parses the message into the above fields */ QueryMessage(DbMessage& d) { @@ -189,11 +189,7 @@ namespace mongo { ntoreturn = d.pullInt(); query = d.nextJsObj(); if ( d.moreJSObjs() ) { - BSONObj o = d.nextJsObj(); - if (!o.isEmpty()){ - fields = auto_ptr< FieldMatcher >(new FieldMatcher() ); - fields->add( o ); - } + fields = d.nextJsObj(); } queryOptions = d.msg().data->dataAsInt(); } diff --git a/db/instance.cpp b/db/instance.cpp index 495bf142154..bff6849408b 100644 --- a/db/instance.cpp +++ b/db/instance.cpp @@ -168,9 +168,6 @@ namespace mongo { CurOp& op = *(c.curop()); try { - if (q.fields.get() && q.fields->errmsg) - uassert( 10053 , q.fields->errmsg, false); - msgdata = runQuery(m, q, op ).release(); } catch ( AssertionException& e ) { diff --git a/db/query.cpp b/db/query.cpp index 7d2a98b0001..706bbfaa202 100644 --- a/db/query.cpp +++ b/db/query.cpp @@ -232,29 +232,6 @@ namespace mongo { BSONObj id_obj = fromjson("{\"_id\":1}"); BSONObj empty_obj = fromjson("{}"); - /* This is for languages whose "objects" are not well ordered (JSON is well ordered). - [ { a : ... } , { b : ... } ] -> { a : ..., b : ... } - */ - inline BSONObj transformOrderFromArrayFormat(BSONObj order) { - /* note: this is slow, but that is ok as order will have very few pieces */ - BSONObjBuilder b; - char p[2] = "0"; - - while ( 1 ) { - BSONObj j = order.getObjectField(p); - if ( j.isEmpty() ) - break; - BSONElement e = j.firstElement(); - uassert( 10102 , "bad order array", !e.eoo()); - uassert( 10103 , "bad order array [2]", e.isNumber()); - b.append(e); - (*p)++; - uassert( 10104 , "too many ordering elements", *p <= '9'); - } - - return b.obj(); - } - //int dump = 0; @@ -324,7 +301,7 @@ namespace mongo { } else { BSONObj js = c->current(); - fillQueryResultFromObj(b, cc->filter.get(), js); + fillQueryResultFromObj(b, cc->fields.get(), js); n++; if ( (ntoreturn>0 && (n >= ntoreturn || b.len() > MaxBytesToReturnToClientAtOnce)) || (ntoreturn==0 && b.len()>1*1024*1024) ) { @@ -683,7 +660,7 @@ namespace mongo { } finish(); return; - } + } } } } @@ -743,11 +720,10 @@ namespace mongo { /* run a query -- includes checking for and running a Command */ auto_ptr< QueryResult > runQuery(Message& m, QueryMessage& q, CurOp& curop ) { StringBuilder& ss = curop.debug().str; + ParsedQuery pq( q ); const char *ns = q.ns; int ntoskip = q.ntoskip; - int _ntoreturn = q.ntoreturn; BSONObj jsobj = q.query; - auto_ptr< FieldMatcher > filter = q.fields; // what fields to return (unspecified = full object) int queryOptions = q.queryOptions; BSONObj snapshotHint; @@ -755,17 +731,7 @@ namespace mongo { log() << "runQuery: " << ns << jsobj << endl; long long nscanned = 0; - bool wantMore = true; - int ntoreturn = _ntoreturn; - if ( _ntoreturn < 0 ) { - /* _ntoreturn greater than zero is simply a hint on how many objects to send back per - "cursor batch". - A negative number indicates a hard limit. - */ - ntoreturn = -_ntoreturn; - wantMore = false; - } - ss << "query " << ns << " ntoreturn:" << ntoreturn; + ss << "query " << ns << " ntoreturn:" << pq.getNumToReturn(); curop.setQuery(jsobj); BSONObjBuilder cmdResBuf; @@ -775,8 +741,8 @@ namespace mongo { int n = 0; Client& c = cc(); - /* we assume you are using findOne() for running a cmd... */ - if ( ntoreturn == 1 && strstr( ns , ".$cmd" ) ){ + + if ( pq.couldBeCommand() ){ BufBuilder bb; bb.skip(sizeof(QueryResult)); @@ -807,65 +773,34 @@ namespace mongo { so that queries to a pair are realtime consistent as much as possible. use setSlaveOk() to query the nonmaster member of a replica pair. */ - uassert( 10107 , "not master", isMaster() || (queryOptions & QueryOption_SlaveOk) || replSettings.slave == SimpleSlave ); + uassert( 10107 , "not master" , isMaster() || pq.hasOption( QueryOption_SlaveOk ) || replSettings.slave == SimpleSlave ); - BSONElement hint; - BSONObj min; - BSONObj max; - bool explain = false; - bool _gotquery = false; - bool snapshot = false; - BSONObj query; - { - BSONElement e = jsobj.getField("$query"); - if ( e.eoo() ) - e = jsobj.getField("query"); - if ( !e.eoo() && (e.type() == Object || e.type() == Array) ) { - query = e.embeddedObject(); - _gotquery = true; - } - } - BSONObj order; - { - BSONElement e = jsobj.getField("$orderby"); - if ( e.eoo() ) - e = jsobj.getField("orderby"); - if ( !e.eoo() ) { - order = e.embeddedObjectUserCheck(); - if ( e.type() == Array ) - order = transformOrderFromArrayFormat(order); - } - } - if ( !_gotquery && order.isEmpty() ) - query = jsobj; - else { - explain = jsobj.getBoolField("$explain"); - if ( useHints ) - hint = jsobj.getField("$hint"); - min = jsobj.getObjectField("$min"); - max = jsobj.getObjectField("$max"); - BSONElement e = jsobj.getField("$snapshot"); - snapshot = !e.eoo() && e.trueValue(); - if( snapshot ) { - uassert( 12001 , "E12001 can't sort with $snapshot", order.isEmpty()); - uassert( 12002 , "E12002 can't use hint with $snapshot", hint.eoo()); - NamespaceDetails *d = nsdetails(ns); - if ( d ){ - int i = d->findIdIndex(); - if( i < 0 ) { - if ( strstr( ns , ".system." ) == 0 ) - log() << "warning: no _id index on $snapshot query, ns:" << ns << endl; - } - else { - /* [dm] the name of an _id index tends to vary, so we build the hint the hard way here. - probably need a better way to specify "use the _id index" as a hint. if someone is - in the query optimizer please fix this then! - */ - BSONObjBuilder b; - b.append("$hint", d->idx(i).indexName()); - snapshotHint = b.obj(); - hint = snapshotHint.firstElement(); - } + BSONElement hint = useHints ? pq.getHint() : BSONElement(); + BSONObj min = pq.getMin(); + BSONObj max = pq.getMax(); + bool explain = pq.isExplain(); + //bool _gotquery = false; + bool snapshot = pq.isSnapshot(); + BSONObj query = pq.getFilter(); + BSONObj order = pq.getOrder(); + + if( snapshot ) { + NamespaceDetails *d = nsdetails(ns); + if ( d ){ + int i = d->findIdIndex(); + if( i < 0 ) { + if ( strstr( ns , ".system." ) == 0 ) + log() << "warning: no _id index on $snapshot query, ns:" << ns << endl; + } + else { + /* [dm] the name of an _id index tends to vary, so we build the hint the hard way here. + probably need a better way to specify "use the _id index" as a hint. if someone is + in the query optimizer please fix this then! + */ + BSONObjBuilder b; + b.append("$hint", d->idx(i).indexName()); + snapshotHint = b.obj(); + hint = snapshotHint.firstElement(); } } } @@ -897,7 +832,7 @@ namespace mongo { ss << " idhack "; if ( found ){ n = 1; - fillQueryResultFromObj( bb , filter.get() , resObject ); + fillQueryResultFromObj( bb , pq.getFields() , resObject ); } qr.reset( (QueryResult *) bb.buf() ); bb.decouple(); @@ -921,7 +856,7 @@ namespace mongo { oldPlan = qps.explain(); } QueryPlanSet qps( ns, query, order, &hint, !explain, min, max ); - UserQueryOp original( ntoskip, ntoreturn, order, wantMore, explain, filter.get(), queryOptions ); + UserQueryOp original( ntoskip, pq.getNumToReturn(), order, pq.wantMore(), explain, pq.getFields() , queryOptions ); shared_ptr< UserQueryOp > o = qps.runOp( original ); UserQueryOp &dqo = *o; massert( 10362 , dqo.exceptionMessage(), dqo.complete() ); @@ -939,7 +874,7 @@ namespace mongo { DEV out() << " query has more, cursorid: " << cursorid << endl; cc->matcher = dqo.matcher(); cc->pos = n; - cc->filter = filter; + cc->fields = pq.getFieldPtr(); cc->originalMessage = m; cc->updateLocation(); if ( !cc->c->ok() && cc->c->tailable() ) { diff --git a/db/query.h b/db/query.h index bc9b183c4c1..c9cdfcb7d83 100644 --- a/db/query.h +++ b/db/query.h @@ -110,6 +110,174 @@ namespace mongo { auto_ptr< QueryResult > runQuery(Message& m, QueryMessage& q, CurOp& curop ); + /* This is for languages whose "objects" are not well ordered (JSON is well ordered). + [ { a : ... } , { b : ... } ] -> { a : ..., b : ... } + */ + inline BSONObj transformOrderFromArrayFormat(BSONObj order) { + /* note: this is slow, but that is ok as order will have very few pieces */ + BSONObjBuilder b; + char p[2] = "0"; + + while ( 1 ) { + BSONObj j = order.getObjectField(p); + if ( j.isEmpty() ) + break; + BSONElement e = j.firstElement(); + uassert( 10102 , "bad order array", !e.eoo()); + uassert( 10103 , "bad order array [2]", e.isNumber()); + b.append(e); + (*p)++; + uassert( 10104 , "too many ordering elements", *p <= '9'); + } + + return b.obj(); + } + + /** + * this represents a total user query + * includes fields from the query message, both possible query levels + * parses everything up front + */ + class ParsedQuery { + public: + ParsedQuery( QueryMessage& qm ) + : _ns( qm.ns ) , _ntoskip( qm.ntoskip ) , _ntoreturn( qm.ntoreturn ) , _options( qm.queryOptions ){ + init( qm.query ); + initFields( qm.fields ); + } + ParsedQuery( const char* ns , int ntoskip , int ntoreturn , int queryoptions , const BSONObj& query , const BSONObj& fields ) + : _ns( ns ) , _ntoskip( ntoskip ) , _ntoreturn( ntoreturn ) , _options( queryoptions ){ + init( query ); + initFields( fields ); + } + + ~ParsedQuery(){} + + const char * ns() const { return _ns; } + + const BSONObj& getFilter() const { return _filter; } + FieldMatcher* getFields() const { return _fields.get(); } + shared_ptr getFieldPtr() const { return _fields; } + + int getSkip() const { return _ntoskip; } + int getNumToReturn() const { return _ntoreturn; } + bool wantMore() const { return _wantMore; } + int getOptions() const { return _options; } + bool hasOption( int x ) const { return x & _options; } + + + bool isExplain() const { return _explain; } + bool isSnapshot() const { return _snapshot; } + + const BSONObj& getMin() const { return _min; } + const BSONObj& getMax() const { return _max; } + const BSONObj& getOrder() const { return _order; } + const BSONElement& getHint() const { return _hint; } + + bool couldBeCommand() const { + /* we assume you are using findOne() for running a cmd... */ + return _ntoreturn == 1 && strstr( _ns , ".$cmd" ); + } + private: + void init( const BSONObj& q ){ + _reset(); + + if ( _ntoreturn < 0 ){ + /* _ntoreturn greater than zero is simply a hint on how many objects to send back per + "cursor batch". + A negative number indicates a hard limit. + */ + _wantMore = false; + _ntoreturn = -_ntoreturn; + } + + + BSONElement e = q["query"]; + if ( e.type() != Object ) + e = q["$query"]; + + if ( e.type() == Object ){ + _filter = e.embeddedObject(); + _initTop( q ); + } + else { + _filter = q; + } + } + + void _reset(){ + _wantMore = true; + _explain = false; + _snapshot = false; + } + + void _initTop( const BSONObj& top ){ + BSONObjIterator i( top ); + while ( i.more() ){ + BSONElement e = i.next(); + const char * name = e.fieldName(); + + if ( strcmp( "$orderby" , name ) == 0 || + strcmp( "orderby" , name ) == 0 ){ + if ( e.type() == Object ) + _order = e.embeddedObject(); + else if ( e.type() == Array ) + _order = transformOrderFromArrayFormat( _order ); + else + assert( 0 ); + } + else if ( strcmp( "$explain" , name ) == 0 ) + _explain = e.trueValue(); + else if ( strcmp( "$snapshot" , name ) == 0 ) + _snapshot = e.trueValue(); + else if ( strcmp( "$min" , name ) == 0 ) + _min = e.embeddedObject(); + else if ( strcmp( "$max" , name ) == 0 ) + _max = e.embeddedObject(); + else if ( strcmp( "$hint" , name ) == 0 ) + _hint = e; + + } + + if ( _snapshot ){ + uassert( 12001 , "E12001 can't sort with $snapshot", _order.isEmpty() ); + uassert( 12002 , "E12002 can't use hint with $snapshot", _hint.eoo() ); + } + + } + + void initFields( const BSONObj& fields ){ + if ( fields.isEmpty() ) + return; + _fields.reset( new FieldMatcher() ); + _fields->add( fields ); + if ( _fields->errmsg ) + uassert( 10053 , _fields->errmsg, false); // TODO: just have FieldMatcher throw now + } + + ParsedQuery( const ParsedQuery& other ){ + assert(0); + } + + const char* _ns; + int _ntoskip; + int _ntoreturn; + int _options; + + BSONObj _filter; + shared_ptr< FieldMatcher > _fields; + + bool _wantMore; + + bool _explain; + bool _snapshot; + BSONObj _min; + BSONObj _max; + BSONElement _hint; + BSONObj _order; + }; + + } // namespace mongo #include "clientcursor.h" diff --git a/dbtests/querytests.cpp b/dbtests/querytests.cpp index 670b13120c6..71a4e7e357c 100644 --- a/dbtests/querytests.cpp +++ b/dbtests/querytests.cpp @@ -906,6 +906,34 @@ namespace QueryTests { private: int _old; }; + + + namespace parsedtests { + class basic1 { + public: + void _test( const BSONObj& in ){ + ParsedQuery q( "a.b" , 5 , 6 , 9 , in , BSONObj() ); + ASSERT_EQUALS( BSON( "x" << 5 ) , q.getFilter() ); + } + void run(){ + _test( BSON( "x" << 5 ) ); + _test( BSON( "query" << BSON( "x" << 5 ) ) ); + _test( BSON( "$query" << BSON( "x" << 5 ) ) ); + + { + ParsedQuery q( "a.b" , 5 , 6 , 9 , BSON( "x" << 5 ) , BSONObj() ); + ASSERT_EQUALS( 6 , q.getNumToReturn() ); + ASSERT( q.wantMore() ); + } + { + ParsedQuery q( "a.b" , 5 , -6 , 9 , BSON( "x" << 5 ) , BSONObj() ); + ASSERT_EQUALS( 6 , q.getNumToReturn() ); + ASSERT( ! q.wantMore() ); + } + } + }; + }; + class All : public Suite { public: @@ -950,6 +978,8 @@ namespace QueryTests { add< HelperTest >(); add< HelperByIdTest >(); add< FindingStart >(); + + add< parsedtests::basic1 >(); } } myall;