diff --git a/SConstruct b/SConstruct index 35d19002433..ebcd05a0dbe 100644 --- a/SConstruct +++ b/SConstruct @@ -311,7 +311,7 @@ if has_option( "full" ): # ------ SOURCE FILE SETUP ----------- -commonFiles = Split( "pch.cpp buildinfo.cpp db/common.cpp db/indexkey.cpp db/jsobj.cpp bson/oid.cpp db/json.cpp db/lasterror.cpp db/nonce.cpp db/queryutil.cpp db/projection.cpp shell/mongo.cpp db/security_key.cpp" ) +commonFiles = Split( "pch.cpp buildinfo.cpp db/common.cpp db/indexkey.cpp db/jsobj.cpp bson/oid.cpp db/json.cpp db/lasterror.cpp db/nonce.cpp db/queryutil.cpp db/querypattern.cpp db/projection.cpp shell/mongo.cpp db/security_key.cpp" ) commonFiles += [ "util/background.cpp" , "util/sock.cpp" , "util/util.cpp" , "util/file_allocator.cpp" , "util/message.cpp" , "util/assert_util.cpp" , "util/log.cpp" , "util/httpclient.cpp" , "util/md5main.cpp" , "util/base64.cpp", "util/concurrency/vars.cpp", "util/concurrency/task.cpp", "util/debug_util.cpp", "util/concurrency/thread_pool.cpp", "util/password.cpp", "util/version.cpp", "util/signal_handlers.cpp", diff --git a/db/btree.h b/db/btree.h index 6d8171090bc..816f2698344 100644 --- a/db/btree.h +++ b/db/btree.h @@ -833,10 +833,14 @@ namespace mongo { }; #pragma pack() + class FieldRangeVector; + class FieldRangeVectorIterator; + class BtreeCursor : public Cursor { public: BtreeCursor( NamespaceDetails *_d, int _idxNo, const IndexDetails&, const BSONObj &startKey, const BSONObj &endKey, bool endKeyInclusive, int direction ); BtreeCursor( NamespaceDetails *_d, int _idxNo, const IndexDetails& _id, const shared_ptr< FieldRangeVector > &_bounds, int _direction ); + virtual ~BtreeCursor(); virtual bool ok() { return !bucket.isNull(); } virtual bool advance(); virtual void noteLocation(); // updates keyAtKeyOfs... @@ -885,25 +889,13 @@ namespace mongo { virtual DiskLoc refLoc() { return currLoc(); } virtual Record* _current() { return currLoc().rec(); } virtual BSONObj current() { return BSONObj(_current()); } - virtual string toString() { - string s = string("BtreeCursor ") + indexDetails.indexName(); - if ( _direction < 0 ) s += " reverse"; - if ( _bounds.get() && _bounds->size() > 1 ) s += " multi"; - return s; - } + virtual string toString(); BSONObj prettyKey( const BSONObj &key ) const { return key.replaceFieldNames( indexDetails.keyPattern() ).clientReadable(); } - virtual BSONObj prettyIndexBounds() const { - if ( !_independentFieldRanges ) { - return BSON( "start" << prettyKey( startKey ) << "end" << prettyKey( endKey ) ); - } - else { - return _bounds->obj(); - } - } + virtual BSONObj prettyIndexBounds() const; void forgetEndKey() { endKey = BSONObj(); } @@ -953,7 +945,7 @@ namespace mongo { BSONObj keyAtKeyOfs; // so we can tell if things moved around on us between the query and the getMore call DiskLoc locAtKeyOfs; const shared_ptr< FieldRangeVector > _bounds; - auto_ptr< FieldRangeVector::Iterator > _boundsIterator; + auto_ptr< FieldRangeVectorIterator > _boundsIterator; const IndexSpec& _spec; shared_ptr< CoveredIndexMatcher > _matcher; bool _independentFieldRanges; diff --git a/db/btreecursor.cpp b/db/btreecursor.cpp index 9cab95f83c6..e1423fca7d7 100644 --- a/db/btreecursor.cpp +++ b/db/btreecursor.cpp @@ -21,6 +21,7 @@ #include "pdfile.h" #include "jsobj.h" #include "curop-inl.h" +#include "queryutil.h" namespace mongo { @@ -55,7 +56,7 @@ namespace mongo { _ordering( Ordering::make( _order ) ), _direction( _direction ), _bounds( ( assert( _bounds.get() ), _bounds ) ), - _boundsIterator( new FieldRangeVector::Iterator( *_bounds ) ), + _boundsIterator( new FieldRangeVectorIterator( *_bounds ) ), _spec( _id.getSpec() ), _independentFieldRanges( true ), _nscanned( 0 ) { @@ -72,6 +73,9 @@ namespace mongo { dassert( _dups.size() == 0 ); } + /** Properly destroy forward declared class members. */ + BtreeCursor::~BtreeCursor() {} + void BtreeCursor::audit() { dassert( d->idxNo((IndexDetails&) indexDetails) == idxNo ); @@ -264,6 +268,22 @@ namespace mongo { skipUnusedKeys( false ); } + + string BtreeCursor::toString() { + string s = string("BtreeCursor ") + indexDetails.indexName(); + if ( _direction < 0 ) s += " reverse"; + if ( _bounds.get() && _bounds->size() > 1 ) s += " multi"; + return s; + } + + BSONObj BtreeCursor::prettyIndexBounds() const { + if ( !_independentFieldRanges ) { + return BSON( "start" << prettyKey( startKey ) << "end" << prettyKey( endKey ) ); + } + else { + return _bounds->obj(); + } + } /* ----------------------------------------------------------------------------- */ diff --git a/db/cap.cpp b/db/cap.cpp index 260b311eec8..3e4d6a22860 100644 --- a/db/cap.cpp +++ b/db/cap.cpp @@ -27,7 +27,6 @@ #include #include #include "query.h" -#include "queryutil.h" #include "json.h" /* diff --git a/db/namespace.h b/db/namespace.h index 64f5c4723b6..6a67e21da3f 100644 --- a/db/namespace.h +++ b/db/namespace.h @@ -20,7 +20,7 @@ #include "../pch.h" #include "jsobj.h" -#include "queryutil.h" +#include "querypattern.h" #include "diskloc.h" #include "../util/hashtab.h" #include "mongommf.h" diff --git a/db/oplog.cpp b/db/oplog.cpp index c73f6293e2d..193fb0481cf 100644 --- a/db/oplog.cpp +++ b/db/oplog.cpp @@ -24,6 +24,7 @@ #include "repl/rs.h" #include "stats/counters.h" #include "../util/file.h" +#include "queryoptimizer.h" namespace mongo { @@ -390,6 +391,121 @@ namespace mongo { // ------------------------------------- + FindingStartCursor::FindingStartCursor( const QueryPlan & qp ) : + _qp( qp ), + _findingStart( true ), + _findingStartMode(), + _findingStartTimer( 0 ) + { init(); } + + void FindingStartCursor::next() { + if ( !_findingStartCursor || !_findingStartCursor->ok() ) { + _findingStart = false; + _c = _qp.newCursor(); // on error, start from beginning + destroyClientCursor(); + return; + } + switch( _findingStartMode ) { + case Initial: { + if ( !_matcher->matches( _findingStartCursor->currKey(), _findingStartCursor->currLoc() ) ) { + _findingStart = false; // found first record out of query range, so scan normally + _c = _qp.newCursor( _findingStartCursor->currLoc() ); + destroyClientCursor(); + return; + } + _findingStartCursor->advance(); + RARELY { + if ( _findingStartTimer.seconds() >= __findingStartInitialTimeout ) { + createClientCursor( startLoc( _findingStartCursor->currLoc() ) ); + _findingStartMode = FindExtent; + return; + } + } + return; + } + case FindExtent: { + if ( !_matcher->matches( _findingStartCursor->currKey(), _findingStartCursor->currLoc() ) ) { + _findingStartMode = InExtent; + return; + } + DiskLoc prev = prevLoc( _findingStartCursor->currLoc() ); + if ( prev.isNull() ) { // hit beginning, so start scanning from here + createClientCursor(); + _findingStartMode = InExtent; + return; + } + // There might be a more efficient implementation than creating new cursor & client cursor each time, + // not worrying about that for now + createClientCursor( prev ); + return; + } + case InExtent: { + if ( _matcher->matches( _findingStartCursor->currKey(), _findingStartCursor->currLoc() ) ) { + _findingStart = false; // found first record in query range, so scan normally + _c = _qp.newCursor( _findingStartCursor->currLoc() ); + destroyClientCursor(); + return; + } + _findingStartCursor->advance(); + return; + } + default: { + massert( 14038, "invalid _findingStartMode", false ); + } + } + } + + DiskLoc FindingStartCursor::startLoc( const DiskLoc &rec ) { + Extent *e = rec.rec()->myExtent( rec ); + if ( !_qp.nsd()->capLooped() || ( e->myLoc != _qp.nsd()->capExtent ) ) + return e->firstRecord; + // Likely we are on the fresh side of capExtent, so return first fresh record. + // If we are on the stale side of capExtent, then the collection is small and it + // doesn't matter if we start the extent scan with capFirstNewRecord. + return _qp.nsd()->capFirstNewRecord; + } + + DiskLoc FindingStartCursor::prevLoc( const DiskLoc &rec ) { + Extent *e = rec.rec()->myExtent( rec ); + if ( _qp.nsd()->capLooped() ) { + if ( e->xprev.isNull() ) + e = _qp.nsd()->lastExtent.ext(); + else + e = e->xprev.ext(); + if ( e->myLoc != _qp.nsd()->capExtent ) + return e->firstRecord; + } + else { + if ( !e->xprev.isNull() ) { + e = e->xprev.ext(); + return e->firstRecord; + } + } + return DiskLoc(); // reached beginning of collection + } + + void FindingStartCursor::createClientCursor( const DiskLoc &startLoc ) { + shared_ptr c = _qp.newCursor( startLoc ); + _findingStartCursor.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns()) ); + } + + void FindingStartCursor::init() { + // Use a ClientCursor here so we can release db mutex while scanning + // oplog (can take quite a while with large oplogs). + shared_ptr c = _qp.newReverseCursor(); + _findingStartCursor.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns(), BSONObj()) ); + _findingStartTimer.reset(); + _findingStartMode = Initial; + BSONElement tsElt = _qp.originalQuery()[ "ts" ]; + massert( 13044, "no ts field in query", !tsElt.eoo() ); + BSONObjBuilder b; + b.append( tsElt ); + BSONObj tsQuery = b.obj(); + _matcher.reset(new CoveredIndexMatcher(tsQuery, _qp.indexKey())); + } + + // ------------------------------------- + struct TestOpTime { TestOpTime() { OpTime t; diff --git a/db/oplog.h b/db/oplog.h index 4b859ecc1e4..601b0a1a0e0 100644 --- a/db/oplog.h +++ b/db/oplog.h @@ -27,7 +27,6 @@ #include "db.h" #include "dbhelpers.h" #include "query.h" -#include "queryoptimizer.h" #include "../client/dbclient.h" #include "../util/optime.h" #include "../util/timer.h" @@ -64,72 +63,14 @@ namespace mongo { extern int __findingStartInitialTimeout; // configurable for testing + class QueryPlan; + class FindingStartCursor { public: - FindingStartCursor( const QueryPlan & qp ) : - _qp( qp ), - _findingStart( true ), - _findingStartMode(), - _findingStartTimer( 0 ) - { init(); } + FindingStartCursor( const QueryPlan & qp ); bool done() const { return !_findingStart; } shared_ptr cRelease() { return _c; } - void next() { - if ( !_findingStartCursor || !_findingStartCursor->ok() ) { - _findingStart = false; - _c = _qp.newCursor(); // on error, start from beginning - destroyClientCursor(); - return; - } - switch( _findingStartMode ) { - case Initial: { - if ( !_matcher->matches( _findingStartCursor->currKey(), _findingStartCursor->currLoc() ) ) { - _findingStart = false; // found first record out of query range, so scan normally - _c = _qp.newCursor( _findingStartCursor->currLoc() ); - destroyClientCursor(); - return; - } - _findingStartCursor->advance(); - RARELY { - if ( _findingStartTimer.seconds() >= __findingStartInitialTimeout ) { - createClientCursor( startLoc( _findingStartCursor->currLoc() ) ); - _findingStartMode = FindExtent; - return; - } - } - return; - } - case FindExtent: { - if ( !_matcher->matches( _findingStartCursor->currKey(), _findingStartCursor->currLoc() ) ) { - _findingStartMode = InExtent; - return; - } - DiskLoc prev = prevLoc( _findingStartCursor->currLoc() ); - if ( prev.isNull() ) { // hit beginning, so start scanning from here - createClientCursor(); - _findingStartMode = InExtent; - return; - } - // There might be a more efficient implementation than creating new cursor & client cursor each time, - // not worrying about that for now - createClientCursor( prev ); - return; - } - case InExtent: { - if ( _matcher->matches( _findingStartCursor->currKey(), _findingStartCursor->currLoc() ) ) { - _findingStart = false; // found first record in query range, so scan normally - _c = _qp.newCursor( _findingStartCursor->currLoc() ); - destroyClientCursor(); - return; - } - _findingStartCursor->advance(); - return; - } - default: { - massert( 14038, "invalid _findingStartMode", false ); - } - } - } + void next(); bool prepareToYield() { if ( _findingStartCursor ) { return _findingStartCursor->prepareToYield( _yieldData ); @@ -153,56 +94,15 @@ namespace mongo { ClientCursor::CleanupPointer _findingStartCursor; shared_ptr _c; ClientCursor::YieldData _yieldData; - DiskLoc startLoc( const DiskLoc &rec ) { - Extent *e = rec.rec()->myExtent( rec ); - if ( !_qp.nsd()->capLooped() || ( e->myLoc != _qp.nsd()->capExtent ) ) - return e->firstRecord; - // Likely we are on the fresh side of capExtent, so return first fresh record. - // If we are on the stale side of capExtent, then the collection is small and it - // doesn't matter if we start the extent scan with capFirstNewRecord. - return _qp.nsd()->capFirstNewRecord; - } + DiskLoc startLoc( const DiskLoc &rec ); // should never have an empty extent in the oplog, so don't worry about that case - DiskLoc prevLoc( const DiskLoc &rec ) { - Extent *e = rec.rec()->myExtent( rec ); - if ( _qp.nsd()->capLooped() ) { - if ( e->xprev.isNull() ) - e = _qp.nsd()->lastExtent.ext(); - else - e = e->xprev.ext(); - if ( e->myLoc != _qp.nsd()->capExtent ) - return e->firstRecord; - } - else { - if ( !e->xprev.isNull() ) { - e = e->xprev.ext(); - return e->firstRecord; - } - } - return DiskLoc(); // reached beginning of collection - } - void createClientCursor( const DiskLoc &startLoc = DiskLoc() ) { - shared_ptr c = _qp.newCursor( startLoc ); - _findingStartCursor.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns()) ); - } + DiskLoc prevLoc( const DiskLoc &rec ); + void createClientCursor( const DiskLoc &startLoc = DiskLoc() ); void destroyClientCursor() { _findingStartCursor.reset( 0 ); } - void init() { - // Use a ClientCursor here so we can release db mutex while scanning - // oplog (can take quite a while with large oplogs). - shared_ptr c = _qp.newReverseCursor(); - _findingStartCursor.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns(), BSONObj()) ); - _findingStartTimer.reset(); - _findingStartMode = Initial; - BSONElement tsElt = _qp.originalQuery()[ "ts" ]; - massert( 13044, "no ts field in query", !tsElt.eoo() ); - BSONObjBuilder b; - b.append( tsElt ); - BSONObj tsQuery = b.obj(); - _matcher.reset(new CoveredIndexMatcher(tsQuery, _qp.indexKey())); - } + void init(); }; void pretouchOperation(const BSONObj& op); diff --git a/db/querypattern.cpp b/db/querypattern.cpp new file mode 100644 index 00000000000..30e3e04d2ac --- /dev/null +++ b/db/querypattern.cpp @@ -0,0 +1,54 @@ +// @file querypattern.cpp - Query pattern matching for selecting similar plans given similar queries. + +/* Copyright 2011 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "querypattern.h" + +namespace mongo { + + /** for testing only - speed unimportant */ + bool QueryPattern::operator==( const QueryPattern &other ) const { + bool less = operator<( other ); + bool more = other.operator<( *this ); + assert( !( less && more ) ); + return !( less || more ); + } + + /** for testing only - speed unimportant */ + bool QueryPattern::operator!=( const QueryPattern &other ) const { + return !operator==( other ); + } + + void QueryPattern::setSort( const BSONObj sort ) { + _sort = normalizeSort( sort ); + } + + BSONObj QueryPattern::normalizeSort( const BSONObj &spec ) { + if ( spec.isEmpty() ) + return spec; + int direction = ( spec.firstElement().number() >= 0 ) ? 1 : -1; + BSONObjIterator i( spec ); + BSONObjBuilder b; + while( i.moreWithEOO() ) { + BSONElement e = i.next(); + if ( e.eoo() ) + break; + b.append( e.fieldName(), direction * ( ( e.number() >= 0 ) ? -1 : 1 ) ); + } + return b.obj(); + } + +} // namespace mongo \ No newline at end of file diff --git a/db/querypattern.h b/db/querypattern.h new file mode 100644 index 00000000000..12c24f24be2 --- /dev/null +++ b/db/querypattern.h @@ -0,0 +1,76 @@ +// @file querypattern.h - Query pattern matching for selecting similar plans given similar queries. + +/* Copyright 2011 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "jsobj.h" + +namespace mongo { + + /** + * Implements query pattern matching, used to determine if a query is + * similar to an earlier query and should use the same plan. + * + * Two queries will generate the same QueryPattern, and therefore match each + * other, if their fields have the same Types and they have the same sort + * spec. + */ + class QueryPattern { + public: + friend class FieldRangeSet; + enum Type { + Equality, + LowerBound, + UpperBound, + UpperAndLowerBound + }; + bool operator<( const QueryPattern &other ) const; + /** for testing only */ + bool operator==( const QueryPattern &other ) const; + /** for testing only */ + bool operator!=( const QueryPattern &other ) const; + private: + QueryPattern() {} + void setSort( const BSONObj sort ); + static BSONObj normalizeSort( const BSONObj &spec ); + map _fieldTypes; + BSONObj _sort; + }; + + inline bool QueryPattern::operator<( const QueryPattern &other ) const { + map::const_iterator i = _fieldTypes.begin(); + map::const_iterator j = other._fieldTypes.begin(); + while( i != _fieldTypes.end() ) { + if ( j == other._fieldTypes.end() ) + return false; + if ( i->first < j->first ) + return true; + else if ( i->first > j->first ) + return false; + if ( i->second < j->second ) + return true; + else if ( i->second > j->second ) + return false; + ++i; + ++j; + } + if ( j != other._fieldTypes.end() ) + return true; + return _sort.woCompare( other._sort ) < 0; + } + +} // namespace mongo \ No newline at end of file diff --git a/db/repl.h b/db/repl.h index 8c3f326f54a..a29ed298a44 100644 --- a/db/repl.h +++ b/db/repl.h @@ -31,7 +31,6 @@ #include "db.h" #include "dbhelpers.h" #include "query.h" -#include "queryoptimizer.h" #include "../client/dbclient.h" #include "../util/optime.h" #include "oplog.h"