From 5073a411035fb45ddc7ccc8a3e90a477d9710ecd Mon Sep 17 00:00:00 2001 From: Mathias Stearn Date: Thu, 25 Feb 2010 21:42:46 -0500 Subject: [PATCH] Convert prefix queries to ranges ~20%+ speedup SERVER-611 --- db/matcher.cpp | 29 ++++++++- db/queryutil.cpp | 151 ++++++++++++++++++++++++----------------------- db/queryutil.h | 11 +++- 3 files changed, 114 insertions(+), 77 deletions(-) diff --git a/db/matcher.cpp b/db/matcher.cpp index 10ce8c87864..97c322ee1a3 100644 --- a/db/matcher.cpp +++ b/db/matcher.cpp @@ -165,14 +165,37 @@ namespace mongo { void Matcher::addRegex( const BSONElement &e, const char *fieldName, bool isNot ) { + if ( fieldName == 0 ) + fieldName = e.fieldName(); + const char* regex = e.regex(); + const char* flags = e.regexFlags(); + + if (!isNot){ //TODO something smarter + bool purePrefix; + string prefix = simpleRegex(regex, flags, &purePrefix); + if (purePrefix){ + { + shared_ptr< BSONObjBuilder > b( new BSONObjBuilder() ); + _builders.push_back( b ); + *b << fieldName << prefix; + addBasic(b->done().firstElement(), BSONObj::GTE , isNot); + } + { + shared_ptr< BSONObjBuilder > b( new BSONObjBuilder() ); + _builders.push_back( b ); + *b << fieldName << simpleRegexEnd(prefix); + addBasic(b->done().firstElement(), BSONObj::LT , isNot); + } + return; + } + } + if ( nRegex >= 4 ) { out() << "ERROR: too many regexes in query" << endl; } else { - if ( fieldName == 0 ) - fieldName = e.fieldName(); RegexMatcher& rm = regexs[nRegex]; - rm.re = new pcrecpp::RE(e.regex(), flags2options(e.regexFlags())); + rm.re = new pcrecpp::RE(regex, flags2options(flags)); rm.fieldName = fieldName; rm.isNot = isNot; nRegex++; diff --git a/db/queryutil.cpp b/db/queryutil.cpp index 19c9bb0d2e0..131b9869210 100644 --- a/db/queryutil.cpp +++ b/db/queryutil.cpp @@ -24,85 +24,95 @@ #include "../util/unittest.h" namespace mongo { - namespace { - /** returns a string that when used as a matcher, would match a super set of regex() - returns "" for complex regular expressions - used to optimize queries in some simple regex cases that start with '^' - */ - inline string simpleRegexHelper(const char* regex, const char* flags){ - string r = ""; + /** returns a string that when used as a matcher, would match a super set of regex() + returns "" for complex regular expressions + used to optimize queries in some simple regex cases that start with '^' - bool extended = false; - while (*flags){ - switch (*(flags++)){ - case 'm': // multiline - continue; - case 'x': // extended - extended = true; - break; - default: - return r; // cant use index - } - } + if purePrefix != NULL, sets it to whether the regex can be converted to a range query + */ + string simpleRegex(const char* regex, const char* flags, bool* purePrefix){ + string r = ""; - if ( *(regex++) != '^' ) - return r; + if (purePrefix) *purePrefix = false; - stringstream ss; - - while(*regex){ - char c = *(regex++); - if ( c == '*' || c == '?' ){ - // These are the only two symbols that make the last char optional - r = ss.str(); - r = r.substr( 0 , r.size() - 1 ); - return r; //breaking here fails with /^a?/ - } else if (c == '\\'){ - // slash followed by non-alphanumeric represents the following char - c = *(regex++); - if ((c >= 'A' && c <= 'Z') || - (c >= 'a' && c <= 'z') || - (c >= '0' && c <= '0') || - (c == '\0')) - { - r = ss.str(); - break; - } else { - ss << c; - } - } else if (strchr("^$.[|()+{", c)){ - // list of "metacharacters" from man pcrepattern - r = ss.str(); - break; - } else if (extended && c == '#'){ - // comment - r = ss.str(); - break; - } else if (extended && isspace(c)){ + bool extended = false; + while (*flags){ + switch (*(flags++)){ + case 'm': // multiline continue; + case 'x': // extended + extended = true; + break; + default: + return r; // cant use index + } + } + + if ( *(regex++) != '^' ) + return r; + + stringstream ss; + + while(*regex){ + char c = *(regex++); + if ( c == '*' || c == '?' ){ + // These are the only two symbols that make the last char optional + r = ss.str(); + r = r.substr( 0 , r.size() - 1 ); + return r; //breaking here fails with /^a?/ + } else if (c == '\\'){ + // slash followed by non-alphanumeric represents the following char + c = *(regex++); + if ((c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '0') || + (c == '\0')) + { + r = ss.str(); + break; } else { - // self-matching char ss << c; } - } - - if ( r.size() == 0 && *regex == 0 ) + } else if (strchr("^$.[|()+{", c)){ + // list of "metacharacters" from man pcrepattern r = ss.str(); - - return r; - } - inline string simpleRegex(const BSONElement& e){ - switch(e.type()){ - case RegEx: - return simpleRegexHelper(e.regex(), e.regexFlags()); - case Object:{ - BSONObj o = e.embeddedObject(); - return simpleRegexHelper(o["$regex"].valuestrsafe(), o["$options"].valuestrsafe()); - } - default: assert(false); return ""; //return squashes compiler warning + break; + } else if (extended && c == '#'){ + // comment + r = ss.str(); + break; + } else if (extended && isspace(c)){ + continue; + } else { + // self-matching char + ss << c; } } + + if ( r.empty() && *regex == 0 ){ + r = ss.str(); + if (purePrefix) *purePrefix = !r.empty(); + } + + return r; + } + inline string simpleRegex(const BSONElement& e){ + switch(e.type()){ + case RegEx: + return simpleRegex(e.regex(), e.regexFlags()); + case Object:{ + BSONObj o = e.embeddedObject(); + return simpleRegex(o["$regex"].valuestrsafe(), o["$options"].valuestrsafe()); + } + default: assert(false); return ""; //return squashes compiler warning + } } + + string simpleRegexEnd( string regex ) { + ++regex[ regex.length() - 1 ]; + return regex; + } + FieldRange::FieldRange( const BSONElement &e, bool isNot, bool optimize ) { // NOTE with $not, we could potentially form a complementary set of intervals. @@ -328,11 +338,6 @@ namespace mongo { return *this; } - string FieldRange::simpleRegexEnd( string regex ) { - ++regex[ regex.length() - 1 ]; - return regex; - } - BSONObj FieldRange::addObj( const BSONObj &o ) { objData_.push_back( o ); return o; diff --git a/db/queryutil.h b/db/queryutil.h index a819e5a4003..9ee1793f3af 100644 --- a/db/queryutil.h +++ b/db/queryutil.h @@ -73,7 +73,6 @@ namespace mongo { private: BSONObj addObj( const BSONObj &o ); - string simpleRegexEnd( string regex ); vector< FieldInterval > intervals_; vector< BSONObj > objData_; string _special; @@ -209,5 +208,15 @@ namespace mongo { BSONObj _source; }; + /** returns a string that when used as a matcher, would match a super set of regex() + returns "" for complex regular expressions + used to optimize queries in some simple regex cases that start with '^' + + if purePrefix != NULL, sets it to whether the regex can be converted to a range query + */ + string simpleRegex(const char* regex, const char* flags, bool* purePrefix=NULL); + + /** returns the upper bound of a query that matches prefix */ + string simpleRegexEnd( string prefix ); } // namespace mongo