diff --git a/db/instance.cpp b/db/instance.cpp index fb055801bea..470155ec9d9 100644 --- a/db/instance.cpp +++ b/db/instance.cpp @@ -283,7 +283,7 @@ namespace mongo { receivedQuery(c , dbresponse, m ); } else if ( op == dbGetMore ) { - DEV log = true; + //DEV log = true; if ( ! receivedGetMore(dbresponse, m, currentOp) ) log = true; } @@ -347,7 +347,7 @@ namespace mongo { int ms = currentOp.totalTimeMillis(); log = log || (logLevel >= 2 && ++ctr % 512 == 0); - DEV log = true; + //DEV log = true; if ( log || ms > logThreshold ) { if( logLevel < 3 && op == dbGetMore && strstr(ns, ".oplog.") && ms < 3000 && !log ) { /* it's normal for getMore on the oplog to be slow because of use of awaitdata flag. */ diff --git a/db/repl/consensus.cpp b/db/repl/consensus.cpp index f502d87d756..97adbc5bb82 100644 --- a/db/repl/consensus.cpp +++ b/db/repl/consensus.cpp @@ -21,15 +21,15 @@ namespace mongo { - class CmdReplSetElect : public Command { - public: + class ReplSetCommand : public Command { + protected: + ReplSetCommand(const char * s) : Command(s) { } virtual bool slaveOk() const { return true; } virtual bool adminOnly() const { return true; } virtual bool logTheOp() { return false; } virtual LockType locktype() const { return NONE; } virtual void help( stringstream &help ) const { help << "internal"; } - CmdReplSetElect() : Command("replSetElect") { } - virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + bool check(string& errmsg, BSONObjBuilder& result) { if( !replSet ) { errmsg = "not running with --replSet"; return false; @@ -39,6 +39,29 @@ namespace mongo { errmsg = ReplSet::startupStatusMsg.empty() ? "replset unknown error 2" : ReplSet::startupStatusMsg; return false; } + return true; + } + }; + + class CmdReplSetFresh : public ReplSetCommand { + public: + CmdReplSetFresh() : ReplSetCommand("replSetFresh") { } + private: + virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + if( !check(errmsg, result) ) + return false; + errmsg = "not done"; + return false; + } + } cmdReplSetFresh; + + class CmdReplSetElect : public ReplSetCommand { + public: + CmdReplSetElect() : ReplSetCommand("replSetElect") { } + private: + virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) { + if( !check(errmsg, result) ) + return false; theReplSet->elect.electCmdReceived(cmdObj, result); return true; } @@ -81,15 +104,16 @@ namespace mongo { /* todo: threading **************** !!!!!!!!!!!!!!!! */ void ReplSet::Consensus::electCmdReceived(BSONObj cmd, BSONObjBuilder& b) { - string name = cmd["name"].String(); + log() << "replSet TEMP ELECT " << cmd.toString() << rsLog; + string set = cmd["set"].String(); unsigned whoid = cmd["whoid"].Int(); int cfgver = cmd["cfgver"].Int(); OID round = cmd["round"].OID(); int myver = rs.config().version; int vote = 0; - if( name != rs.name() ) { - log() << "replSet error received an elect request for '" << name << "' but our set name is '" << rs.name() << "'" << rsLog; + if( set != rs.name() ) { + log() << "replSet error received an elect request for '" << set << "' but our set name is '" << rs.name() << "'" << rsLog; } else if( myver < cfgver ) { @@ -101,8 +125,10 @@ namespace mongo { vote = -10000; } else { - try { + try { vote = yea(whoid); + rs.relinquish(); + log() << "replSet info voting yea" << rsLog; } catch(VoteException&) { log() << "replSet voting no already voted for another" << rsLog; @@ -113,11 +139,43 @@ namespace mongo { b.append("round", round); } + void ReplSet::getTargets(list& L) { + for( Member *m = head(); m; m=m->next() ) + if( m->hbinfo().up() ) + L.push_back( Target(m->fullName()) ); + } + + bool ReplSet::Consensus::weAreFreshest() { + BSONObj cmd = BSON( + "replSetFresh" << 1 << + "set" << rs.name() << + "who" << rs._self->fullName() << + "cfgver" << rs._cfg->version ); + list L; + rs.getTargets(L); + multiCommand(cmd, L); + int nok = 0; + for( list::iterator i = L.begin(); i != L.end(); i++ ) { + if( i->ok ) { + nok++; + if( i->result["fresher"].trueValue() ) + return false; + } + } + log() << "replSet temp we are freshest, nok:" << nok << rsLog; + return true; + } + void ReplSet::Consensus::_electSelf() { - log() << "replSet info electSelf" << rsLog; + if( !weAreFreshest() ) { + log() << "replSet info not going to elect self, we are not freshest" << rsLog; + return; + } + time_t start = time(0); ReplSet::Member& me = *rs._self; int tally = yea( me.id() ); + log() << "replSet info electSelf" << rsLog; BSONObj electCmd = BSON( "replSetElect" << 1 << @@ -129,14 +187,12 @@ namespace mongo { ); list L; - for( Member *m = rs.head(); m; m=m->next() ) - if( m->hbinfo().up() ) - L.push_back( Target(m->fullName()) ); - + rs.getTargets(L); multiCommand(electCmd, L); for( list::iterator i = L.begin(); i != L.end(); i++ ) { - cout << "TEMP elect res: " << i->result.toString() << endl; + log() << "replSet TEMP elect res: " << i->result.toString() << rsLog; + Target& t = *i; if( i->ok ) { int v = i->result["vote"].Int(); tally += v; @@ -148,12 +204,11 @@ namespace mongo { log() << "replSet too much time passed during election, ignoring result" << rsLog; } /* succeeded. */ - rs._myState = PRIMARY; - log() << "replSet elected self as primary" << rsLog; + rs.assumePrimary(); return; } else { - log() << "replSet couldn't elect self, majority not achieved tally:" << tally << rsLog; + log() << "replSet couldn't elect self, only received " << tally << " votes" << rsLog; } } diff --git a/db/repl/heartbeat.cpp b/db/repl/heartbeat.cpp index 141271cf7dd..01560776664 100644 --- a/db/repl/heartbeat.cpp +++ b/db/repl/heartbeat.cpp @@ -135,7 +135,11 @@ namespace mongo { } m = mem; theReplSet->mgr->send( boost::bind(&ReplSet::msgUpdateHBInfo, theReplSet, mem) ); - if( mem.changed(old) ) { + + static time_t last = 0; + time_t now = time(0); + if( mem.changed(old) || now-last>4 ) { + last = now; theReplSet->mgr->send( boost::bind(&ReplSet::Manager::msgCheckNewState, theReplSet->mgr) ); } } diff --git a/db/repl/manager.cpp b/db/repl/manager.cpp index 05c822551f6..8ad8b32083a 100644 --- a/db/repl/manager.cpp +++ b/db/repl/manager.cpp @@ -29,7 +29,7 @@ namespace mongo { /* check members OTHER THAN US to see if they think they are primary */ const ReplSet::Member * ReplSet::Manager::findOtherPrimary() { - Member *m = _rs->head(); + Member *m = rs->head(); Member *p = 0; while( m ) { if( m->state() == PRIMARY ) { @@ -38,83 +38,64 @@ namespace mongo { } m = m->next(); } + if( p ) + noteARemoteIsPrimary(p); return p; } - ReplSet::Manager::Manager(ReplSet *rs) : task::Port("ReplSet::Manager"), _rs(rs), _primary(NOPRIMARY) + ReplSet::Manager::Manager(ReplSet *_rs) : task::Port("ReplSet::Manager"), rs(_rs), _primary(NOPRIMARY) { } void ReplSet::Manager::noteARemoteIsPrimary(const Member *m) { - _rs->_self->lhb() = "finish code #1"; - log() << "replSet finish notearemoteisprimary " << m->fullName() << rsLog; + if( !rs->primary() ) + rs->_currentPrimary = m; + rs->_self->lhb() = ""; } /** called as the health threads get new results */ void ReplSet::Manager::msgCheckNewState() { - { - const Member *p = _rs->currentPrimary(); - const Member *p2 = findOtherPrimary(); - try { p2 = findOtherPrimary(); } - catch(string s) { - /* two other nodes think they are primary (asynchronously polled) -- wait for things to settle down. */ - log() << "replSet warning DIAG TODO " << s << rsLog; - return; - } - - if( p == p2 && p ) return; - - if( p2 ) { - /* someone else thinks they are primary. */ - if( p == p2 ) // already match - return; - if( p == 0 ) - noteARemoteIsPrimary(p2); return; - if( p != _rs->_self ) - noteARemoteIsPrimary(p2); return; - /* we thought we were primary, yet now someone else thinks they are. */ - if( !_rs->elect.aMajoritySeemsToBeUp() ) - noteARemoteIsPrimary(p2); return; - /* ignore for now, keep thinking we are master */ - return; - } - - if( p ) { - /* we are already primary, and nothing significant out there has changed. */ - /* todo: if !aMajoritySeemsToBeUp, relinquish */ - assert( p == _rs->_self ); - return; - } - - /* no one seems to be primary. shall we try to elect ourself? */ - if( !_rs->elect.aMajoritySeemsToBeUp() ) { - _rs->_self->lhb() = "can't see a majority, won't elect self"; - return; - } - - _rs->_self->lhb() = ""; + const Member *p = rs->currentPrimary(); + const Member *p2; + try { p2 = findOtherPrimary(); } + catch(string s) { + /* two other nodes think they are primary (asynchronously polled) -- wait for things to settle down. */ + log() << "replSet warning DIAG TODO 2primary" << s << rsLog; + return; } - _rs->elect.electSelf(); + + if( p == p2 && p ) return; + + if( p2 ) { + /* someone else thinks they are primary. */ + if( p == p2 ) // already match + return; + if( p == 0 ) + noteARemoteIsPrimary(p2); return; + if( p != rs->_self ) + noteARemoteIsPrimary(p2); return; + /* we thought we were primary, yet now someone else thinks they are. */ + if( !rs->elect.aMajoritySeemsToBeUp() ) + noteARemoteIsPrimary(p2); return; + /* ignore for now, keep thinking we are master */ + return; + } + + if( p ) { + /* we are already primary, and nothing significant out there has changed. */ + /* todo: if !aMajoritySeemsToBeUp, relinquish */ + assert( p == rs->_self ); + return; + } + + /* no one seems to be primary. shall we try to elect ourself? */ + if( !rs->elect.aMajoritySeemsToBeUp() ) { + rs->_self->lhb() = "can't see a majority, won't consider electing self"; + return; + } + + rs->_self->lhb() = ""; + rs->elect.electSelf(); } -/* bool ReplSet::Manager::got(const any& msg) { - if( msg.type() == typeid(Messages) ) { - assert( CheckNewState == any_cast(msg) ); - checkNewState(); - } - else if( msg.type() == typeid(BSONObj) ) { - log() << "replSet todo finish code in replset manager: call receivedNewConfig" << rsLog; - BSONObj o = any_cast(msg); - log() << "replSet " << o.toString() << rsLog; - } - else if( msg.type() == typeid(HeartbeatInfo) ) { - const HeartbeatInfo& h = any_cast(msg); - _rs->updateHBInfo(h); - } - else { - assert(false); - } - return true; - }*/ - } diff --git a/db/repl/multicmd.h b/db/repl/multicmd.h index 91726f5b29c..a0a878eca81 100644 --- a/db/repl/multicmd.h +++ b/db/repl/multicmd.h @@ -46,7 +46,7 @@ namespace mongo { void run() { try { ScopedConn c(d.toHost); - c->runCommand("admin", cmd, d.result); + d.ok = c->runCommand("admin", cmd, d.result); } catch(DBException&) { DEV log() << "dev caught dbexception on multiCommand " << d.toHost << rsLog; diff --git a/db/repl/replset.cpp b/db/repl/replset.cpp index d35e336abc4..7e4f6d1775b 100644 --- a/db/repl/replset.cpp +++ b/db/repl/replset.cpp @@ -25,6 +25,21 @@ namespace mongo { bool replSet = false; ReplSet *theReplSet = 0; + void ReplSet::assumePrimary() { + _myState = PRIMARY; + _currentPrimary = _self; + log() << "replSet self is now primary" << rsLog; + } + + void ReplSet::relinquish() { + if( state() == PRIMARY ) { + _myState = RECOVERING; + log() << "replSet info relinquished primary state" << rsLog; + } + else if( state() == STARTUP2 ) + _myState = RECOVERING; + } + void ReplSet::msgUpdateHBInfo(HeartbeatInfo h) { for( Member *m = _members.head(); m; m=m->next() ) { if( m->id() == h.id() ) { diff --git a/db/repl/replset.h b/db/repl/replset.h index 93f7d948e6d..83811f4378d 100644 --- a/db/repl/replset.h +++ b/db/repl/replset.h @@ -28,9 +28,9 @@ namespace mongo { + struct Target; extern bool replSet; // true if using repl sets extern class ReplSet *theReplSet; // null until initialized - extern Tee *rsLog; /* information about the entire repl set, such as the various servers in the set, and their state */ @@ -52,6 +52,9 @@ namespace mongo { MemberState state() const { return _myState; } string name() const { return _name; } /* @return replica set's logical name */ + void relinquish(); + void assumePrimary(); + /* cfgString format is replsetname/host1,host2:port,... where :port is optional. @@ -89,6 +92,7 @@ namespace mongo { Atomic ly; unsigned yea(unsigned memberId); // throws VoteException void _electSelf(); + bool weAreFreshest(); public: Consensus(ReplSet *t) : rs(*t) { } int totalVotes() const; @@ -98,38 +102,40 @@ namespace mongo { } elect; public: - struct Member : public List1::Base { + class Member : public List1::Base { + public: Member(HostAndPort h, unsigned ord, const ReplSetConfig::MemberCfg *c); + string fullName() const { return h().toString(); } const ReplSetConfig::MemberCfg& config() const { return *_config; } - void summarizeAsHtml(stringstream& s) const; const HeartbeatInfo& hbinfo() const { return _hbinfo; } string lhb() { return _hbinfo.lastHeartbeatMsg; } - MemberState state() const { return _state; } + MemberState state() const { return _hbinfo.hbstate; } const HostAndPort& h() const { return _h; } unsigned id() const { return _hbinfo.id(); } + + void summarizeAsHtml(stringstream& s) const; friend class ReplSet; private: const ReplSetConfig::MemberCfg *_config; /* todo: when this changes??? */ HostAndPort _h; - MemberState _state; HeartbeatInfo _hbinfo; }; list memberHostnames() const; const Member* currentPrimary() const { return _currentPrimary; } + bool primary() const { return _myState == PRIMARY; } const ReplSetConfig::MemberCfg& myConfig() const { return _self->config(); } void msgUpdateHBInfo(HeartbeatInfo); private: const Member *_currentPrimary; - Member *_self; - /* all members of the set EXCEPT self. */ - List1 _members; + Member *_self; + List1 _members; /* all members of the set EXCEPT self. */ public: class Manager : public task::Port { bool got(const any&); - ReplSet *_rs; + ReplSet *rs; int _primary; const Member* findOtherPrimary(); void noteARemoteIsPrimary(const Member *); @@ -142,6 +148,7 @@ namespace mongo { private: Member* head() const { return _members.head(); } + void getTargets(list&); static string stateAsStr(MemberState state); static string stateAsHtml(MemberState state); void startThreads(); diff --git a/util/mongoutils/html.h b/util/mongoutils/html.h index 44c8ce14235..fd98452cced 100644 --- a/util/mongoutils/html.h +++ b/util/mongoutils/html.h @@ -100,7 +100,7 @@ namespace mongoutils { inline string yellow(string contentHtml, bool color=true) { if( !color ) return contentHtml; stringstream ss; - ss << "" << contentHtml << ""; + ss << "" << contentHtml << ""; return ss.str(); } inline string green(string contentHtml, bool color=true) {