From 62fbef60e708eaaf02de89be7af18edea4e5677e Mon Sep 17 00:00:00 2001 From: dwight Date: Wed, 15 Dec 2010 02:07:45 -0500 Subject: [PATCH] str stuff --- db/db.cpp | 16 ++++++------- db/db.vcxproj | 1 + db/db.vcxproj.filters | 6 +++++ db/dur.cpp | 23 ++++++++++++------- db/dur_journal.cpp | 16 ++++++++----- db/dur_journalformat.h | 27 +++++++++++++--------- db/dur_recover.cpp | 15 +++++++++---- db/durop.cpp | 51 +++++++++++++++++++++++++++--------------- db/durop.h | 7 +++--- db/mongommf.cpp | 14 +++--------- db/mongommf.h | 6 ++--- util/mongoutils/str.h | 16 ++++++++++++- 12 files changed, 126 insertions(+), 72 deletions(-) diff --git a/db/db.cpp b/db/db.cpp index 35e49076869..2ce37b92196 100644 --- a/db/db.cpp +++ b/db/db.cpp @@ -553,6 +553,14 @@ sendmore: Client::initThread("initandlisten"); _diaglog.init(); + if (cmdLine.dur) + dur::enableDurability(); + + getDur().startup(); + + if( cmdLine.durOptions & CmdLine::DurRecoverOnly ) + return; + clearTmpCollections(); Module::initAll(); @@ -563,14 +571,6 @@ sendmore: globalScriptEngine->setGetInterruptSpecCallback( jsGetInterruptSpecCallback ); } - if (cmdLine.dur) - dur::enableDurability(); - - getDur().startup(); - - if( cmdLine.durOptions & CmdLine::DurRecoverOnly ) - return; - repairDatabasesAndCheckVersion(); /* we didn't want to pre-open all fiels for the repair check above. for regular diff --git a/db/db.vcxproj b/db/db.vcxproj index da3496fac59..5a425ba852e 100644 --- a/db/db.vcxproj +++ b/db/db.vcxproj @@ -633,6 +633,7 @@ + diff --git a/db/db.vcxproj.filters b/db/db.vcxproj.filters index 6e2dc48dad4..e1a462a0ec7 100755 --- a/db/db.vcxproj.filters +++ b/db/db.vcxproj.filters @@ -843,6 +843,9 @@ db\geo + + util\core + @@ -914,6 +917,9 @@ {d565a775-7a99-4860-b25f-441e1655b7c6} + + {e504bcc8-efa8-451e-a040-d94339fa649e} + diff --git a/db/dur.cpp b/db/dur.cpp index 7226459a51e..734b10610f4 100644 --- a/db/dur.cpp +++ b/db/dur.cpp @@ -194,31 +194,33 @@ namespace mongo { /** we will build an output buffer ourself and then use O_DIRECT we could be in read lock for this - caller handles locking - */ + caller handles locking + */ static void PREPLOGBUFFER() { assert( cmdLine.dur ); AlignedBuilder& bb = commitJob._ab; bb.reset(); - unsigned lenOfs; + unsigned lenOfs; // we will need to backfill the length when prep is wrapping up // JSectHeader { bb.appendStr("\nHH\n", false); lenOfs = bb.skip(4); } - // ops other than basic writes + // ops other than basic writes (DurOp's) { for( vector< shared_ptr >::iterator i = commitJob.ops().begin(); i != commitJob.ops().end(); ++i ) { (*i)->serialize(bb); } } - // write intents + // write intents. note there is no particular order to these : if we have + // two writes to the same location during the group commit interval, it is likely + // (although not assured) that it is journalled here once. { scoped_lock lk(privateViews._mutex()); - string lastFilePath; + RelativePath lastFilePath; for( vector::iterator i = commitJob.writes().begin(); i != commitJob.writes().end(); i++ ) { size_t ofs; MongoMMF *mmf = privateViews._find(i->p, ofs); @@ -229,14 +231,18 @@ namespace mongo { } if( !mmf->willNeedRemap() ) { - mmf->willNeedRemap() = true; // usually it will already be dirty so don't bother writing then + // tag this mmf as needed a remap of its private view later. + // usually it will already be dirty/already set, so we do the if above first + // to avoid possibility of cpu cache line contention + mmf->willNeedRemap() = true; } i->w_ptr = ((char*)mmf->view_write()) + ofs; if( mmf->relativePath() != lastFilePath ) { lastFilePath = mmf->relativePath(); + //assert( !str::startsWith(lastFilePath, dbpath) ); // dbpath should be stripped this is a relative path JDbContext c; bb.appendStruct(c); - bb.appendStr(lastFilePath); + bb.appendStr(lastFilePath.toString()); } JEntry e; e.len = i->len; @@ -517,6 +523,7 @@ namespace mongo { #endif } + /** at startup, recover, and then start the journal threads */ void DurableImpl::startup() { if( !cmdLine.dur ) return; diff --git a/db/dur_journal.cpp b/db/dur_journal.cpp index 0b162ef46be..037467fcc56 100644 --- a/db/dur_journal.cpp +++ b/db/dur_journal.cpp @@ -1,4 +1,4 @@ -// @file dur_journal.cpp +// @file dur_journal.cpp writing to the writeahead logging journal /** * Copyright (C) 2010 10gen Inc. @@ -76,11 +76,11 @@ namespace mongo { static const unsigned long long DataLimit = 1 * 1024 * 1024 * 1024; public: string dir; // set by journalMakeDir() during initialization - MVar &toUnlink; + MVar &toUnlink; // unlinks of old journal threads are via background thread Journal() : toUnlink(*(new MVar)), /* freeing MVar at program termination would be problematic */ - _lfMutex("lfMutex") + _lfMutex("JournalLfMutex") { _written = 0; _nextFileNumber = 0; @@ -112,7 +112,7 @@ namespace mongo { mutex _lfMutex; // lock when using _lf }; - static Journal j; + Journal j; path Journal::getFilePathFor(int filenumber) const { filesystem::path p(dir); @@ -120,7 +120,9 @@ namespace mongo { return p; } - /** never throws */ + /** never throws + @return true if journal dir is not emptya + */ bool haveJournalFiles() { try { for ( boost::filesystem::directory_iterator i( getJournalDir() ); @@ -210,6 +212,7 @@ namespace mongo { _open(); } + /** background removal of old journal files */ void unlinkThread() { Client::initThread("unlink"); while( 1 ) { @@ -297,5 +300,6 @@ namespace mongo { } /* todo - test (and handle) disk full on journal append + test (and handle) disk full on journal append. best quick thing to do is to terminate. + if we roll back operations, there are nuances such as is ReplSetImpl::lastOpTimeWritten too new in ram then? */ diff --git a/db/dur_journalformat.h b/db/dur_journalformat.h index 32338a10453..863bfb3e4d3 100644 --- a/db/dur_journalformat.h +++ b/db/dur_journalformat.h @@ -25,31 +25,37 @@ namespace mongo { namespace dur { #pragma pack(1) - /** header for a journal/j._ file */ + /** beginning header for a journal/j._ file + there is nothing important int this header at this time. except perhaps version #. + */ struct JHeader { JHeader() { } JHeader(string fname); char magic[2]; // "j\n". j means journal, then a linefeed, fwiw if you were to run "less" on the file or something... + // x4142 is asci--readable if you look at the file with head/less -- thus the starting values were near + // that. simply incrementing the version # is safe on a fwd basis. enum { CurrentVersion = 0x4142 }; unsigned short _version; // these are just for diagnostic ease (make header more useful as plain text) char n1; // '\n' - char ts[20]; // offset 6 + char ts[20]; // offset 6. ascii timestamp of file generation. for user reading, not used by code. char n2; // '\n' - char dbpath[128]; // offset 27 + char dbpath[128]; // offset 27. path/filename of this file for human reading and diagnostics. not used by code. char n3, n4; // '\n', '\n' - char reserved3[8192 - 68 - 96 + 10 -4]; // 8KB total for the file header - char txt2[2]; // "\n\n" offset 8190 + char reserved3[8034]; // 8KB total for the file header + char txt2[2]; // "\n\n" offset 8190 bool versionOk() const { return _version == CurrentVersion; } bool valid() const { return magic[0] == 'j' && txt2[1] == '\n'; } }; - /** "Section" header. A section corresponds to a group commit. */ + /** "Section" header. A section corresponds to a group commit. + len is length of the entire section including header and footer. + */ struct JSectHeader { char magic[4]; // "\nhh\n" unsigned len; // length in bytes of the whole section @@ -65,12 +71,16 @@ namespace mongo { OpCode_Min = 0xfffff000 // higher than max len: OpCode_Min + sizeof(JHeader) > 2^32 }; - unsigned len; // or opcode, see structs below + union { + unsigned len; + OpCodes opcode; + }; unsigned ofs; // offset in file int fileNo; // char data[] follows }; + /** group commit section footer. md5 is a key field. */ struct JSectFooter { JSectFooter(const void* begin, int len) { // needs buffer to compute hash sentinel = JEntry::OpCode_Footer; @@ -90,14 +100,11 @@ namespace mongo { bool checkHash(const void* begin, int len) const { if (*(int*)hash == 0) return true; // TODO(mathias): remove this - // skip section header since size modified after hashing (const char*&)begin += sizeof(JSectHeader); len -= sizeof(JSectHeader); - md5digest current; md5(begin, len, current); - return (memcmp(hash, current, sizeof(hash)) == 0); } diff --git a/db/dur_recover.cpp b/db/dur_recover.cpp index c7e0b9aabdd..499229cb638 100644 --- a/db/dur_recover.cpp +++ b/db/dur_recover.cpp @@ -218,7 +218,16 @@ namespace mongo { */ fn = ss.str(); } - p = f->map(fn.c_str()); + path full(dbpath); + try { + // relative name -> full path name + full /= fn; + p = f->map(full.string().c_str()); + } + catch(DBException&) { + log() << "recover error opening file " << full.string() << endl; + throw; + } uassert(13534, str::stream() << "recovery error couldn't open " << fn, p); if( cmdLine.durOptions & CmdLine::DurDumpJournal ) log() << " opened " << fn << ' ' << f->length()/1024.0/1024.0 << endl; @@ -259,9 +268,7 @@ namespace mongo { log() << ss.str() << endl; } if( apply ) { - path filepath = dbpath; - filepath /= fqe.dbName; - void *p = ptr(filepath.string().c_str(), fqe.e.fileNo, fqe.e.ofs); + void *p = ptr(fqe.dbName, fqe.e.fileNo, fqe.e.ofs); memcpy(p, fqe.srcData, fqe.e.len); } } else { diff --git a/db/durop.cpp b/db/durop.cpp index dc9f07e1ab8..8509ea20315 100644 --- a/db/durop.cpp +++ b/db/durop.cpp @@ -21,6 +21,7 @@ #include "../util/alignedbuilder.h" #include "../util/mongoutils/str.h" #include "../util/file.h" +#include "mongommf.h" #include "durop.h" using namespace mongoutils; @@ -46,7 +47,7 @@ namespace mongo { op = shared_ptr( new DropDbOp(br) ); break; default: - massert(13546, str::stream() << "dur recover unrecognized opcode in journal " << opcode, false); + massert(13546, (str::stream() << "dur recover unrecognized opcode in journal " << opcode), false); } return op; } @@ -78,53 +79,67 @@ namespace mongo { _deleteDataFiles(_db.c_str()); } - FileCreatedOp::FileCreatedOp(BufReader& log) : DurOp(JEntry::OpCode_FileCreated) { + FileCreatedOp::FileCreatedOp(string f, unsigned long long l) : + DurOp(JEntry::OpCode_FileCreated) + { + _p = RelativePath::fromFullPath(f); + _len = l; + } + + FileCreatedOp::FileCreatedOp(BufReader& log) : DurOp(JEntry::OpCode_FileCreated) + { unsigned long long reserved; log.read(reserved); log.read(reserved); log.read(_len); // size of file, not length of name - log.readStr(_filename); + string s; + log.readStr(s); + _p._p = path(s); } void FileCreatedOp::_serialize(AlignedBuilder& ab) { ab.appendNum((unsigned long long) 0); // reserved for future use ab.appendNum((unsigned long long) 0); // reserved for future use ab.appendNum(_len); - /*string fn; - if( str::startsWith(_filename, dbpath) ) - fn = str::after(_filename, dbpath); - else { - fn = _filename; - log() << "warning logging creation of file " << _filename << " which is not below dbpath " << dbpath << endl; - }*/ - ab.appendStr(_filename); + ab.appendStr(_p.toString()); } string FileCreatedOp::toString() { - return str::stream() << "FileCreatedOp " << _filename << ' ' << _len/1024.0/1024.0; + return str::stream() << "FileCreatedOp " << _p.toString() << ' ' << _len/1024.0/1024.0; } + // if an operation deletes or creates a file (or moves etc.), it may need files closed. bool FileCreatedOp::needFilesClosed() { - return exists( _filename ); + return exists( _p.asFullPath() ); } void FileCreatedOp::replay() { // i believe the code assumes new files are filled with zeros. thus we have to recreate the file, // or rewrite at least, even if it were the right length. perhaps one day we should change that // although easier to avoid defects if we assume it is zeros perhaps. - if( exists( _filename ) ) { + string full = _p.asFullPath(); + if( exists(full) ) { try { - remove(_filename); + remove(full); } catch(std::exception& e) { log(1) << "recover info FileCreateOp::replay unlink " << e.what() << endl; } } - log() << "recover create file " << _filename << ' ' << _len/1024.0/1024.0 << "MB" << endl; + log() << "recover create file " << full << ' ' << _len/1024.0/1024.0 << "MB" << endl; + if( MemoryMappedFile::exists(full) ) { + // first delete if exists. + try { + remove(full); + } + catch(...) { + log() << "warning could not delete file " << full << endl; + } + } File f; - f.open(_filename.c_str()); - massert(13547, str::stream() << "recover couldn't create file " << _filename, f.is_open()); + f.open(full.c_str()); + massert(13547, str::stream() << "recover couldn't create file " << full, f.is_open()); unsigned long long left = _len; const unsigned blksz = 64 * 1024; scoped_ptr v( new char[blksz] ); diff --git a/db/durop.h b/db/durop.h index c204a82bb20..2000ba3a616 100644 --- a/db/durop.h +++ b/db/durop.h @@ -20,6 +20,7 @@ #include "dur_journalformat.h" #include "bufreader.h" +#include "../util/paths.h" namespace mongo { @@ -76,15 +77,15 @@ namespace mongo { class FileCreatedOp : public DurOp { public: FileCreatedOp(BufReader& log); - FileCreatedOp(string f, unsigned long long l) : - DurOp(JEntry::OpCode_FileCreated), _filename(f), _len(l) { } + /** param f filename to create with path */ + FileCreatedOp(string f, unsigned long long l); virtual void replay(); virtual string toString(); virtual bool needFilesClosed(); protected: virtual void _serialize(AlignedBuilder& ab); private: - string _filename; + RelativePath _p; unsigned long long _len; // size of file, not length of name }; diff --git a/db/mongommf.cpp b/db/mongommf.cpp index 308b999d76c..3be78702e35 100644 --- a/db/mongommf.cpp +++ b/db/mongommf.cpp @@ -153,23 +153,15 @@ namespace mongo { void MongoMMF::setPath(string f) { string suffix; - string fullpath; - bool ok = str::rSplitOn(f, '.', fullpath, suffix); + string prefix; + bool ok = str::rSplitOn(f, '.', prefix, suffix); uassert(13520, str::stream() << "MongoMMF only supports filenames in a certain format " << f, ok); if( suffix == "ns" ) _fileSuffixNo = -1; else _fileSuffixNo = (int) str::toUnsigned(suffix); - string relative = str::after(fullpath, dbpath); - uassert(13600, - str::stream() << "MongoMMF file path is not under the db path? " << fullpath << ' ' << dbpath, - relative != fullpath); - - if( str::startsWith(relative, "/") || str::startsWith(relative, "\\") ) { - relative.erase(0, 1); - } - _relativePath = relative; + _p = RelativePath::fromFullPath(prefix); } bool MongoMMF::open(string fname, bool sequentialHint) { diff --git a/db/mongommf.h b/db/mongommf.h index 5846ca7383a..e06faccaa97 100644 --- a/db/mongommf.h +++ b/db/mongommf.h @@ -18,7 +18,7 @@ #pragma once #include "../util/mmap.h" -//#include "../util/moveablebuffer.h" +#include "../util/paths.h" namespace mongo { @@ -67,7 +67,7 @@ namespace mongo { fileSuffixNo() is 3 if the suffix is "ns", fileSuffixNo -1 */ - string relativePath() const { return _relativePath; } + RelativePath relativePath() const { return _p; } int fileSuffixNo() const { return _fileSuffixNo; } void* view_write() { return _view_write; } @@ -87,7 +87,7 @@ namespace mongo { void *_view_private; void *_view_readonly; // for _DEBUG build bool _willNeedRemap; - string _relativePath; // e.g. "somepath/dbname" + RelativePath _p; // e.g. "somepath/dbname" int _fileSuffixNo; // e.g. 3. -1="ns" void setPath(string pathAndFileName); diff --git a/util/mongoutils/str.h b/util/mongoutils/str.h index 95a71931bb6..9ffb30834cc 100644 --- a/util/mongoutils/str.h +++ b/util/mongoutils/str.h @@ -97,6 +97,8 @@ namespace mongoutils { /** @return true if s contains x */ inline bool contains(string s, string x) { return strstr(s.c_str(), x.c_str()) != 0; } + inline bool contains(string s, char x) { + return strchr(s.c_str(), x) != 0; } /** @return everything befor the character x, else entire string */ inline string before(const string& s, char x) { @@ -147,7 +149,7 @@ namespace mongoutils { R = string(p+1); return true; } - /** split scanning reverse direction */ + /** split scanning reverse direction. Splits ONCE ONLY. */ inline bool rSplitOn(const string &s, char c, string& L, string& R) { const char *start = s.c_str(); const char *p = strrchr(start, c); @@ -160,6 +162,7 @@ namespace mongoutils { return true; } + /** @return umber of occurrences of c in s */ inline unsigned count( const string& s , char c ){ unsigned n=0; for ( unsigned i=0; i