diff --git a/db/db.cpp b/db/db.cpp
index 35e49076869..2ce37b92196 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -553,6 +553,14 @@ sendmore:
Client::initThread("initandlisten");
_diaglog.init();
+ if (cmdLine.dur)
+ dur::enableDurability();
+
+ getDur().startup();
+
+ if( cmdLine.durOptions & CmdLine::DurRecoverOnly )
+ return;
+
clearTmpCollections();
Module::initAll();
@@ -563,14 +571,6 @@ sendmore:
globalScriptEngine->setGetInterruptSpecCallback( jsGetInterruptSpecCallback );
}
- if (cmdLine.dur)
- dur::enableDurability();
-
- getDur().startup();
-
- if( cmdLine.durOptions & CmdLine::DurRecoverOnly )
- return;
-
repairDatabasesAndCheckVersion();
/* we didn't want to pre-open all fiels for the repair check above. for regular
diff --git a/db/db.vcxproj b/db/db.vcxproj
index da3496fac59..5a425ba852e 100644
--- a/db/db.vcxproj
+++ b/db/db.vcxproj
@@ -633,6 +633,7 @@
+
diff --git a/db/db.vcxproj.filters b/db/db.vcxproj.filters
index 6e2dc48dad4..e1a462a0ec7 100755
--- a/db/db.vcxproj.filters
+++ b/db/db.vcxproj.filters
@@ -843,6 +843,9 @@
db\geo
+
+ util\core
+
@@ -914,6 +917,9 @@
{d565a775-7a99-4860-b25f-441e1655b7c6}
+
+ {e504bcc8-efa8-451e-a040-d94339fa649e}
+
diff --git a/db/dur.cpp b/db/dur.cpp
index 7226459a51e..734b10610f4 100644
--- a/db/dur.cpp
+++ b/db/dur.cpp
@@ -194,31 +194,33 @@ namespace mongo {
/** we will build an output buffer ourself and then use O_DIRECT
we could be in read lock for this
- caller handles locking
- */
+ caller handles locking
+ */
static void PREPLOGBUFFER() {
assert( cmdLine.dur );
AlignedBuilder& bb = commitJob._ab;
bb.reset();
- unsigned lenOfs;
+ unsigned lenOfs; // we will need to backfill the length when prep is wrapping up
// JSectHeader
{
bb.appendStr("\nHH\n", false);
lenOfs = bb.skip(4);
}
- // ops other than basic writes
+ // ops other than basic writes (DurOp's)
{
for( vector< shared_ptr >::iterator i = commitJob.ops().begin(); i != commitJob.ops().end(); ++i ) {
(*i)->serialize(bb);
}
}
- // write intents
+ // write intents. note there is no particular order to these : if we have
+ // two writes to the same location during the group commit interval, it is likely
+ // (although not assured) that it is journalled here once.
{
scoped_lock lk(privateViews._mutex());
- string lastFilePath;
+ RelativePath lastFilePath;
for( vector::iterator i = commitJob.writes().begin(); i != commitJob.writes().end(); i++ ) {
size_t ofs;
MongoMMF *mmf = privateViews._find(i->p, ofs);
@@ -229,14 +231,18 @@ namespace mongo {
}
if( !mmf->willNeedRemap() ) {
- mmf->willNeedRemap() = true; // usually it will already be dirty so don't bother writing then
+ // tag this mmf as needed a remap of its private view later.
+ // usually it will already be dirty/already set, so we do the if above first
+ // to avoid possibility of cpu cache line contention
+ mmf->willNeedRemap() = true;
}
i->w_ptr = ((char*)mmf->view_write()) + ofs;
if( mmf->relativePath() != lastFilePath ) {
lastFilePath = mmf->relativePath();
+ //assert( !str::startsWith(lastFilePath, dbpath) ); // dbpath should be stripped this is a relative path
JDbContext c;
bb.appendStruct(c);
- bb.appendStr(lastFilePath);
+ bb.appendStr(lastFilePath.toString());
}
JEntry e;
e.len = i->len;
@@ -517,6 +523,7 @@ namespace mongo {
#endif
}
+ /** at startup, recover, and then start the journal threads */
void DurableImpl::startup() {
if( !cmdLine.dur )
return;
diff --git a/db/dur_journal.cpp b/db/dur_journal.cpp
index 0b162ef46be..037467fcc56 100644
--- a/db/dur_journal.cpp
+++ b/db/dur_journal.cpp
@@ -1,4 +1,4 @@
-// @file dur_journal.cpp
+// @file dur_journal.cpp writing to the writeahead logging journal
/**
* Copyright (C) 2010 10gen Inc.
@@ -76,11 +76,11 @@ namespace mongo {
static const unsigned long long DataLimit = 1 * 1024 * 1024 * 1024;
public:
string dir; // set by journalMakeDir() during initialization
- MVar &toUnlink;
+ MVar &toUnlink; // unlinks of old journal threads are via background thread
Journal() :
toUnlink(*(new MVar)), /* freeing MVar at program termination would be problematic */
- _lfMutex("lfMutex")
+ _lfMutex("JournalLfMutex")
{
_written = 0;
_nextFileNumber = 0;
@@ -112,7 +112,7 @@ namespace mongo {
mutex _lfMutex; // lock when using _lf
};
- static Journal j;
+ Journal j;
path Journal::getFilePathFor(int filenumber) const {
filesystem::path p(dir);
@@ -120,7 +120,9 @@ namespace mongo {
return p;
}
- /** never throws */
+ /** never throws
+ @return true if journal dir is not emptya
+ */
bool haveJournalFiles() {
try {
for ( boost::filesystem::directory_iterator i( getJournalDir() );
@@ -210,6 +212,7 @@ namespace mongo {
_open();
}
+ /** background removal of old journal files */
void unlinkThread() {
Client::initThread("unlink");
while( 1 ) {
@@ -297,5 +300,6 @@ namespace mongo {
}
/* todo
- test (and handle) disk full on journal append
+ test (and handle) disk full on journal append. best quick thing to do is to terminate.
+ if we roll back operations, there are nuances such as is ReplSetImpl::lastOpTimeWritten too new in ram then?
*/
diff --git a/db/dur_journalformat.h b/db/dur_journalformat.h
index 32338a10453..863bfb3e4d3 100644
--- a/db/dur_journalformat.h
+++ b/db/dur_journalformat.h
@@ -25,31 +25,37 @@ namespace mongo {
namespace dur {
#pragma pack(1)
- /** header for a journal/j._ file */
+ /** beginning header for a journal/j._ file
+ there is nothing important int this header at this time. except perhaps version #.
+ */
struct JHeader {
JHeader() { }
JHeader(string fname);
char magic[2]; // "j\n". j means journal, then a linefeed, fwiw if you were to run "less" on the file or something...
+ // x4142 is asci--readable if you look at the file with head/less -- thus the starting values were near
+ // that. simply incrementing the version # is safe on a fwd basis.
enum { CurrentVersion = 0x4142 };
unsigned short _version;
// these are just for diagnostic ease (make header more useful as plain text)
char n1; // '\n'
- char ts[20]; // offset 6
+ char ts[20]; // offset 6. ascii timestamp of file generation. for user reading, not used by code.
char n2; // '\n'
- char dbpath[128]; // offset 27
+ char dbpath[128]; // offset 27. path/filename of this file for human reading and diagnostics. not used by code.
char n3, n4; // '\n', '\n'
- char reserved3[8192 - 68 - 96 + 10 -4]; // 8KB total for the file header
- char txt2[2]; // "\n\n" offset 8190
+ char reserved3[8034]; // 8KB total for the file header
+ char txt2[2]; // "\n\n" offset 8190
bool versionOk() const { return _version == CurrentVersion; }
bool valid() const { return magic[0] == 'j' && txt2[1] == '\n'; }
};
- /** "Section" header. A section corresponds to a group commit. */
+ /** "Section" header. A section corresponds to a group commit.
+ len is length of the entire section including header and footer.
+ */
struct JSectHeader {
char magic[4]; // "\nhh\n"
unsigned len; // length in bytes of the whole section
@@ -65,12 +71,16 @@ namespace mongo {
OpCode_Min = 0xfffff000 // higher than max len: OpCode_Min + sizeof(JHeader) > 2^32
};
- unsigned len; // or opcode, see structs below
+ union {
+ unsigned len;
+ OpCodes opcode;
+ };
unsigned ofs; // offset in file
int fileNo;
// char data[] follows
};
+ /** group commit section footer. md5 is a key field. */
struct JSectFooter {
JSectFooter(const void* begin, int len) { // needs buffer to compute hash
sentinel = JEntry::OpCode_Footer;
@@ -90,14 +100,11 @@ namespace mongo {
bool checkHash(const void* begin, int len) const {
if (*(int*)hash == 0) return true; // TODO(mathias): remove this
-
// skip section header since size modified after hashing
(const char*&)begin += sizeof(JSectHeader);
len -= sizeof(JSectHeader);
-
md5digest current;
md5(begin, len, current);
-
return (memcmp(hash, current, sizeof(hash)) == 0);
}
diff --git a/db/dur_recover.cpp b/db/dur_recover.cpp
index c7e0b9aabdd..499229cb638 100644
--- a/db/dur_recover.cpp
+++ b/db/dur_recover.cpp
@@ -218,7 +218,16 @@ namespace mongo {
*/
fn = ss.str();
}
- p = f->map(fn.c_str());
+ path full(dbpath);
+ try {
+ // relative name -> full path name
+ full /= fn;
+ p = f->map(full.string().c_str());
+ }
+ catch(DBException&) {
+ log() << "recover error opening file " << full.string() << endl;
+ throw;
+ }
uassert(13534, str::stream() << "recovery error couldn't open " << fn, p);
if( cmdLine.durOptions & CmdLine::DurDumpJournal )
log() << " opened " << fn << ' ' << f->length()/1024.0/1024.0 << endl;
@@ -259,9 +268,7 @@ namespace mongo {
log() << ss.str() << endl;
}
if( apply ) {
- path filepath = dbpath;
- filepath /= fqe.dbName;
- void *p = ptr(filepath.string().c_str(), fqe.e.fileNo, fqe.e.ofs);
+ void *p = ptr(fqe.dbName, fqe.e.fileNo, fqe.e.ofs);
memcpy(p, fqe.srcData, fqe.e.len);
}
} else {
diff --git a/db/durop.cpp b/db/durop.cpp
index dc9f07e1ab8..8509ea20315 100644
--- a/db/durop.cpp
+++ b/db/durop.cpp
@@ -21,6 +21,7 @@
#include "../util/alignedbuilder.h"
#include "../util/mongoutils/str.h"
#include "../util/file.h"
+#include "mongommf.h"
#include "durop.h"
using namespace mongoutils;
@@ -46,7 +47,7 @@ namespace mongo {
op = shared_ptr( new DropDbOp(br) );
break;
default:
- massert(13546, str::stream() << "dur recover unrecognized opcode in journal " << opcode, false);
+ massert(13546, (str::stream() << "dur recover unrecognized opcode in journal " << opcode), false);
}
return op;
}
@@ -78,53 +79,67 @@ namespace mongo {
_deleteDataFiles(_db.c_str());
}
- FileCreatedOp::FileCreatedOp(BufReader& log) : DurOp(JEntry::OpCode_FileCreated) {
+ FileCreatedOp::FileCreatedOp(string f, unsigned long long l) :
+ DurOp(JEntry::OpCode_FileCreated)
+ {
+ _p = RelativePath::fromFullPath(f);
+ _len = l;
+ }
+
+ FileCreatedOp::FileCreatedOp(BufReader& log) : DurOp(JEntry::OpCode_FileCreated)
+ {
unsigned long long reserved;
log.read(reserved);
log.read(reserved);
log.read(_len); // size of file, not length of name
- log.readStr(_filename);
+ string s;
+ log.readStr(s);
+ _p._p = path(s);
}
void FileCreatedOp::_serialize(AlignedBuilder& ab) {
ab.appendNum((unsigned long long) 0); // reserved for future use
ab.appendNum((unsigned long long) 0); // reserved for future use
ab.appendNum(_len);
- /*string fn;
- if( str::startsWith(_filename, dbpath) )
- fn = str::after(_filename, dbpath);
- else {
- fn = _filename;
- log() << "warning logging creation of file " << _filename << " which is not below dbpath " << dbpath << endl;
- }*/
- ab.appendStr(_filename);
+ ab.appendStr(_p.toString());
}
string FileCreatedOp::toString() {
- return str::stream() << "FileCreatedOp " << _filename << ' ' << _len/1024.0/1024.0;
+ return str::stream() << "FileCreatedOp " << _p.toString() << ' ' << _len/1024.0/1024.0;
}
+ // if an operation deletes or creates a file (or moves etc.), it may need files closed.
bool FileCreatedOp::needFilesClosed() {
- return exists( _filename );
+ return exists( _p.asFullPath() );
}
void FileCreatedOp::replay() {
// i believe the code assumes new files are filled with zeros. thus we have to recreate the file,
// or rewrite at least, even if it were the right length. perhaps one day we should change that
// although easier to avoid defects if we assume it is zeros perhaps.
- if( exists( _filename ) ) {
+ string full = _p.asFullPath();
+ if( exists(full) ) {
try {
- remove(_filename);
+ remove(full);
}
catch(std::exception& e) {
log(1) << "recover info FileCreateOp::replay unlink " << e.what() << endl;
}
}
- log() << "recover create file " << _filename << ' ' << _len/1024.0/1024.0 << "MB" << endl;
+ log() << "recover create file " << full << ' ' << _len/1024.0/1024.0 << "MB" << endl;
+ if( MemoryMappedFile::exists(full) ) {
+ // first delete if exists.
+ try {
+ remove(full);
+ }
+ catch(...) {
+ log() << "warning could not delete file " << full << endl;
+ }
+ }
File f;
- f.open(_filename.c_str());
- massert(13547, str::stream() << "recover couldn't create file " << _filename, f.is_open());
+ f.open(full.c_str());
+ massert(13547, str::stream() << "recover couldn't create file " << full, f.is_open());
unsigned long long left = _len;
const unsigned blksz = 64 * 1024;
scoped_ptr v( new char[blksz] );
diff --git a/db/durop.h b/db/durop.h
index c204a82bb20..2000ba3a616 100644
--- a/db/durop.h
+++ b/db/durop.h
@@ -20,6 +20,7 @@
#include "dur_journalformat.h"
#include "bufreader.h"
+#include "../util/paths.h"
namespace mongo {
@@ -76,15 +77,15 @@ namespace mongo {
class FileCreatedOp : public DurOp {
public:
FileCreatedOp(BufReader& log);
- FileCreatedOp(string f, unsigned long long l) :
- DurOp(JEntry::OpCode_FileCreated), _filename(f), _len(l) { }
+ /** param f filename to create with path */
+ FileCreatedOp(string f, unsigned long long l);
virtual void replay();
virtual string toString();
virtual bool needFilesClosed();
protected:
virtual void _serialize(AlignedBuilder& ab);
private:
- string _filename;
+ RelativePath _p;
unsigned long long _len; // size of file, not length of name
};
diff --git a/db/mongommf.cpp b/db/mongommf.cpp
index 308b999d76c..3be78702e35 100644
--- a/db/mongommf.cpp
+++ b/db/mongommf.cpp
@@ -153,23 +153,15 @@ namespace mongo {
void MongoMMF::setPath(string f) {
string suffix;
- string fullpath;
- bool ok = str::rSplitOn(f, '.', fullpath, suffix);
+ string prefix;
+ bool ok = str::rSplitOn(f, '.', prefix, suffix);
uassert(13520, str::stream() << "MongoMMF only supports filenames in a certain format " << f, ok);
if( suffix == "ns" )
_fileSuffixNo = -1;
else
_fileSuffixNo = (int) str::toUnsigned(suffix);
- string relative = str::after(fullpath, dbpath);
- uassert(13600,
- str::stream() << "MongoMMF file path is not under the db path? " << fullpath << ' ' << dbpath,
- relative != fullpath);
-
- if( str::startsWith(relative, "/") || str::startsWith(relative, "\\") ) {
- relative.erase(0, 1);
- }
- _relativePath = relative;
+ _p = RelativePath::fromFullPath(prefix);
}
bool MongoMMF::open(string fname, bool sequentialHint) {
diff --git a/db/mongommf.h b/db/mongommf.h
index 5846ca7383a..e06faccaa97 100644
--- a/db/mongommf.h
+++ b/db/mongommf.h
@@ -18,7 +18,7 @@
#pragma once
#include "../util/mmap.h"
-//#include "../util/moveablebuffer.h"
+#include "../util/paths.h"
namespace mongo {
@@ -67,7 +67,7 @@ namespace mongo {
fileSuffixNo() is 3
if the suffix is "ns", fileSuffixNo -1
*/
- string relativePath() const { return _relativePath; }
+ RelativePath relativePath() const { return _p; }
int fileSuffixNo() const { return _fileSuffixNo; }
void* view_write() { return _view_write; }
@@ -87,7 +87,7 @@ namespace mongo {
void *_view_private;
void *_view_readonly; // for _DEBUG build
bool _willNeedRemap;
- string _relativePath; // e.g. "somepath/dbname"
+ RelativePath _p; // e.g. "somepath/dbname"
int _fileSuffixNo; // e.g. 3. -1="ns"
void setPath(string pathAndFileName);
diff --git a/util/mongoutils/str.h b/util/mongoutils/str.h
index 95a71931bb6..9ffb30834cc 100644
--- a/util/mongoutils/str.h
+++ b/util/mongoutils/str.h
@@ -97,6 +97,8 @@ namespace mongoutils {
/** @return true if s contains x */
inline bool contains(string s, string x) {
return strstr(s.c_str(), x.c_str()) != 0; }
+ inline bool contains(string s, char x) {
+ return strchr(s.c_str(), x) != 0; }
/** @return everything befor the character x, else entire string */
inline string before(const string& s, char x) {
@@ -147,7 +149,7 @@ namespace mongoutils {
R = string(p+1);
return true;
}
- /** split scanning reverse direction */
+ /** split scanning reverse direction. Splits ONCE ONLY. */
inline bool rSplitOn(const string &s, char c, string& L, string& R) {
const char *start = s.c_str();
const char *p = strrchr(start, c);
@@ -160,6 +162,7 @@ namespace mongoutils {
return true;
}
+ /** @return umber of occurrences of c in s */
inline unsigned count( const string& s , char c ){
unsigned n=0;
for ( unsigned i=0; i