xuemingqiang/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-49215 Differentiate internal idents used for resumable index build information from other internal idents

Browse Source
This commit is contained in:
Gregory Noma
2020-09-08 12:41:03 -04:00
committed by Evergreen Agent
parent 4ff797fa43
commit b68df4e87d
8 changed files with 94 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/mongo/db/catalog/multi_index_block.cpp
View File

@@ -872,7 +872,9 @@ boost::optional<ResumeIndexInfo> MultiIndexBlock::_abortWithoutCleanup(Operation
void MultiIndexBlock::_writeStateToDisk(OperationContext* opCtx) const {
auto obj = _constructStateObject();
auto rs = opCtx->getServiceContext()->getStorageEngine()->makeTemporaryRecordStore(opCtx);
auto rs = opCtx->getServiceContext()
->getStorageEngine()
->makeTemporaryRecordStoreForResumableIndexBuild(opCtx);
WriteUnitOfWork wuow(opCtx);

5
src/mongo/db/storage/durable_catalog.h
View File

@@ -127,6 +127,11 @@ public:
*/
virtual std::string newInternalIdent() = 0;
/**
* Generate an internal resumable index build ident name.
*/
virtual std::string newInternalResumableIndexBuildIdent() = 0;
/**
* On success, returns the RecordId which identifies the new record store in the durable catalog
* in addition to ownership of the new RecordStore.

15
src/mongo/db/storage/durable_catalog_impl.cpp
View File

@@ -65,6 +65,7 @@ const char kNamespaceFieldName[] = "ns";
const char kNonRepairableFeaturesFieldName[] = "nonRepairable";
const char kRepairableFeaturesFieldName[] = "repairable";
const char kInternalIdentPrefix[] = "internal-";
const char kResumableIndexBuildIdentStem[] = "resumable-index-build-";
void appendPositionsOfBitsSet(uint64_t value, StringBuilder* sb) {
invariant(sb);
@@ -427,8 +428,17 @@ bool DurableCatalogImpl::_hasEntryCollidingWithRand() const {
}
std::string DurableCatalogImpl::newInternalIdent() {
return _newInternalIdent("");
}
std::string DurableCatalogImpl::newInternalResumableIndexBuildIdent() {
return _newInternalIdent(kResumableIndexBuildIdentStem);
}
std::string DurableCatalogImpl::_newInternalIdent(StringData identStem) {
StringBuilder buf;
buf << kInternalIdentPrefix;
buf << identStem;
buf << _next.fetchAndAdd(1) << '-' << _rand;
return buf.str();
}
@@ -765,6 +775,11 @@ bool DurableCatalogImpl::isInternalIdent(StringData ident) const {
return ident.find(kInternalIdentPrefix) != std::string::npos;
}
bool DurableCatalogImpl::isResumableIndexBuildIdent(StringData ident) const {
invariant(isInternalIdent(ident), ident.toString());
return ident.find(kResumableIndexBuildIdentStem) != std::string::npos;
}
bool DurableCatalogImpl::isCollectionIdent(StringData ident) const {
// Internal idents prefixed "internal-" should not be considered collections, because
// they are not eligible for orphan recovery through repair.

5
src/mongo/db/storage/durable_catalog_impl.h
View File

@@ -92,6 +92,8 @@ public:
bool isInternalIdent(StringData ident) const;
bool isResumableIndexBuildIdent(StringData ident) const;
bool isCollectionIdent(StringData ident) const;
FeatureTracker* getFeatureTracker() const {
@@ -108,6 +110,7 @@ public:
std::string getFilesystemPathForDb(const std::string& dbName) const;
std::string newInternalIdent();
std::string newInternalResumableIndexBuildIdent();
StatusWith<std::pair<RecordId, std::unique_ptr<RecordStore>>> createCollection(
OperationContext* opCtx,
@@ -229,6 +232,8 @@ private:
*/
std::string _newUniqueIdent(NamespaceString nss, const char* kind);
std::string _newInternalIdent(StringData identStem);
// Helpers only used by constructor and init(). Don't call from elsewhere.
static std::string _newRand();
bool _hasEntryCollidingWithRand() const;

8
src/mongo/db/storage/storage_engine.h
View File

@@ -375,6 +375,14 @@ public:
virtual std::unique_ptr<TemporaryRecordStore> makeTemporaryRecordStore(
OperationContext* opCtx) = 0;
/**
* Creates a temporary RecordStore on the storage engine for a resumable index build. On
* startup after an unclean shutdown, the storage engine will drop any un-dropped temporary
* record stores.
*/
virtual std::unique_ptr<TemporaryRecordStore> makeTemporaryRecordStoreForResumableIndexBuild(
OperationContext* opCtx) = 0;
/**
* Creates a temporary RecordStore on the storage engine from an existing ident on disk. On
* startup after an unclean shutdown, the storage engine will drop any un-dropped temporary

82
src/mongo/db/storage/storage_engine_impl.cpp
View File

@@ -326,7 +326,7 @@ Status StorageEngineImpl::_recoverOrphanedCollection(OperationContext* opCtx,
return Status::OK();
}
bool StorageEngineImpl::_handleInternalIdents(
bool StorageEngineImpl::_handleInternalIdent(
OperationContext* opCtx,
const std::string& ident,
InternalIdentReconcilePolicy internalIdentReconcilePolicy,
@@ -345,14 +345,15 @@ bool StorageEngineImpl::_handleInternalIdents(
return true;
}
if (!_catalog->isResumableIndexBuildIdent(ident)) {
return false;
}
// When starting up after a clean shutdown and resumable index builds are supported, find the
// internal idents that contain the relevant information to resume each index build and recover
// the state.
auto rs = _engine->getRecordStore(opCtx, "", ident, CollectionOptions());
// Look at the contents to determine whether this ident will contain information for
// resuming an index build.
// TODO SERVER-49215: differentiate the internal idents without looking at the contents.
auto cursor = rs->getCursor(opCtx);
auto record = cursor->next();
if (record) {
@@ -360,36 +361,35 @@ bool StorageEngineImpl::_handleInternalIdents(
// Parse the documents here so that we can restart the build if the document doesn't
// contain all the necessary information to be able to resume building the index.
if (doc.hasField("phase")) {
ResumeIndexInfo resumeInfo;
try {
if (MONGO_unlikely(failToParseResumeIndexInfo.shouldFail())) {
uasserted(ErrorCodes::FailPointEnabled,
"failToParseResumeIndexInfo fail point is enabled");
}
resumeInfo = ResumeIndexInfo::parse(IDLParserErrorContext("ResumeIndexInfo"), doc);
} catch (const DBException& e) {
LOGV2(4916300, "Failed to parse resumable index info", "error"_attr = e.toStatus());
// Ignore the error so that we can restart the index build instead of resume it. We
// should drop the internal ident if we failed to parse.
internalIdentsToDrop->insert(ident);
return true;
ResumeIndexInfo resumeInfo;
try {
if (MONGO_unlikely(failToParseResumeIndexInfo.shouldFail())) {
uasserted(ErrorCodes::FailPointEnabled,
"failToParseResumeIndexInfo fail point is enabled");
}
reconcileResult->indexBuildsToResume.push_back(resumeInfo);
resumeInfo = ResumeIndexInfo::parse(IDLParserErrorContext("ResumeIndexInfo"), doc);
} catch (const DBException& e) {
LOGV2(4916300, "Failed to parse resumable index info", "error"_attr = e.toStatus());
// Once we have parsed the resume info, we can safely drop the internal ident.
// Ignore the error so that we can restart the index build instead of resume it. We
// should drop the internal ident if we failed to parse.
internalIdentsToDrop->insert(ident);
LOGV2(4916301,
"Found unfinished index build to resume",
"buildUUID"_attr = resumeInfo.getBuildUUID(),
"collectionUUID"_attr = resumeInfo.getCollectionUUID(),
"phase"_attr = IndexBuildPhase_serializer(resumeInfo.getPhase()));
return true;
}
reconcileResult->indexBuildsToResume.push_back(resumeInfo);
// Once we have parsed the resume info, we can safely drop the internal ident.
internalIdentsToDrop->insert(ident);
LOGV2(4916301,
"Found unfinished index build to resume",
"buildUUID"_attr = resumeInfo.getBuildUUID(),
"collectionUUID"_attr = resumeInfo.getCollectionUUID(),
"phase"_attr = IndexBuildPhase_serializer(resumeInfo.getPhase()));
return true;
}
return false;
@@ -448,12 +448,12 @@ StatusWith<StorageEngine::ReconcileResult> StorageEngineImpl::reconcileCatalogAn
continue;
}
if (_handleInternalIdents(opCtx,
it,
internalIdentReconcilePolicy,
&reconcileResult,
&internalIdentsToDrop,
&allInternalIdents)) {
if (_handleInternalIdent(opCtx,
it,
internalIdentReconcilePolicy,
&reconcileResult,
&internalIdentsToDrop,
&allInternalIdents)) {
continue;
}
@@ -864,10 +864,18 @@ std::unique_ptr<TemporaryRecordStore> StorageEngineImpl::makeTemporaryRecordStor
OperationContext* opCtx) {
std::unique_ptr<RecordStore> rs =
_engine->makeTemporaryRecordStore(opCtx, _catalog->newInternalIdent());
LOGV2_DEBUG(22258,
LOGV2_DEBUG(22258, 1, "Created temporary record store", "ident"_attr = rs->getIdent());
return std::make_unique<TemporaryKVRecordStore>(getEngine(), std::move(rs));
}
std::unique_ptr<TemporaryRecordStore>
StorageEngineImpl::makeTemporaryRecordStoreForResumableIndexBuild(OperationContext* opCtx) {
std::unique_ptr<RecordStore> rs =
_engine->makeTemporaryRecordStore(opCtx, _catalog->newInternalResumableIndexBuildIdent());
LOGV2_DEBUG(4921500,
1,
"created temporary record store: {rs_getIdent}",
"rs_getIdent"_attr = rs->getIdent());
"Created temporary record store for resumable index build",
"ident"_attr = rs->getIdent());
return std::make_unique<TemporaryKVRecordStore>(getEngine(), std::move(rs));
}

15
src/mongo/db/storage/storage_engine_impl.h
View File

@@ -113,6 +113,9 @@ public:
virtual std::unique_ptr<TemporaryRecordStore> makeTemporaryRecordStore(
OperationContext* opCtx) override;
virtual std::unique_ptr<TemporaryRecordStore> makeTemporaryRecordStoreForResumableIndexBuild(
OperationContext* opCtx) override;
virtual std::unique_ptr<TemporaryRecordStore> makeTemporaryRecordStoreFromExistingIdent(
OperationContext* opCtx, StringData ident) override;
@@ -386,12 +389,12 @@ private:
* Returns whether the given ident is an internal ident and if it should be dropped or used to
* resume an index build.
*/
bool _handleInternalIdents(OperationContext* opCtx,
const std::string& ident,
InternalIdentReconcilePolicy internalIdentReconcilePolicy,
ReconcileResult* reconcileResult,
std::set<std::string>* internalIdentsToDrop,
std::set<std::string>* allInternalIdents);
bool _handleInternalIdent(OperationContext* opCtx,
const std::string& ident,
InternalIdentReconcilePolicy internalIdentReconcilePolicy,
ReconcileResult* reconcileResult,
std::set<std::string>* internalIdentsToDrop,
std::set<std::string>* allInternalIdents);
class RemoveDBChange;

4
src/mongo/db/storage/storage_engine_mock.h
View File

@@ -93,6 +93,10 @@ public:
std::unique_ptr<TemporaryRecordStore> makeTemporaryRecordStore(OperationContext* opCtx) final {
return {};
}
std::unique_ptr<TemporaryRecordStore> makeTemporaryRecordStoreForResumableIndexBuild(
OperationContext* opCtx) final {
return {};
}
std::unique_ptr<TemporaryRecordStore> makeTemporaryRecordStoreFromExistingIdent(
OperationContext* opCtx, StringData ident) final {
return {};