xuemingqiang/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-99745 Make hashed shard key index optional upon sharding a collection (#47588)

Browse Source 
GitOrigin-RevId: 8967eaf6bcd42371e027382666c86fd3e580d5a4
This commit is contained in:
Cheahuychou Mao
2026-02-06 11:36:07 -05:00
committed by MongoDB Bot
parent 56d6b10cf5
commit 5293e0a60f
9 changed files with 413 additions and 3488 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3487
.github/CODEOWNERS vendored
View File

File diff suppressed because it is too large Load Diff

19
jstests/core_sharding/ddl/shard_collection_basic.js
View File

@@ -6,6 +6,9 @@
* ]
*/
import {isStableFCVSuite} from "jstests/libs/feature_compatibility_version.js";
import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js";
let kDbName = db.getName();
db.dropDatabase();
@@ -144,6 +147,22 @@ assert.commandFailedWithCode(
6373200,
);
if (
isStableFCVSuite() &&
FeatureFlagUtil.isPresentAndEnabled(db, "featureFlagHashedShardKeyIndexOptionalUponShardingCollection")
) {
jsTestLog("Cannot specify both implicitlyCreateIndex and skipHashedShardKeyIndexCreation.");
assert.commandFailedWithCode(
db.adminCommand({
shardCollection: kDbName + ".foo",
key: {x: "hashed"},
implicitlyCreateIndex: false,
skipHashedShardKeyIndexCreation: true,
}),
ErrorCodes.InvalidOptions,
);
}
//
// Test shardCollection's idempotency
//

3
jstests/sharding/OWNERS.yml
View File

@@ -160,3 +160,6 @@ filters:
- "set_fcv*.js":
approvers:
- 10gen/server-catalog-and-routing-routing-and-topology
- "hashed_shard_key_index_optional_shard_collection.js":
approvers:
- 10gen/server-cluster-scalability

336
jstests/sharding/hashed_shard_key_index_optional_shard_collection.js Normal file
View File

@@ -0,0 +1,336 @@
/*
* Test running shardCollection commands against a collection without a shard key index with the
* "skipHashedShardKeyIndexCreation" option and check the following:
* 1. If the shard key is not hashed or featureFlagHashedShardKeyIndexOptionalUponShardingCollection
* is not enabled, the command fails with InvalidOptions.
* 2. If the shardCollection command is run with "skipHashedShardKeyIndexCreation" false:
* - If the collection is non-existent or empty, then the command succeeds and the shard key
* index is created.
* - Otherwise, the command fails with InvalidOptions.
* 3. If the shardCollection command is run with "skipHashedShardKeyIndexCreation" true:
* - If the shard key is hashed and featureFlagHashedShardKeyIndexOptionalUponShardingCollection
* is enabled, then the command succeeds and the shard key index doesn't get created.
* - Otherwise, the command fails with InvalidOptions.
*
* For each resulting sharded collection, check that:
* - find commands that filter by shard key equality uses IXSCAN if the collection has a shard key
* index (because the shardCollection command was run with "skipHashedShardKeyIndexCreation"
* false) or a non-shard key compatible index.
* - moveChunk commands against the given hashed sharded collection fails with IndexNotFound if
* the shardCollection command was run with "skipHashedShardKeyIndexCreation" true.
*
* Consider the shard key {x: "hashed"}, the shard key indexes for this shard key include
* {x: "hashed"}, {x: "hashed", y: 1} and the non-shard key but compatible indexes for this shard
* key include {x: 1}, {x: 1, y: 1} and {x: 1, y: "hashed"}.
*
* @tags: [
* requires_fcv_83,
* featureFlagHashedShardKeyIndexOptionalUponShardingCollection
* ]
*/
import {getWinningPlanFromExplain} from "jstests/libs/query/analyze_plan.js";
import {ShardingTest} from "jstests/libs/shardingtest.js";
import {AnalyzeShardKeyUtil} from "jstests/sharding/analyze_shard_key/libs/analyze_shard_key_util.js";
const dbName = "testDb";
let shardNames = [];
function makeDocument(shardKey, indexKey, val) {
let doc = {};
for (let fieldName in shardKey) {
AnalyzeShardKeyUtil.setDottedField(doc, fieldName, val);
}
if (indexKey) {
for (let fieldName in indexKey) {
AnalyzeShardKeyUtil.setDottedField(doc, fieldName, val);
}
}
return doc;
}
function checkIfIndexExists(st, dbName, collName, indexKey) {
const indexes = st.s.getDB(dbName).getCollection(collName).getIndexes();
jsTest.log("Checking indexes " + tojson({dbName, collName, indexes}));
return indexes.some((index) => bsonWoCompare(index.key, indexKey) == 0);
}
/*
* Checks that find commands that filter by shard key equality against the given sharded
* collection uses IXSCAN if the collection has a shard key index (because the shardCollection
* command was run with "skipHashedShardKeyIndexCreation" false) or a non-shard key compatible
* index.
*/
function testFindCommand(st, dbName, collName, doc, testCase) {
const coll = st.s.getDB(dbName).getCollection(collName);
const shardKeyValue = AnalyzeShardKeyUtil.extractShardKeyValueFromDocument(doc, testCase.shardKey);
const explain = coll.find(shardKeyValue).explain();
const winningPlan = getWinningPlanFromExplain(explain);
const isIdHashedShardKey = bsonWoCompare(testCase.shardKey, {_id: "hashed"}) == 0;
if (isIdHashedShardKey) {
const isClusteredColl = AnalyzeShardKeyUtil.isClusterCollection(st.s, dbName, collName);
if (isClusteredColl) {
assert.eq(winningPlan.stage, "EXPRESS_CLUSTERED_IXSCAN", winningPlan);
assert(!winningPlan.keyPattern);
} else {
assert.eq(winningPlan.stage, "EXPRESS_IXSCAN", winningPlan);
assert.eq(winningPlan.keyPattern, "{ _id: 1 }", winningPlan);
}
} else if (!testCase.skipHashedShardKeyIndexCreation || testCase.containsCompatibleIndex) {
assert.eq(winningPlan.stage, "FETCH", winningPlan);
assert.eq(winningPlan.inputStage.stage, "IXSCAN", winningPlan);
if (testCase.skipHashedShardKeyIndexCreation) {
assert(bsonWoCompare(winningPlan.inputStage.keyPattern, testCase.indexKey) == 0, winningPlan);
} else {
assert(
bsonWoCompare(winningPlan.inputStage.keyPattern, testCase.shardKey) == 0 ||
bsonWoCompare(winningPlan.inputStage.keyPattern, testCase.indexKey) == 0,
winningPlan,
);
}
}
}
/*
* Checks that moveChunk commands against the given sharded collection fails with IndexNotFound if
* the shardCollection command was run with "skipHashedShardKeyIndexCreation" true.
*/
function testMoveChunkCommand(st, dbName, collName, doc, testCase) {
const ns = dbName + "." + collName;
const shardKeyValue = AnalyzeShardKeyUtil.extractShardKeyValueFromDocument(doc, testCase.shardKey);
let failed = false;
for (let shardName of shardNames) {
const res = st.s.adminCommand({moveChunk: ns, find: shardKeyValue, to: shardName});
if (testCase.skipHashedShardKeyIndexCreation) {
assert.commandWorkedOrFailedWithCode(res, ErrorCodes.IndexNotFound);
if (res.code == ErrorCodes.IndexNotFound) {
failed = true;
}
} else {
assert.commandWorked(res);
}
}
if (testCase.skipHashedShardKeyIndexCreation) {
assert(failed);
}
}
function testNonExistentCollection(st, testCase) {
jsTest.log("Test sharding a non-existent collection " + tojson(testCase));
const collName1 = "testColl1";
const ns1 = dbName + "." + collName1;
const coll = st.s.getCollection(ns1);
const res = st.s.adminCommand({
shardCollection: ns1,
key: testCase.shardKey,
skipHashedShardKeyIndexCreation: testCase.skipHashedShardKeyIndexCreation,
});
if (testCase.expectErrorCode) {
assert.commandFailedWithCode(res, testCase.expectErrorCode);
const expectedIndexExists = false;
assert.eq(checkIfIndexExists(st, dbName, collName1, testCase.shardKey), expectedIndexExists, {testCase});
} else {
assert.commandWorked(res);
const expectedIndexExists = !testCase.skipHashedShardKeyIndexCreation;
assert.eq(checkIfIndexExists(st, dbName, collName1, testCase.shardKey), expectedIndexExists, {testCase});
const doc = makeDocument(testCase.shardKey, testCase.indexKey, 1);
assert.commandWorked(coll.insert(doc));
testFindCommand(st, dbName, collName1, doc, testCase);
testMoveChunkCommand(st, dbName, collName1, doc, testCase, expectedIndexExists);
}
assert(coll.drop());
}
function testExistentCollection(st, testCase) {
jsTest.log("Test sharding an existent collection " + tojson(testCase));
const collName2 = "testColl2";
const ns2 = dbName + "." + collName2;
const db = st.s.getDB(dbName);
const coll = db.getCollection(collName2);
let doc;
if (testCase.indexKey) {
assert.commandWorked(coll.createIndex(testCase.indexKey));
} else {
assert.commandWorked(db.createCollection(collName2));
}
if (!testCase.isEmptyCollection) {
doc = makeDocument(testCase.shardKey, testCase.indexKey, 1);
assert.commandWorked(coll.insert(doc));
}
const res = st.s.adminCommand({
shardCollection: ns2,
key: testCase.shardKey,
skipHashedShardKeyIndexCreation: testCase.skipHashedShardKeyIndexCreation,
});
if (testCase.expectErrorCode) {
assert.commandFailedWithCode(res, testCase.expectErrorCode);
const expectedIndexExists = false;
assert.eq(checkIfIndexExists(st, dbName, collName2, testCase.shardKey), expectedIndexExists, {testCase});
} else {
assert.commandWorked(res);
const expectedIndexExists = !testCase.skipHashedShardKeyIndexCreation;
assert.eq(checkIfIndexExists(st, dbName, collName2, testCase.shardKey), expectedIndexExists, {testCase});
if (testCase.isEmptyCollection) {
doc = makeDocument(testCase.shardKey, testCase.indexKey, 2);
assert.commandWorked(coll.insert(doc));
}
testFindCommand(st, dbName, collName2, doc, testCase);
testMoveChunkCommand(st, dbName, collName2, doc, testCase, expectedIndexExists);
}
assert(coll.drop());
}
// Test cases of non-shard key indexes. shardCollection commands with
// 'skipHashedShardKeyIndexCreation' true should only succeed when the shard key is hashed and
// featureFlagHashedShardKeyIndexOptionalUponShardingCollection is true.
const shardKeyTestCases = [
{
shardKey: {_id: "hashed"},
indexKeyTestCases: [
{key: {a: "hashed"}, isCompatible: false},
{key: {a: 1}, isCompatible: false},
],
},
{
shardKey: {a: 1},
indexKeyTestCases: [{key: {a: "hashed"}, isCompatible: true}],
},
{
shardKey: {a: "hashed"},
indexKeyTestCases: [
{key: {a: 1}, isCompatible: true},
{key: {a: 1, b: 1}, isCompatible: true},
{key: {a: 1, b: "hashed"}, isCompatible: true},
{key: {b: 1, a: "hashed"}, isCompatible: false},
{key: {b: 1, a: 1}, isCompatible: false},
],
},
{
shardKey: {a: 1, b: 1},
indexKeyTestCases: [
{key: {a: "hashed", b: 1}, isCompatible: true},
{key: {a: 1, b: "hashed"}, isCompatible: true},
],
},
{
shardKey: {a: 1, b: "hashed"},
indexKeyTestCases: [
{key: {a: "hashed", b: 1}, isCompatible: true},
{key: {a: 1, b: 1, c: 1}, isCompatible: true},
{key: {a: 1}, isCompatible: false},
{key: {b: "hashed"}, isCompatible: false},
{key: {b: "hashed", a: 1}, isCompatible: false},
],
},
{
shardKey: {"a.x": 1, b: 1},
indexKeyTestCases: [
{key: {"a.x": 1, b: "hashed"}, isCompatible: true},
{key: {"a.x": "hashed", b: 1}, isCompatible: true},
],
},
{
shardKey: {"a.x": "hashed", b: 1},
indexKeyTestCases: [
{key: {"a.x": 1, b: "hashed"}, isCompatible: true},
{key: {"a.x": 1, b: 1, c: 1}, isCompatible: true},
{key: {"a.x": "hashed"}, isCompatible: false},
{key: {b: 1}, isCompatible: false},
{key: {b: 1, "a.x": "hashed"}, isCompatible: false},
],
},
];
function runTest(hashedShardKeyIndexOptionalUponShardingCollection) {
jsTest.log(
"Testing with " +
tojson({
hashedShardKeyIndexOptionalUponShardingCollection,
}),
);
const st = new ShardingTest({
shards: 2,
rs: {
nodes: 1,
setParameter: {
featureFlagHashedShardKeyIndexOptionalUponShardingCollection:
hashedShardKeyIndexOptionalUponShardingCollection,
},
},
});
shardNames = [];
shardNames.push(st.shard0.shardName);
shardNames.push(st.shard1.shardName);
assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName}));
for (let shardKeyTestCase of shardKeyTestCases) {
const isHashedShardKey = AnalyzeShardKeyUtil.isHashedKeyPattern(shardKeyTestCase.shardKey);
jsTest.log("Testing " + tojson({shardKeyTestCase}));
for (let skipHashedShardKeyIndexCreation of [true, false]) {
const expectValidationError = !hashedShardKeyIndexOptionalUponShardingCollection || !isHashedShardKey;
testNonExistentCollection(st, {
shardKey: shardKeyTestCase.shardKey,
skipHashedShardKeyIndexCreation,
expectErrorCode: (() => {
if (expectValidationError) {
return ErrorCodes.InvalidOptions;
}
return null;
})(),
});
for (let isEmptyCollection of [true, false]) {
testExistentCollection(st, {
shardKey: shardKeyTestCase.shardKey,
isEmptyCollection,
skipHashedShardKeyIndexCreation,
expectErrorCode: (() => {
if (expectValidationError) {
return ErrorCodes.InvalidOptions;
}
if (skipHashedShardKeyIndexCreation) {
return null;
}
return isEmptyCollection ? null : ErrorCodes.InvalidOptions;
})(),
});
for (let indexTestCase of shardKeyTestCase.indexKeyTestCases) {
testExistentCollection(st, {
shardKey: shardKeyTestCase.shardKey,
indexKey: indexTestCase.key,
containsCompatibleIndex: indexTestCase.isCompatible,
isEmptyCollection,
skipHashedShardKeyIndexCreation,
expectErrorCode: (() => {
if (expectValidationError) {
return ErrorCodes.InvalidOptions;
}
if (skipHashedShardKeyIndexCreation) {
return null;
}
return isEmptyCollection ? null : ErrorCodes.InvalidOptions;
})(),
});
}
}
}
}
st.stop();
}
for (let hashedShardKeyIndexOptionalUponShardingCollection of [true, false]) {
runTest(hashedShardKeyIndexOptionalUponShardingCollection);
}

2
src/mongo/db/global_catalog/ddl/cluster_shard_collection_cmd.cpp
View File

@@ -126,6 +126,8 @@ public:
serverRequest.setTimeseries(clusterRequest.getTimeseries());
serverRequest.setCollectionUUID(clusterRequest.getCollectionUUID());
serverRequest.setImplicitlyCreateIndex(clusterRequest.getImplicitlyCreateIndex());
serverRequest.setSkipHashedShardKeyIndexCreation(
clusterRequest.getSkipHashedShardKeyIndexCreation());
serverRequest.setEnforceUniquenessCheck(clusterRequest.getEnforceUniquenessCheck());
ShardsvrCreateCollection shardsvrCreateCommand(nss);

31
src/mongo/db/global_catalog/ddl/create_collection_coordinator.cpp
View File

@@ -1066,6 +1066,31 @@ void checkCommandArguments(OperationContext* opCtx,
"dataShard and registerExistingCollectionInGlobalCatalog cannot be specified in the "
"same request",
!(request.getDataShard() && request.getRegisterExistingCollectionInGlobalCatalog()));
if (request.getSkipHashedShardKeyIndexCreation()) {
uassert(
ErrorCodes::InvalidOptions,
fmt::format("Cannot specify both '{}' and '{}'",
ShardsvrCreateCollectionRequest::kSkipHashedShardKeyIndexCreationFieldName,
ShardsvrCreateCollectionRequest::kImplicitlyCreateIndexFieldName),
!request.getImplicitlyCreateIndex().has_value());
const ShardKeyPattern shardKeyPattern{*request.getShardKey()};
uassert(
ErrorCodes::InvalidOptions,
fmt::format("Can only specify '{}' when sharding on a hashed shard key",
ShardsvrCreateCollectionRequest::kSkipHashedShardKeyIndexCreationFieldName),
shardKeyPattern.isHashedPattern());
uassert(
ErrorCodes::InvalidOptions,
fmt::format("Can only specify '{}' when "
"featureFlagHashedShardKeyIndexOptionalUponShardingCollection is enabled",
ShardsvrCreateCollectionRequest::kSkipHashedShardKeyIndexCreationFieldName),
feature_flags::gFeatureFlagHashedShardKeyIndexOptionalUponShardingCollection.isEnabled(
VersionContext::getDecoration(opCtx),
serverGlobalParams.featureCompatibility.acquireFCVSnapshot()));
}
}
/**
@@ -1313,7 +1338,11 @@ boost::optional<UUID> createCollectionAndIndexes(
return *sharding_ddl_util::getCollectionUUID(opCtx, translatedNss);
}
auto indexCreated = false;
if (request.getImplicitlyCreateIndex().value_or(true)) {
if (request.getSkipHashedShardKeyIndexCreation().value_or(false)) {
LOGV2(9974501,
"Skip checking for the shard key index and implicitly creating it per the settings"
"in the create collection request");
} else if (request.getImplicitlyCreateIndex().value_or(true)) {
indexCreated = shardkeyutil::validateShardKeyIndexExistsOrCreateIfPossible(
opCtx,
translatedNss,

8
src/mongo/db/global_catalog/ddl/shard_collection.idl
View File

@@ -63,6 +63,14 @@ structs:
implicitlyCreateIndex:
description: "Creates an index on the shard key pattern if the collection is empty."
type: optionalBool
skipHashedShardKeyIndexCreation:
type: bool
description:
"This setting determines whether to skip creating an index on the shard key
pattern if the shard key lacks an existing supporting index. This applies
regardless of whether the collection is empty or non-existent. Note that
this option can only be set when using a hashed shard key."
optional: true
enforceUniquenessCheck:
description: >-
Controls whether this command verifies that any unique indexes are prefixed by the shard

8
src/mongo/db/global_catalog/ddl/sharded_ddl_commands.idl
View File

@@ -192,6 +192,14 @@ structs:
implicitlyCreateIndex:
description: "Creates an index on the shard key pattern if the collection is empty."
type: optionalBool
skipHashedShardKeyIndexCreation:
type: bool
description:
"This setting determines whether to skip creating an index on the shard
key pattern if the shard key lacks an existing supporting index. This
applies regardless of whether the collection is empty or non-existent.
Note that this option can only be set when using a hashed shard key."
optional: true
enforceUniquenessCheck:
description: >-
Controls whether this command verifies that any unique indexes are prefixed by

7
src/mongo/db/sharding_environment/sharding_feature_flags.idl
View File

@@ -257,3 +257,10 @@ feature_flags:
cpp_varname: feature_flags::gFeatureFlagCheckVersioningCorrectness
default: false
fcv_gated: true
featureFlagHashedShardKeyIndexOptionalUponShardingCollection:
description:
"Feature flag to make shardCollection and reshardCollection not require a hashed
shard key to have a supporting index."
cpp_varname: feature_flags::gFeatureFlagHashedShardKeyIndexOptionalUponShardingCollection
default: false
fcv_gated: true