xuemingqiang/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-116221: Reference python3.13 in v5 toolchain (#47179)

Browse Source 
GitOrigin-RevId: d810136e7a469c63fa0051464b29f88924512d22
This commit is contained in:
Nick Jefferies
2026-02-05 11:02:51 -05:00
committed by MongoDB Bot
parent beb3aab2dd
commit 9f3133e98b
22 changed files with 231 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.devcontainer/Dockerfile
View File

@@ -130,10 +130,10 @@ RUN echo 'export PATH="$HOME/.local/bin:${PATH}"' > /etc/profile.d/03-local-bin.
USER $USERNAME
ENV PATH="/home/${USERNAME}/.local/bin:${PATH}"
RUN /opt/mongodbtoolchain/v5/bin/python3 -m venv /tmp/pipx-venv && \
/tmp/pipx-venv/bin/python -m pip install --upgrade "pip<20.3" && \
RUN /opt/mongodbtoolchain/v5/bin/python3.13 -m venv /tmp/pipx-venv && \
/tmp/pipx-venv/bin/python -m pip install --upgrade "pip==25.3" && \
/tmp/pipx-venv/bin/python -m pip install pipx && \
/tmp/pipx-venv/bin/pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3 --force && \
/tmp/pipx-venv/bin/pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3.13 --force && \
rm -rf /tmp/pipx-venv
# Note: PATH is configured via /etc/profile.d, not ~/.bashrc, to avoid modifying home volume

2
.devcontainer/post-create.sh
View File

@@ -138,7 +138,7 @@ echo "Creating Python virtual environment..."
venv_created=false
if [ ! -d "${WORKSPACE_FOLDER}/python3-venv/bin" ]; then
export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
/opt/mongodbtoolchain/v5/bin/python3 -m venv "${WORKSPACE_FOLDER}/python3-venv"
/opt/mongodbtoolchain/v5/bin/python3.13 -m venv "${WORKSPACE_FOLDER}/python3-venv"
venv_created=true
# Install dependencies in the newly created venv

28
.devcontainer/toolchain_config.env
View File

@@ -1,20 +1,20 @@
# Generated by toolchain.py
# DO NOT EDIT MANUALLY - run: python3 toolchain.py generate
# DO NOT EDIT MANUALLY - run: python3 toolchain.py
#
# Generated: 2025-10-01T12:25:08.235721
# Generated: 2026-01-30T19:44:10.636425
# ARM64 Toolchain
# Last Modified: 2025-07-21T20:49:55+00:00
# Toolchain ID: mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce
TOOLCHAIN_ARM64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz"
TOOLCHAIN_ARM64_SHA256="cdd2a58ed4a67dfa1be237d6042b7523552b543bb104c20db8f29068f3899fd6"
TOOLCHAIN_ARM64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz"
TOOLCHAIN_ARM64_LAST_MODIFIED="2025-07-21T20:49:55+00:00"
# Last Modified: 2026-01-30T18:35:20+00:00
# Toolchain ID: mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37
TOOLCHAIN_ARM64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37.tar.gz"
TOOLCHAIN_ARM64_SHA256="d4baf5bff8f3ef053b234028a0d3148193eaa59840c8c0fc3544809cf2d4c9df"
TOOLCHAIN_ARM64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37.tar.gz"
TOOLCHAIN_ARM64_LAST_MODIFIED="2026-01-30T18:35:20+00:00"
# AMD64 Toolchain
# Last Modified: 2025-07-21T19:57:00+00:00
# Toolchain ID: mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce
TOOLCHAIN_AMD64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz"
TOOLCHAIN_AMD64_SHA256="55b927124ffbf040b0caf19cb7b440679d71e57ae29c1c40df0891b884dcd2cd"
TOOLCHAIN_AMD64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz"
TOOLCHAIN_AMD64_LAST_MODIFIED="2025-07-21T19:57:00+00:00"
# Last Modified: 2026-01-16T22:50:49+00:00
# Toolchain ID: mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd
TOOLCHAIN_AMD64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd.tar.gz"
TOOLCHAIN_AMD64_SHA256="aebfc49a6f7ee9c1430807086b964c3756d491048d68572c808af2fc2e805374"
TOOLCHAIN_AMD64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd.tar.gz"
TOOLCHAIN_AMD64_LAST_MODIFIED="2026-01-16T22:50:49+00:00"

18
buildscripts/auto_install_db_contrib_tool.sh
View File

@@ -25,8 +25,8 @@ if [[ -f "$HOME/.zshrc" ]]; then
fi
if ! command -v db-contrib-tool &>/dev/null; then
if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then
actual_version=$(python3 -c 'import sys; print(sys.version)')
if ! python3.13 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then
actual_version=$(python3.13 -c 'import sys; print(sys.version)')
echo "You must have python3.7+ installed. Detected version $actual_version."
echo "To avoid unexpected issues, python3.7+ will not be automatically installed."
echo "Please, do it yourself."
@@ -48,28 +48,28 @@ if ! command -v db-contrib-tool &>/dev/null; then
source "$rc_file"
fi
pipx install db-contrib-tool --python $(command -v python3) --force
pipx install db-contrib-tool --python $(command -v python3.13) --force
echo
else
if ! python3 -m pipx --version &>/dev/null; then
if ! python3.13 -m pipx --version &>/dev/null; then
echo "Couldn't find pipx. Installing it as python3 module:"
echo " $(command -v python3) -m pip install pipx"
echo " $(command -v python3.13) -m pip install pipx"
echo
python3 -m pip install pipx
python3.13 -m pip install pipx
echo
else
echo "Found pipx installed as python3 module:"
echo " $(command -v python3) -m pipx --version"
echo " $(command -v python3.13) -m pipx --version"
echo "Using it to install 'db-contrib-tool'."
echo
fi
python3 -m pipx ensurepath &>/dev/null
python3.13 -m pipx ensurepath &>/dev/null
if [[ -f "$rc_file" ]]; then
source "$rc_file"
fi
python3 -m pipx install db-contrib-tool --force
python3.13 -m pipx install db-contrib-tool --force
echo
fi
fi

2
buildscripts/resmokelib/generate_fuzz_config/config_fuzzer_limits.py
View File

@@ -175,7 +175,7 @@ config_fuzzer_params = {
# batch limit operations.
"replBatchLimitOperations": {
"min": 1,
"max": 0.2 * 1000 * 1000,
"max": int(0.2 * 1000 * 1000), # Must be int for randint() compatibility
"period": 5,
"fuzz_at": ["startup", "runtime"],
},

51
buildscripts/resmokelib/generate_fuzz_config/mongo_fuzzer_configs.py
View File

@@ -8,14 +8,26 @@ import stat
from buildscripts.resmokelib import config, utils
def generate_normal_wt_parameters(rng, value):
def safe_randint(rng, min_val, max_val, param_name=None):
"""Wrapper for randint() that validates integer arguments and provides context on failure."""
if not isinstance(min_val, int) or not isinstance(max_val, int):
context = f" (parameter: {param_name})" if param_name else ""
raise TypeError(
f"randint() requires integer arguments{context}. "
f"Got min={min_val} (type={type(min_val).__name__}), max={max_val} (type={type(max_val).__name__}). "
f"Fix: use int() in config file or add 'isUniform': True for float ranges."
)
return rng.randint(min_val, max_val)
def generate_normal_wt_parameters(rng, value, param_name=None):
"""Returns the value assigned the WiredTiger parameters (both eviction or table) based on the fields of the parameters in the config_fuzzer_wt_limits.py."""
if "choices" in value:
ret = rng.choice(value["choices"])
if "multiplier" in value:
ret *= value["multiplier"]
elif "min" in value and "max" in value:
ret = rng.randint(value["min"], value["max"])
ret = safe_randint(rng, value["min"], value["max"], param_name)
return ret
@@ -96,7 +108,7 @@ def generate_eviction_configs(rng):
ret = generate_special_eviction_configs(rng, ret, params)
ret.update(
{
key: generate_normal_wt_parameters(rng, value)
key: generate_normal_wt_parameters(rng, value, key)
for key, value in params.items()
if key not in excluded_normal_params
}
@@ -165,7 +177,7 @@ def generate_table_configs(rng):
ret.update(
{
key: generate_normal_wt_parameters(rng, value)
key: generate_normal_wt_parameters(rng, value, key)
for key, value in params.items()
if key not in excluded_normal_params
}
@@ -222,7 +234,22 @@ def generate_encryption_config(rng: random.Random):
return ret
def generate_normal_mongo_parameters(rng, value):
def generate_runtime_mongod_parameter(rng, value, param_name):
"""Generate a runtime mongod parameter value, handling mongod-specific special cases.
This function is used for runtime fuzzing of mongod parameters and ensures consistent
behavior with startup generation for parameters that have special handling.
"""
# Special case: flowControlThresholdLagPercentage uses rng.random() which returns [0.0, 1.0)
# This matches the behavior in generate_flow_control_parameters() used at startup.
if param_name == "flowControlThresholdLagPercentage":
return rng.random()
# Default to normal generation for all other parameters
return generate_normal_mongo_parameters(rng, value, param_name)
def generate_normal_mongo_parameters(rng, value, param_name=None):
"""Returns the value assigned the mongod or mongos parameter based on the fields of the parameters in the config_fuzzer_limits.py."""
if "document" in value:
@@ -231,7 +258,7 @@ def generate_normal_mongo_parameters(rng, value):
if "exclude_prob" in doc_value and rng.random() < doc_value["exclude_prob"]:
# Exclude this key from the document
continue
ret[doc_key] = generate_normal_mongo_parameters(rng, doc_value)
ret[doc_key] = generate_normal_mongo_parameters(rng, doc_value, doc_key)
elif "isUniform" in value:
ret = rng.uniform(value["min"], value["max"])
elif "isRandomizedChoice" in value:
@@ -241,7 +268,7 @@ def generate_normal_mongo_parameters(rng, value):
elif "choices" in value:
ret = rng.choice(value["choices"])
elif "min" in value and "max" in value:
ret = rng.randint(value["min"], value["max"])
ret = safe_randint(rng, value["min"], value["max"], param_name)
if "multiplier" in value:
ret *= value["multiplier"]
elif "default" in value:
@@ -338,10 +365,12 @@ def generate_flow_control_parameters(rng, ret, flow_control_params, params):
ret["enableFlowControl"] = rng.choice(params["enableFlowControl"]["choices"])
if ret["enableFlowControl"]:
for name in flow_control_params:
if name == "flowControlThresholdLagPercentage":
continue # Handled specially below with rng.random()
if "isUniform" in params[name]:
ret[name] = rng.uniform(params[name]["min"], params[name]["max"])
else:
ret[name] = rng.randint(params[name]["min"], params[name]["max"])
ret[name] = safe_randint(rng, params[name]["min"], params[name]["max"], name)
ret["flowControlThresholdLagPercentage"] = rng.random()
return ret
@@ -398,7 +427,7 @@ def generate_mongod_parameters(rng):
# Range through all other parameters and assign the parameters based on the keys that are available or the parameter set lists defined above.
ret.update(
{
key: generate_normal_mongo_parameters(rng, value)
key: generate_normal_mongo_parameters(rng, value, key)
for key, value in params.items()
if key not in excluded_normal_params and key not in flow_control_params
}
@@ -416,7 +445,7 @@ def generate_mongod_extra_configs(rng):
)
generated_config = {
key: generate_normal_mongo_parameters(rng, value)
key: generate_normal_mongo_parameters(rng, value, key)
for key, value in config_fuzzer_extra_configs["mongod"].items()
if not (value.get("enterprise_only", False) and "enterprise" not in config.MODULES)
}
@@ -444,7 +473,7 @@ def generate_mongos_parameters(rng):
and not (val.get("enterprise_only", False) and "enterprise" not in config.MODULES)
}
return {key: generate_normal_mongo_parameters(rng, value) for key, value in params.items()}
return {key: generate_normal_mongo_parameters(rng, value, key) for key, value in params.items()}
def fuzz_mongod_set_parameters(seed, user_provided_params):

2
buildscripts/resmokelib/powercycle/setup/__init__.py
View File

@@ -59,7 +59,7 @@ class SetUpEC2Instance(PowercycleCommand):
# Set up virtualenv on remote.
venv = powercycle_constants.VIRTUALENV_DIR
python = (
"/opt/mongodbtoolchain/v4/bin/python3"
"/opt/mongodbtoolchain/v5/bin/python3.13"
if "python" not in self.expansions
else self.expansions["python"]
)

2
buildscripts/resmokelib/run/__init__.py
View File

@@ -2079,7 +2079,7 @@ class RunPlugin(PluginInterface):
dest="fuzz_mongod_configs",
help="Randomly chooses mongod parameters that were not specified.",
metavar="MODE",
choices=("normal"),
choices=("normal",),
)
mongodb_server_options.add_argument(

14
buildscripts/resmokelib/testing/hooks/fuzz_runtime_parameters.py
View File

@@ -12,6 +12,7 @@ from pymongo.errors import OperationFailure
from buildscripts.resmokelib import config, errors
from buildscripts.resmokelib.generate_fuzz_config.mongo_fuzzer_configs import (
generate_normal_mongo_parameters,
generate_runtime_mongod_parameter,
)
from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface
from buildscripts.resmokelib.testing.fixtures import replicaset, shardedcluster, standalone
@@ -39,7 +40,7 @@ class RuntimeParametersState:
"""Encapsulates the runtime-state of a set of parameters we are fuzzing. Tracks the last time we set a parameter value and holds
the logic for generating new values."""
def __init__(self, spec, seed):
def __init__(self, spec, seed, generator_func=None):
# Initialize the runtime state of each parameter in the spec, including the lastSet time at now, so we start setting the parameters
# at appropriate intervals after the suite begins.
now = time.time()
@@ -47,6 +48,10 @@ class RuntimeParametersState:
key: {**copy.deepcopy(value), "lastSet": now} for key, value in spec.items()
}
self._rng = random.Random(seed)
# Use provided generator function, or default to generate_normal_mongo_parameters for backward compatibility
self._generator_func = (
generator_func if generator_func is not None else generate_normal_mongo_parameters
)
def generate_parameters(self):
"""Returns a dictionary of what parameters should be set now, along with values to set them to, based on the last time the
@@ -55,7 +60,7 @@ class RuntimeParametersState:
now = time.time()
for key, value in self._params.items():
if now - value["lastSet"] >= value["period"]:
ret[key] = generate_normal_mongo_parameters(self._rng, value)
ret[key] = self._generator_func(self._rng, value, key)
value["lastSet"] = now
return ret
@@ -153,7 +158,10 @@ class FuzzRuntimeParameters(interface.Hook):
validate_runtime_parameter_spec(cluster_params)
# Construct the runtime state before the suite begins.
# The initial lastSet time of each parameter is the start time of the suite.
self._mongod_param_state = RuntimeParametersState(runtime_mongod_params, self._seed)
# Use generate_runtime_mongod_parameter for mongod params to handle special cases.
self._mongod_param_state = RuntimeParametersState(
runtime_mongod_params, self._seed, generate_runtime_mongod_parameter
)
self._mongos_param_state = RuntimeParametersState(runtime_mongos_params, self._seed)
self._cluster_param_state = RuntimeParametersState(cluster_params, self._seed)

10
buildscripts/tests/resmoke_end2end/README.md
View File

@@ -1,11 +1,17 @@
First, activate the virtual environment:
```
mongodb_repo_root$ source python3-venv/bin/activate
```
- All end-to-end resmoke tests can be run via a resmoke suite itself:
```
mongodb_repo_root$ /opt/mongodbtoolchain/v4/bin/python3 buildscripts/resmoke.py run --suites resmoke_end2end_tests
(python3-venv) mongodb_repo_root$ python buildscripts/resmoke.py run --suites resmoke_end2end_tests
```
- Finer grained control of tests can also be run with by invoking python's unittest main by hand. E.g:
```
mongodb_repo_root$ /opt/mongodbtoolchain/v4/bin/python3 -m unittest -v buildscripts.tests.resmoke_end2end.test_resmoke.TestTestSelection.test_at_sign_as_replay_file
(python3-venv) mongodb_repo_root$ python -m unittest -v buildscripts.tests.resmoke_end2end.test_resmoke.TestTestSelection.test_at_sign_as_replay_file
```

2
buildscripts/tests/resmoke_end2end/test_resmoke.py
View File

@@ -933,7 +933,7 @@ class TestCoreAnalyzerFunctions(unittest.TestCase):
task_name = "test_tast_name"
execution = "0"
generated_task_name = get_generated_task_name(task_name, execution)
self.assertEquals(matches_generated_task_pattern(task_name, generated_task_name), execution)
self.assertEqual(matches_generated_task_pattern(task_name, generated_task_name), execution)
self.assertIsNone(matches_generated_task_pattern("not_same_task", generated_task_name))

2
docs/devcontainer/troubleshooting.md
View File

@@ -421,7 +421,7 @@ For additional VS Code-specific troubleshooting, see:
```bash
rm -rf python3-venv
/opt/mongodbtoolchain/v5/bin/python3 -m venv python3-venv
/opt/mongodbtoolchain/v5/bin/python3.13 -m venv python3-venv
source python3-venv/bin/activate
poetry install --no-root --sync
```

8
etc/set_up_workstation.sh
View File

@@ -56,7 +56,7 @@ setup_mongo_venv() {
# PYTHON_KEYRING_BACKEND is needed to make poetry install work
# See guide https://wiki.corp.mongodb.com/display/KERNEL/Virtual+Workstation
export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
/opt/mongodbtoolchain/v4/bin/python3 -m venv python3-venv
/opt/mongodbtoolchain/v5/bin/python3.13 -m venv python3-venv
source ./python3-venv/bin/activate
POETRY_VIRTUALENVS_IN_PROJECT=true poetry install --no-root --sync
@@ -86,17 +86,17 @@ setup_pipx() {
else
export PATH="$PATH:$HOME/.local/bin"
local venv_name="tmp-pipx-venv"
/opt/mongodbtoolchain/v4/bin/python3 -m venv $venv_name
/opt/mongodbtoolchain/v5/bin/python3.13 -m venv $venv_name
# virtualenv doesn't like nounset
set +o nounset
source $venv_name/bin/activate
set -o nounset
python -m pip install --upgrade "pip<20.3"
python -m pip --disable-pip-version-check install "pip==25.3" "wheel==0.45.1"
python -m pip install pipx
pipx install pipx --python /opt/mongodbtoolchain/v4/bin/python3 --force
pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3.13 --force
pipx ensurepath --force
set +o nounset

13
evergreen/prelude_python.sh
View File

@@ -5,9 +5,16 @@ elif [ "$(uname)" = "Darwin" ]; then
python='/Library/Frameworks/Python.Framework/Versions/3.13/bin/python3'
echo "Executing on mac, setting python to ${python}"
else
if [ -f /opt/mongodbtoolchain/v5/bin/python3 ]; then
python="/opt/mongodbtoolchain/v5/bin/python3"
echo "Found python in v5 toolchain, setting python to ${python}"
# Check if v5 toolchain exists - it requires Python 3.13
if [ -d /opt/mongodbtoolchain/v5 ]; then
if [ -f /opt/mongodbtoolchain/v5/bin/python3.13 ]; then
python="/opt/mongodbtoolchain/v5/bin/python3.13"
echo "Found python 3.13 in v5 toolchain, setting python to ${python}"
else
echo "ERROR: v5 toolchain exists but Python 3.13 is not available at /opt/mongodbtoolchain/v5/bin/python3.13"
echo "The v5 toolchain requires Python 3.13. Please ensure python3.13 is installed in the toolchain."
return 1
fi
elif [ -f /opt/mongodbtoolchain/v4/bin/python3 ]; then
python="/opt/mongodbtoolchain/v4/bin/python3"
echo "Found python in v4 toolchain, setting python to ${python}"

2
evergreen/selinux_test_setup.sh
View File

@@ -31,7 +31,7 @@ if ! sudo --non-interactive rpm --install --verbose --verbose --hash --nodeps "$
fi
# install packages needed by check_has_tag.py
PYTHON=/opt/mongodbtoolchain/v5/bin/python3
PYTHON=/opt/mongodbtoolchain/v5/bin/python3.13
if [[ (-f "$PYTHON" || -L "$PYTHON") && -x "$PYTHON" ]]; then
echo "==== Found python3 in $PYTHON"
$PYTHON -m pip install pyyaml

2
jstests/libs/python.js
View File

@@ -20,7 +20,7 @@ export function getPython3Binary() {
}
const paths = [
"/opt/mongodbtoolchain/v5/bin/python3",
"/opt/mongodbtoolchain/v5/bin/python3.13",
"/opt/mongodbtoolchain/v4/bin/python3",
"/cygdrive/c/python/python313/python.exe",
"c:/python/python313/python.exe",

2
jstests/query_golden/expected_output/automaticCE/CBRForNoMultiplanningResults/plan_stability2
View File

@@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

2
jstests/query_golden/expected_output/automaticCE/HistogramCEWithHeuristicFallback/plan_stability2
View File

@@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

2
jstests/query_golden/expected_output/histogramCE/plan_stability2
View File

@@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

2
jstests/query_golden/expected_output/plan_stability2
View File

@@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

2
jstests/query_golden/expected_output/samplingCE/plan_stability2
View File

@@ -7,7 +7,7 @@
[jsTest] ----
[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment.
[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment.
[jsTest] ----
{">>>pipelines":[

126
jstests/sharding/libs/proxy_protocol_server.py
View File

@@ -8,13 +8,18 @@ The most recent source code is available on [GitHub][2].
The installed version is listed in the
"[tool.poetry.group.testing.dependencies]" section of `pyproject.toml`.
This wrapper adds two patches:
1. Logs "Now listening on [...]" when the server is ready
2. Calls server.close_clients() during shutdown for Python 3.13 compatibility
[1]: https://pypi.org/project/proxy-protocol/
[2]: https://github.com/icgood/proxy-protocol/
"""
import sys
from asyncio.base_events import BaseEventLoop
import proxyprotocol.server.main
from proxyprotocol.server.main import main
# We want to know when the proxy protocol server is ready to accept connections; so, we log to
@@ -23,16 +28,123 @@ from proxyprotocol.server.main import main
# thing is to "monkey patch" the standard method
# [asyncio.base_events.BaseEventLoop.create_server][1] so that it logs after the server is created.
#
# Additionally, we store a reference to the server objects so they can be properly shut down.
#
# [1]: https://github.com/python/cpython/blob/5c19c5bac6abf3da97d1d9b80cfa16e003897096/Lib/asyncio/base_events.py#L1429
original_create_server = BaseEventLoop.create_server
_servers = []
_original_create_server = BaseEventLoop.create_server
async def monkeypatched_create_server(self, protocol_factory, host, port, *args, **kwargs):
result = await original_create_server(self, protocol_factory, host, port, *args, **kwargs)
print(f"Now listening on {host}:{port}")
return result
async def _patched_create_server(self, protocol_factory, host, port, *args, **kwargs):
server = await _original_create_server(self, protocol_factory, host, port, *args, **kwargs)
_servers.append(server)
print(f"Now listening on {host}:{port}", flush=True)
return server
BaseEventLoop.create_server = _patched_create_server
# Monkey-patch the library's run() function to add close_clients() call during shutdown.
#
# IMPORTANT: The code below is copied from proxyprotocol.server.main.run() v0.11.3
# (see python3-venv/lib/python3.*/site-packages/proxyprotocol/server/main.py)
# with minimal modifications for Python 3.13 compatibility.
#
# CHANGES FROM LIBRARY SOURCE:
# 1. Added server.close_clients() call in handle_signal() before forever.cancel()
# - Python 3.13 added Server.close_clients() to forcibly close active connections
# - Python 3.12+ fixed wait_closed() to correctly wait for connections
# - Library's original code relied on Python 3.10/3.11 bug where wait_closed()
# returned immediately even with active connections
# 2. Added logging to track signal handling and shutdown progress
#
_original_run = proxyprotocol.server.main.run
async def _patched_run(args):
import asyncio
import signal
from asyncio import CancelledError
from contextlib import AsyncExitStack
from functools import partial
from proxyprotocol.dnsbl import Dnsbl
from proxyprotocol.server import Address
from proxyprotocol.server.protocol import DownstreamProtocol, UpstreamProtocol
# --- BEGIN: Code copied from library's run() function ---
loop = asyncio.get_running_loop()
services = [(Address(source, server=True), Address(dest)) for (source, dest) in args.services]
buf_len = args.buf_len
dnsbl = Dnsbl.load(args.dnsbl, timeout=args.dnsbl_timeout)
new_server = partial(DownstreamProtocol, UpstreamProtocol, loop, buf_len, dnsbl)
servers = [
await loop.create_server(
partial(new_server, dest), source.host, source.port or 0, ssl=source.ssl
)
for source, dest in services
]
async with AsyncExitStack() as stack:
for server in servers:
await stack.enter_async_context(server)
forever = asyncio.gather(*[server.serve_forever() for server in servers])
# --- BEGIN MODIFICATION: Added proper shutdown sequence ---
def handle_signal():
print("Proxy server received shutdown signal", flush=True)
# Step 1: Stop accepting new connections
try:
for server in servers:
server.close()
print("Proxy server: stopped accepting new connections", flush=True)
except Exception as e:
print(f"Proxy server: error in close(): {e}", flush=True)
# Step 2: Close existing client connections gracefully
try:
for server in servers:
server.close_clients()
print("Proxy server: initiated graceful client close", flush=True)
except Exception as e:
print(f"Proxy server: error in close_clients(): {e}", flush=True)
# Step 3: Schedule abort as a safety fallback after 1 second
def abort_remaining():
print("Proxy server: aborting any remaining connections", flush=True)
try:
for server in servers:
server.abort_clients()
except Exception as e:
print(f"Proxy server: error in abort_clients(): {e}", flush=True)
loop.call_later(1.0, abort_remaining)
# CHANGE: Use proper shutdown sequence per Python 3.13 docs:
# 1. close() to stop accepting new connections
# 2. close_clients() to gracefully close existing connections
# 3. Schedule abort_clients() as fallback after timeout
forever.cancel()
# --- END MODIFICATION ---
loop.add_signal_handler(signal.SIGINT, handle_signal)
loop.add_signal_handler(signal.SIGTERM, handle_signal)
try:
await forever
except CancelledError:
pass
# --- END: Code copied from library's run() function ---
print("Proxy server shutdown complete", flush=True)
return 0
proxyprotocol.server.main.run = _patched_run
if __name__ == "__main__":
BaseEventLoop.create_server = monkeypatched_create_server
sys.exit(main())