diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 45636b64c5c..45d5acc5309 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -130,10 +130,10 @@ RUN echo 'export PATH="$HOME/.local/bin:${PATH}"' > /etc/profile.d/03-local-bin. USER $USERNAME ENV PATH="/home/${USERNAME}/.local/bin:${PATH}" -RUN /opt/mongodbtoolchain/v5/bin/python3 -m venv /tmp/pipx-venv && \ - /tmp/pipx-venv/bin/python -m pip install --upgrade "pip<20.3" && \ +RUN /opt/mongodbtoolchain/v5/bin/python3.13 -m venv /tmp/pipx-venv && \ + /tmp/pipx-venv/bin/python -m pip install --upgrade "pip==25.3" && \ /tmp/pipx-venv/bin/python -m pip install pipx && \ - /tmp/pipx-venv/bin/pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3 --force && \ + /tmp/pipx-venv/bin/pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3.13 --force && \ rm -rf /tmp/pipx-venv # Note: PATH is configured via /etc/profile.d, not ~/.bashrc, to avoid modifying home volume diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index a6828e51da0..403a6534c9d 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -138,7 +138,7 @@ echo "Creating Python virtual environment..." venv_created=false if [ ! -d "${WORKSPACE_FOLDER}/python3-venv/bin" ]; then export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring - /opt/mongodbtoolchain/v5/bin/python3 -m venv "${WORKSPACE_FOLDER}/python3-venv" + /opt/mongodbtoolchain/v5/bin/python3.13 -m venv "${WORKSPACE_FOLDER}/python3-venv" venv_created=true # Install dependencies in the newly created venv diff --git a/.devcontainer/toolchain_config.env b/.devcontainer/toolchain_config.env index 0bdfe917d28..ed68e127197 100644 --- a/.devcontainer/toolchain_config.env +++ b/.devcontainer/toolchain_config.env @@ -1,20 +1,20 @@ # Generated by toolchain.py -# DO NOT EDIT MANUALLY - run: python3 toolchain.py generate +# DO NOT EDIT MANUALLY - run: python3 toolchain.py # -# Generated: 2025-10-01T12:25:08.235721 +# Generated: 2026-01-30T19:44:10.636425 # ARM64 Toolchain -# Last Modified: 2025-07-21T20:49:55+00:00 -# Toolchain ID: mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce -TOOLCHAIN_ARM64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz" -TOOLCHAIN_ARM64_SHA256="cdd2a58ed4a67dfa1be237d6042b7523552b543bb104c20db8f29068f3899fd6" -TOOLCHAIN_ARM64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-arm64-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz" -TOOLCHAIN_ARM64_LAST_MODIFIED="2025-07-21T20:49:55+00:00" +# Last Modified: 2026-01-30T18:35:20+00:00 +# Toolchain ID: mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37 +TOOLCHAIN_ARM64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37.tar.gz" +TOOLCHAIN_ARM64_SHA256="d4baf5bff8f3ef053b234028a0d3148193eaa59840c8c0fc3544809cf2d4c9df" +TOOLCHAIN_ARM64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-arm64-f8b0e9c403c5fcdde2f10f382e9f4fe7ec61cc37.tar.gz" +TOOLCHAIN_ARM64_LAST_MODIFIED="2026-01-30T18:35:20+00:00" # AMD64 Toolchain -# Last Modified: 2025-07-21T19:57:00+00:00 -# Toolchain ID: mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce -TOOLCHAIN_AMD64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz" -TOOLCHAIN_AMD64_SHA256="55b927124ffbf040b0caf19cb7b440679d71e57ae29c1c40df0891b884dcd2cd" -TOOLCHAIN_AMD64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-c36013b8bab41fcd3cbfd5e4b4590cd0c10ea6ce.tar.gz" -TOOLCHAIN_AMD64_LAST_MODIFIED="2025-07-21T19:57:00+00:00" +# Last Modified: 2026-01-16T22:50:49+00:00 +# Toolchain ID: mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd +TOOLCHAIN_AMD64_URL="https://s3.amazonaws.com/boxes.10gen.com/build/toolchain/mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd.tar.gz" +TOOLCHAIN_AMD64_SHA256="aebfc49a6f7ee9c1430807086b964c3756d491048d68572c808af2fc2e805374" +TOOLCHAIN_AMD64_KEY="build/toolchain/mongodbtoolchain-ubuntu2404-e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd.tar.gz" +TOOLCHAIN_AMD64_LAST_MODIFIED="2026-01-16T22:50:49+00:00" diff --git a/buildscripts/auto_install_db_contrib_tool.sh b/buildscripts/auto_install_db_contrib_tool.sh index 5572727c379..a07104fdbf7 100755 --- a/buildscripts/auto_install_db_contrib_tool.sh +++ b/buildscripts/auto_install_db_contrib_tool.sh @@ -25,8 +25,8 @@ if [[ -f "$HOME/.zshrc" ]]; then fi if ! command -v db-contrib-tool &>/dev/null; then - if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then - actual_version=$(python3 -c 'import sys; print(sys.version)') + if ! python3.13 -c "import sys; sys.exit(sys.version_info < (3, 7))" &>/dev/null; then + actual_version=$(python3.13 -c 'import sys; print(sys.version)') echo "You must have python3.7+ installed. Detected version $actual_version." echo "To avoid unexpected issues, python3.7+ will not be automatically installed." echo "Please, do it yourself." @@ -48,28 +48,28 @@ if ! command -v db-contrib-tool &>/dev/null; then source "$rc_file" fi - pipx install db-contrib-tool --python $(command -v python3) --force + pipx install db-contrib-tool --python $(command -v python3.13) --force echo else - if ! python3 -m pipx --version &>/dev/null; then + if ! python3.13 -m pipx --version &>/dev/null; then echo "Couldn't find pipx. Installing it as python3 module:" - echo " $(command -v python3) -m pip install pipx" + echo " $(command -v python3.13) -m pip install pipx" echo - python3 -m pip install pipx + python3.13 -m pip install pipx echo else echo "Found pipx installed as python3 module:" - echo " $(command -v python3) -m pipx --version" + echo " $(command -v python3.13) -m pipx --version" echo "Using it to install 'db-contrib-tool'." echo fi - python3 -m pipx ensurepath &>/dev/null + python3.13 -m pipx ensurepath &>/dev/null if [[ -f "$rc_file" ]]; then source "$rc_file" fi - python3 -m pipx install db-contrib-tool --force + python3.13 -m pipx install db-contrib-tool --force echo fi fi diff --git a/buildscripts/resmokelib/generate_fuzz_config/config_fuzzer_limits.py b/buildscripts/resmokelib/generate_fuzz_config/config_fuzzer_limits.py index f805947d8f0..61b0060f2a9 100644 --- a/buildscripts/resmokelib/generate_fuzz_config/config_fuzzer_limits.py +++ b/buildscripts/resmokelib/generate_fuzz_config/config_fuzzer_limits.py @@ -175,7 +175,7 @@ config_fuzzer_params = { # batch limit operations. "replBatchLimitOperations": { "min": 1, - "max": 0.2 * 1000 * 1000, + "max": int(0.2 * 1000 * 1000), # Must be int for randint() compatibility "period": 5, "fuzz_at": ["startup", "runtime"], }, diff --git a/buildscripts/resmokelib/generate_fuzz_config/mongo_fuzzer_configs.py b/buildscripts/resmokelib/generate_fuzz_config/mongo_fuzzer_configs.py index b1477d4683d..2eb2165996c 100644 --- a/buildscripts/resmokelib/generate_fuzz_config/mongo_fuzzer_configs.py +++ b/buildscripts/resmokelib/generate_fuzz_config/mongo_fuzzer_configs.py @@ -8,14 +8,26 @@ import stat from buildscripts.resmokelib import config, utils -def generate_normal_wt_parameters(rng, value): +def safe_randint(rng, min_val, max_val, param_name=None): + """Wrapper for randint() that validates integer arguments and provides context on failure.""" + if not isinstance(min_val, int) or not isinstance(max_val, int): + context = f" (parameter: {param_name})" if param_name else "" + raise TypeError( + f"randint() requires integer arguments{context}. " + f"Got min={min_val} (type={type(min_val).__name__}), max={max_val} (type={type(max_val).__name__}). " + f"Fix: use int() in config file or add 'isUniform': True for float ranges." + ) + return rng.randint(min_val, max_val) + + +def generate_normal_wt_parameters(rng, value, param_name=None): """Returns the value assigned the WiredTiger parameters (both eviction or table) based on the fields of the parameters in the config_fuzzer_wt_limits.py.""" if "choices" in value: ret = rng.choice(value["choices"]) if "multiplier" in value: ret *= value["multiplier"] elif "min" in value and "max" in value: - ret = rng.randint(value["min"], value["max"]) + ret = safe_randint(rng, value["min"], value["max"], param_name) return ret @@ -96,7 +108,7 @@ def generate_eviction_configs(rng): ret = generate_special_eviction_configs(rng, ret, params) ret.update( { - key: generate_normal_wt_parameters(rng, value) + key: generate_normal_wt_parameters(rng, value, key) for key, value in params.items() if key not in excluded_normal_params } @@ -165,7 +177,7 @@ def generate_table_configs(rng): ret.update( { - key: generate_normal_wt_parameters(rng, value) + key: generate_normal_wt_parameters(rng, value, key) for key, value in params.items() if key not in excluded_normal_params } @@ -222,7 +234,22 @@ def generate_encryption_config(rng: random.Random): return ret -def generate_normal_mongo_parameters(rng, value): +def generate_runtime_mongod_parameter(rng, value, param_name): + """Generate a runtime mongod parameter value, handling mongod-specific special cases. + + This function is used for runtime fuzzing of mongod parameters and ensures consistent + behavior with startup generation for parameters that have special handling. + """ + # Special case: flowControlThresholdLagPercentage uses rng.random() which returns [0.0, 1.0) + # This matches the behavior in generate_flow_control_parameters() used at startup. + if param_name == "flowControlThresholdLagPercentage": + return rng.random() + + # Default to normal generation for all other parameters + return generate_normal_mongo_parameters(rng, value, param_name) + + +def generate_normal_mongo_parameters(rng, value, param_name=None): """Returns the value assigned the mongod or mongos parameter based on the fields of the parameters in the config_fuzzer_limits.py.""" if "document" in value: @@ -231,7 +258,7 @@ def generate_normal_mongo_parameters(rng, value): if "exclude_prob" in doc_value and rng.random() < doc_value["exclude_prob"]: # Exclude this key from the document continue - ret[doc_key] = generate_normal_mongo_parameters(rng, doc_value) + ret[doc_key] = generate_normal_mongo_parameters(rng, doc_value, doc_key) elif "isUniform" in value: ret = rng.uniform(value["min"], value["max"]) elif "isRandomizedChoice" in value: @@ -241,7 +268,7 @@ def generate_normal_mongo_parameters(rng, value): elif "choices" in value: ret = rng.choice(value["choices"]) elif "min" in value and "max" in value: - ret = rng.randint(value["min"], value["max"]) + ret = safe_randint(rng, value["min"], value["max"], param_name) if "multiplier" in value: ret *= value["multiplier"] elif "default" in value: @@ -338,10 +365,12 @@ def generate_flow_control_parameters(rng, ret, flow_control_params, params): ret["enableFlowControl"] = rng.choice(params["enableFlowControl"]["choices"]) if ret["enableFlowControl"]: for name in flow_control_params: + if name == "flowControlThresholdLagPercentage": + continue # Handled specially below with rng.random() if "isUniform" in params[name]: ret[name] = rng.uniform(params[name]["min"], params[name]["max"]) else: - ret[name] = rng.randint(params[name]["min"], params[name]["max"]) + ret[name] = safe_randint(rng, params[name]["min"], params[name]["max"], name) ret["flowControlThresholdLagPercentage"] = rng.random() return ret @@ -398,7 +427,7 @@ def generate_mongod_parameters(rng): # Range through all other parameters and assign the parameters based on the keys that are available or the parameter set lists defined above. ret.update( { - key: generate_normal_mongo_parameters(rng, value) + key: generate_normal_mongo_parameters(rng, value, key) for key, value in params.items() if key not in excluded_normal_params and key not in flow_control_params } @@ -416,7 +445,7 @@ def generate_mongod_extra_configs(rng): ) generated_config = { - key: generate_normal_mongo_parameters(rng, value) + key: generate_normal_mongo_parameters(rng, value, key) for key, value in config_fuzzer_extra_configs["mongod"].items() if not (value.get("enterprise_only", False) and "enterprise" not in config.MODULES) } @@ -444,7 +473,7 @@ def generate_mongos_parameters(rng): and not (val.get("enterprise_only", False) and "enterprise" not in config.MODULES) } - return {key: generate_normal_mongo_parameters(rng, value) for key, value in params.items()} + return {key: generate_normal_mongo_parameters(rng, value, key) for key, value in params.items()} def fuzz_mongod_set_parameters(seed, user_provided_params): diff --git a/buildscripts/resmokelib/powercycle/setup/__init__.py b/buildscripts/resmokelib/powercycle/setup/__init__.py index de4a0733d77..2c958cc24d1 100644 --- a/buildscripts/resmokelib/powercycle/setup/__init__.py +++ b/buildscripts/resmokelib/powercycle/setup/__init__.py @@ -59,7 +59,7 @@ class SetUpEC2Instance(PowercycleCommand): # Set up virtualenv on remote. venv = powercycle_constants.VIRTUALENV_DIR python = ( - "/opt/mongodbtoolchain/v4/bin/python3" + "/opt/mongodbtoolchain/v5/bin/python3.13" if "python" not in self.expansions else self.expansions["python"] ) diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 4401aec4d6b..6c52c4ad4d6 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -2079,7 +2079,7 @@ class RunPlugin(PluginInterface): dest="fuzz_mongod_configs", help="Randomly chooses mongod parameters that were not specified.", metavar="MODE", - choices=("normal"), + choices=("normal",), ) mongodb_server_options.add_argument( diff --git a/buildscripts/resmokelib/testing/hooks/fuzz_runtime_parameters.py b/buildscripts/resmokelib/testing/hooks/fuzz_runtime_parameters.py index 8dc9c7ce750..261bae2b78f 100644 --- a/buildscripts/resmokelib/testing/hooks/fuzz_runtime_parameters.py +++ b/buildscripts/resmokelib/testing/hooks/fuzz_runtime_parameters.py @@ -12,6 +12,7 @@ from pymongo.errors import OperationFailure from buildscripts.resmokelib import config, errors from buildscripts.resmokelib.generate_fuzz_config.mongo_fuzzer_configs import ( generate_normal_mongo_parameters, + generate_runtime_mongod_parameter, ) from buildscripts.resmokelib.testing.fixtures import interface as fixture_interface from buildscripts.resmokelib.testing.fixtures import replicaset, shardedcluster, standalone @@ -39,7 +40,7 @@ class RuntimeParametersState: """Encapsulates the runtime-state of a set of parameters we are fuzzing. Tracks the last time we set a parameter value and holds the logic for generating new values.""" - def __init__(self, spec, seed): + def __init__(self, spec, seed, generator_func=None): # Initialize the runtime state of each parameter in the spec, including the lastSet time at now, so we start setting the parameters # at appropriate intervals after the suite begins. now = time.time() @@ -47,6 +48,10 @@ class RuntimeParametersState: key: {**copy.deepcopy(value), "lastSet": now} for key, value in spec.items() } self._rng = random.Random(seed) + # Use provided generator function, or default to generate_normal_mongo_parameters for backward compatibility + self._generator_func = ( + generator_func if generator_func is not None else generate_normal_mongo_parameters + ) def generate_parameters(self): """Returns a dictionary of what parameters should be set now, along with values to set them to, based on the last time the @@ -55,7 +60,7 @@ class RuntimeParametersState: now = time.time() for key, value in self._params.items(): if now - value["lastSet"] >= value["period"]: - ret[key] = generate_normal_mongo_parameters(self._rng, value) + ret[key] = self._generator_func(self._rng, value, key) value["lastSet"] = now return ret @@ -153,7 +158,10 @@ class FuzzRuntimeParameters(interface.Hook): validate_runtime_parameter_spec(cluster_params) # Construct the runtime state before the suite begins. # The initial lastSet time of each parameter is the start time of the suite. - self._mongod_param_state = RuntimeParametersState(runtime_mongod_params, self._seed) + # Use generate_runtime_mongod_parameter for mongod params to handle special cases. + self._mongod_param_state = RuntimeParametersState( + runtime_mongod_params, self._seed, generate_runtime_mongod_parameter + ) self._mongos_param_state = RuntimeParametersState(runtime_mongos_params, self._seed) self._cluster_param_state = RuntimeParametersState(cluster_params, self._seed) diff --git a/buildscripts/tests/resmoke_end2end/README.md b/buildscripts/tests/resmoke_end2end/README.md index d8da43abb5e..7fcd4d1f89b 100644 --- a/buildscripts/tests/resmoke_end2end/README.md +++ b/buildscripts/tests/resmoke_end2end/README.md @@ -1,11 +1,17 @@ +First, activate the virtual environment: + +``` +mongodb_repo_root$ source python3-venv/bin/activate +``` + - All end-to-end resmoke tests can be run via a resmoke suite itself: ``` -mongodb_repo_root$ /opt/mongodbtoolchain/v4/bin/python3 buildscripts/resmoke.py run --suites resmoke_end2end_tests +(python3-venv) mongodb_repo_root$ python buildscripts/resmoke.py run --suites resmoke_end2end_tests ``` - Finer grained control of tests can also be run with by invoking python's unittest main by hand. E.g: ``` -mongodb_repo_root$ /opt/mongodbtoolchain/v4/bin/python3 -m unittest -v buildscripts.tests.resmoke_end2end.test_resmoke.TestTestSelection.test_at_sign_as_replay_file +(python3-venv) mongodb_repo_root$ python -m unittest -v buildscripts.tests.resmoke_end2end.test_resmoke.TestTestSelection.test_at_sign_as_replay_file ``` diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py index 2279f6b1080..4e65529f7ed 100644 --- a/buildscripts/tests/resmoke_end2end/test_resmoke.py +++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py @@ -933,7 +933,7 @@ class TestCoreAnalyzerFunctions(unittest.TestCase): task_name = "test_tast_name" execution = "0" generated_task_name = get_generated_task_name(task_name, execution) - self.assertEquals(matches_generated_task_pattern(task_name, generated_task_name), execution) + self.assertEqual(matches_generated_task_pattern(task_name, generated_task_name), execution) self.assertIsNone(matches_generated_task_pattern("not_same_task", generated_task_name)) diff --git a/docs/devcontainer/troubleshooting.md b/docs/devcontainer/troubleshooting.md index 4776d509352..ada6ee55c0e 100644 --- a/docs/devcontainer/troubleshooting.md +++ b/docs/devcontainer/troubleshooting.md @@ -421,7 +421,7 @@ For additional VS Code-specific troubleshooting, see: ```bash rm -rf python3-venv - /opt/mongodbtoolchain/v5/bin/python3 -m venv python3-venv + /opt/mongodbtoolchain/v5/bin/python3.13 -m venv python3-venv source python3-venv/bin/activate poetry install --no-root --sync ``` diff --git a/etc/set_up_workstation.sh b/etc/set_up_workstation.sh index 7ac1f473a55..8123b42907f 100755 --- a/etc/set_up_workstation.sh +++ b/etc/set_up_workstation.sh @@ -56,7 +56,7 @@ setup_mongo_venv() { # PYTHON_KEYRING_BACKEND is needed to make poetry install work # See guide https://wiki.corp.mongodb.com/display/KERNEL/Virtual+Workstation export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring - /opt/mongodbtoolchain/v4/bin/python3 -m venv python3-venv + /opt/mongodbtoolchain/v5/bin/python3.13 -m venv python3-venv source ./python3-venv/bin/activate POETRY_VIRTUALENVS_IN_PROJECT=true poetry install --no-root --sync @@ -86,17 +86,17 @@ setup_pipx() { else export PATH="$PATH:$HOME/.local/bin" local venv_name="tmp-pipx-venv" - /opt/mongodbtoolchain/v4/bin/python3 -m venv $venv_name + /opt/mongodbtoolchain/v5/bin/python3.13 -m venv $venv_name # virtualenv doesn't like nounset set +o nounset source $venv_name/bin/activate set -o nounset - python -m pip install --upgrade "pip<20.3" + python -m pip --disable-pip-version-check install "pip==25.3" "wheel==0.45.1" python -m pip install pipx - pipx install pipx --python /opt/mongodbtoolchain/v4/bin/python3 --force + pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3.13 --force pipx ensurepath --force set +o nounset diff --git a/evergreen/prelude_python.sh b/evergreen/prelude_python.sh index 5df9675b9da..200fd7f0269 100644 --- a/evergreen/prelude_python.sh +++ b/evergreen/prelude_python.sh @@ -5,9 +5,16 @@ elif [ "$(uname)" = "Darwin" ]; then python='/Library/Frameworks/Python.Framework/Versions/3.13/bin/python3' echo "Executing on mac, setting python to ${python}" else - if [ -f /opt/mongodbtoolchain/v5/bin/python3 ]; then - python="/opt/mongodbtoolchain/v5/bin/python3" - echo "Found python in v5 toolchain, setting python to ${python}" + # Check if v5 toolchain exists - it requires Python 3.13 + if [ -d /opt/mongodbtoolchain/v5 ]; then + if [ -f /opt/mongodbtoolchain/v5/bin/python3.13 ]; then + python="/opt/mongodbtoolchain/v5/bin/python3.13" + echo "Found python 3.13 in v5 toolchain, setting python to ${python}" + else + echo "ERROR: v5 toolchain exists but Python 3.13 is not available at /opt/mongodbtoolchain/v5/bin/python3.13" + echo "The v5 toolchain requires Python 3.13. Please ensure python3.13 is installed in the toolchain." + return 1 + fi elif [ -f /opt/mongodbtoolchain/v4/bin/python3 ]; then python="/opt/mongodbtoolchain/v4/bin/python3" echo "Found python in v4 toolchain, setting python to ${python}" diff --git a/evergreen/selinux_test_setup.sh b/evergreen/selinux_test_setup.sh index ea82b141696..de5a1d53dae 100755 --- a/evergreen/selinux_test_setup.sh +++ b/evergreen/selinux_test_setup.sh @@ -31,7 +31,7 @@ if ! sudo --non-interactive rpm --install --verbose --verbose --hash --nodeps "$ fi # install packages needed by check_has_tag.py -PYTHON=/opt/mongodbtoolchain/v5/bin/python3 +PYTHON=/opt/mongodbtoolchain/v5/bin/python3.13 if [[ (-f "$PYTHON" || -L "$PYTHON") && -x "$PYTHON" ]]; then echo "==== Found python3 in $PYTHON" $PYTHON -m pip install pyyaml diff --git a/jstests/libs/python.js b/jstests/libs/python.js index 2281d583e92..117cd66372c 100644 --- a/jstests/libs/python.js +++ b/jstests/libs/python.js @@ -20,7 +20,7 @@ export function getPython3Binary() { } const paths = [ - "/opt/mongodbtoolchain/v5/bin/python3", + "/opt/mongodbtoolchain/v5/bin/python3.13", "/opt/mongodbtoolchain/v4/bin/python3", "/cygdrive/c/python/python313/python.exe", "c:/python/python313/python.exe", diff --git a/jstests/query_golden/expected_output/automaticCE/CBRForNoMultiplanningResults/plan_stability2 b/jstests/query_golden/expected_output/automaticCE/CBRForNoMultiplanningResults/plan_stability2 index 464dd18ce81..a80d92dbbd6 100644 --- a/jstests/query_golden/expected_output/automaticCE/CBRForNoMultiplanningResults/plan_stability2 +++ b/jstests/query_golden/expected_output/automaticCE/CBRForNoMultiplanningResults/plan_stability2 @@ -7,7 +7,7 @@ [jsTest] ---- -[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment. +[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment. [jsTest] ---- {">>>pipelines":[ diff --git a/jstests/query_golden/expected_output/automaticCE/HistogramCEWithHeuristicFallback/plan_stability2 b/jstests/query_golden/expected_output/automaticCE/HistogramCEWithHeuristicFallback/plan_stability2 index c586c5f6953..c5a51a576d0 100644 --- a/jstests/query_golden/expected_output/automaticCE/HistogramCEWithHeuristicFallback/plan_stability2 +++ b/jstests/query_golden/expected_output/automaticCE/HistogramCEWithHeuristicFallback/plan_stability2 @@ -7,7 +7,7 @@ [jsTest] ---- -[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment. +[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment. [jsTest] ---- {">>>pipelines":[ diff --git a/jstests/query_golden/expected_output/histogramCE/plan_stability2 b/jstests/query_golden/expected_output/histogramCE/plan_stability2 index 0e5b93d2eab..cfb95ca3cc2 100644 --- a/jstests/query_golden/expected_output/histogramCE/plan_stability2 +++ b/jstests/query_golden/expected_output/histogramCE/plan_stability2 @@ -7,7 +7,7 @@ [jsTest] ---- -[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment. +[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment. [jsTest] ---- {">>>pipelines":[ diff --git a/jstests/query_golden/expected_output/plan_stability2 b/jstests/query_golden/expected_output/plan_stability2 index d66834ee72c..8271dcf47a4 100644 --- a/jstests/query_golden/expected_output/plan_stability2 +++ b/jstests/query_golden/expected_output/plan_stability2 @@ -7,7 +7,7 @@ [jsTest] ---- -[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment. +[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment. [jsTest] ---- {">>>pipelines":[ diff --git a/jstests/query_golden/expected_output/samplingCE/plan_stability2 b/jstests/query_golden/expected_output/samplingCE/plan_stability2 index 79107bcbb8a..e1f048cb157 100644 --- a/jstests/query_golden/expected_output/samplingCE/plan_stability2 +++ b/jstests/query_golden/expected_output/samplingCE/plan_stability2 @@ -7,7 +7,7 @@ [jsTest] ---- -[jsTest] Found python 3.10 by default. Likely this is because we are using a virtual environment. +[jsTest] Found python 3.13 by default. Likely this is because we are using a virtual environment. [jsTest] ---- {">>>pipelines":[ diff --git a/jstests/sharding/libs/proxy_protocol_server.py b/jstests/sharding/libs/proxy_protocol_server.py index 43de9a2121e..254c2c4f624 100644 --- a/jstests/sharding/libs/proxy_protocol_server.py +++ b/jstests/sharding/libs/proxy_protocol_server.py @@ -8,13 +8,18 @@ The most recent source code is available on [GitHub][2]. The installed version is listed in the "[tool.poetry.group.testing.dependencies]" section of `pyproject.toml`. +This wrapper adds two patches: +1. Logs "Now listening on [...]" when the server is ready +2. Calls server.close_clients() during shutdown for Python 3.13 compatibility + [1]: https://pypi.org/project/proxy-protocol/ [2]: https://github.com/icgood/proxy-protocol/ """ import sys - from asyncio.base_events import BaseEventLoop + +import proxyprotocol.server.main from proxyprotocol.server.main import main # We want to know when the proxy protocol server is ready to accept connections; so, we log to @@ -23,16 +28,123 @@ from proxyprotocol.server.main import main # thing is to "monkey patch" the standard method # [asyncio.base_events.BaseEventLoop.create_server][1] so that it logs after the server is created. # +# Additionally, we store a reference to the server objects so they can be properly shut down. +# # [1]: https://github.com/python/cpython/blob/5c19c5bac6abf3da97d1d9b80cfa16e003897096/Lib/asyncio/base_events.py#L1429 -original_create_server = BaseEventLoop.create_server +_servers = [] +_original_create_server = BaseEventLoop.create_server -async def monkeypatched_create_server(self, protocol_factory, host, port, *args, **kwargs): - result = await original_create_server(self, protocol_factory, host, port, *args, **kwargs) - print(f"Now listening on {host}:{port}") - return result +async def _patched_create_server(self, protocol_factory, host, port, *args, **kwargs): + server = await _original_create_server(self, protocol_factory, host, port, *args, **kwargs) + _servers.append(server) + print(f"Now listening on {host}:{port}", flush=True) + return server +BaseEventLoop.create_server = _patched_create_server + +# Monkey-patch the library's run() function to add close_clients() call during shutdown. +# +# IMPORTANT: The code below is copied from proxyprotocol.server.main.run() v0.11.3 +# (see python3-venv/lib/python3.*/site-packages/proxyprotocol/server/main.py) +# with minimal modifications for Python 3.13 compatibility. +# +# CHANGES FROM LIBRARY SOURCE: +# 1. Added server.close_clients() call in handle_signal() before forever.cancel() +# - Python 3.13 added Server.close_clients() to forcibly close active connections +# - Python 3.12+ fixed wait_closed() to correctly wait for connections +# - Library's original code relied on Python 3.10/3.11 bug where wait_closed() +# returned immediately even with active connections +# 2. Added logging to track signal handling and shutdown progress +# +_original_run = proxyprotocol.server.main.run + + +async def _patched_run(args): + import asyncio + import signal + from asyncio import CancelledError + from contextlib import AsyncExitStack + from functools import partial + + from proxyprotocol.dnsbl import Dnsbl + from proxyprotocol.server import Address + from proxyprotocol.server.protocol import DownstreamProtocol, UpstreamProtocol + + # --- BEGIN: Code copied from library's run() function --- + loop = asyncio.get_running_loop() + + services = [(Address(source, server=True), Address(dest)) for (source, dest) in args.services] + buf_len = args.buf_len + dnsbl = Dnsbl.load(args.dnsbl, timeout=args.dnsbl_timeout) + new_server = partial(DownstreamProtocol, UpstreamProtocol, loop, buf_len, dnsbl) + + servers = [ + await loop.create_server( + partial(new_server, dest), source.host, source.port or 0, ssl=source.ssl + ) + for source, dest in services + ] + + async with AsyncExitStack() as stack: + for server in servers: + await stack.enter_async_context(server) + forever = asyncio.gather(*[server.serve_forever() for server in servers]) + + # --- BEGIN MODIFICATION: Added proper shutdown sequence --- + def handle_signal(): + print("Proxy server received shutdown signal", flush=True) + + # Step 1: Stop accepting new connections + try: + for server in servers: + server.close() + print("Proxy server: stopped accepting new connections", flush=True) + except Exception as e: + print(f"Proxy server: error in close(): {e}", flush=True) + + # Step 2: Close existing client connections gracefully + try: + for server in servers: + server.close_clients() + print("Proxy server: initiated graceful client close", flush=True) + except Exception as e: + print(f"Proxy server: error in close_clients(): {e}", flush=True) + + # Step 3: Schedule abort as a safety fallback after 1 second + def abort_remaining(): + print("Proxy server: aborting any remaining connections", flush=True) + try: + for server in servers: + server.abort_clients() + except Exception as e: + print(f"Proxy server: error in abort_clients(): {e}", flush=True) + + loop.call_later(1.0, abort_remaining) + + # CHANGE: Use proper shutdown sequence per Python 3.13 docs: + # 1. close() to stop accepting new connections + # 2. close_clients() to gracefully close existing connections + # 3. Schedule abort_clients() as fallback after timeout + forever.cancel() + + # --- END MODIFICATION --- + + loop.add_signal_handler(signal.SIGINT, handle_signal) + loop.add_signal_handler(signal.SIGTERM, handle_signal) + + try: + await forever + except CancelledError: + pass + # --- END: Code copied from library's run() function --- + + print("Proxy server shutdown complete", flush=True) + return 0 + + +proxyprotocol.server.main.run = _patched_run + if __name__ == "__main__": - BaseEventLoop.create_server = monkeypatched_create_server sys.exit(main())