SERVER-72262 Bump Python minimum version to 3.9/3.10

.mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.7
+python_version = 3.10
 
 disallow_untyped_defs = False
 # Do not error on imported files since all imported files may not be mypy clean.

SConstruct

@@ -38,7 +38,9 @@ import mongo.toolchain as mongo_toolchain
 import mongo.generators as mongo_generators
 import mongo.install_actions as install_actions
 
-EnsurePythonVersion(3, 6)
+# TODO SERVER-79172
+# We cannot set the limit to python 3.10 since python 3.9 is needed for windows testing
+EnsurePythonVersion(3, 9)
 EnsureSConsVersion(3, 1, 1)
 
 utc_starttime = datetime.utcnow()

buildscripts/ciconfig/evergreen.py

@@ -6,11 +6,11 @@ configuration file.
 from __future__ import annotations
 
 import datetime
-import distutils.spawn
+import shutil
 import os
 import subprocess
+import structlog
 import sys
-import time
 from typing import Set, List, Optional
 
 import yaml
@@ -20,20 +20,46 @@ from buildscripts.util import runcommand
 ENTERPRISE_MODULE_NAME = "enterprise"
 ASAN_SIGNATURE = "detect_leaks=1"
 
+LOGGER = structlog.get_logger(__name__)
+
 
 def parse_evergreen_file(path, evergreen_binary="evergreen"):
     """Read an Evergreen file and return EvergreenProjectConfig instance."""
     if evergreen_binary:
-        if not distutils.spawn.find_executable(evergreen_binary):
+        print(f"os.environ={os.environ}")
+        if not shutil.which(evergreen_binary):
+            # On windows in python3.8 there was an update to no longer use HOME in os.path.expanduser
+            # However, cygwin is weird and has HOME but not USERPROFILE
+            # So we just check if HOME is set and USERPROFILE is not
+            # Then we just set USERPROFILE and unset it after
+            # Bug is here: https://bugs.python.org/issue36264
+
+            prev_environ = os.environ.copy()
+            if sys.platform in ("win32", "cygwin"):
+                LOGGER.info(f"Previous os.environ={os.environ} before updating 'USERPROFILE'")
+                if 'HOME' in os.environ:
+                    os.environ['USERPROFILE'] = os.environ['HOME']
+                else:
+                    LOGGER.warn(
+                        "'HOME' enviorment variable unset. This will likely cause us to be unable to find evergreen binary."
+                    )
+
             default_evergreen_location = os.path.expanduser(os.path.join("~", "evergreen"))
+
+            # Restore enviorment if it was modified above on windows
+            os.environ.clear()
+            os.environ.update(prev_environ)
+
             if os.path.exists(default_evergreen_location):
                 evergreen_binary = default_evergreen_location
             elif os.path.exists(f"{default_evergreen_location}.exe"):
                 evergreen_binary = f"{default_evergreen_location}.exe"
             else:
                 raise EnvironmentError(
-                    "Executable '{}' does not exist or is not in the PATH.".format(
+                    f"Executable {evergreen_binary} (default location: {default_evergreen_location}) does not exist or is not in the PATH. PATH={os.environ.get('PATH')}"
-                        evergreen_binary))
+                )
+        else:
+            evergreen_binary = shutil.which(evergreen_binary)
 
         # Call 'evergreen evaluate path' to pre-process the project configuration file.
         cmd = [evergreen_binary, "evaluate", path]

buildscripts/linter/runner.py

@@ -82,7 +82,7 @@ def _find_linter(linter, config_dict):
     python_dir = os.path.dirname(sys.executable)
     if sys.platform == "win32":
         # On Windows, these scripts are installed in %PYTHONDIR%\scripts like
-        # 'C:\Python37\scripts', and have .exe extensions.
+        # 'C:\python\python310\scripts', and have .exe extensions.
         python_dir = os.path.join(python_dir, "scripts")
 
         cmd_str = os.path.join(python_dir, linter.cmd_name)

buildscripts/resmokelib/utils/archival.py

@@ -126,6 +126,29 @@ class Archival(object):
     def _get_s3_client():
         # Since boto3 is a 3rd party module, we import locally.
         import boto3
+        import botocore.session
+        botocore.session.Session()
+
+        if sys.platform in ("win32", "cygwin"):
+            # These overriden values can be found here
+            # https://github.com/boto/botocore/blob/13468bc9d8923eccd0816ce2dd9cd8de5a6f6e0e/botocore/configprovider.py#L49C7-L49C7
+            # This is due to the backwards breaking changed python introduced https://bugs.python.org/issue36264
+            botocore_session = botocore.session.Session(
+                session_vars={
+                    'config_file': (
+                        None,
+                        'AWS_CONFIG_FILE',
+                        os.path.join(os.environ['HOME'], '.aws', 'config'),
+                        None,
+                    ),
+                    'credentials_file': (
+                        None,
+                        'AWS_SHARED_CREDENTIALS_FILE',
+                        os.path.join(os.environ['HOME'], '.aws', 'credentials'),
+                        None,
+                    ),
+                })
+            boto3.setup_default_session(botocore_session=botocore_session)
         return boto3.client("s3")
 
     def archive_files_to_s3(self, display_name, input_files, s3_bucket, s3_path):

buildscripts/tests/ciconfig/test_evergreen.py

@@ -14,8 +14,12 @@ class TestEvergreenProjectConfig(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        env = os.environ.copy()
         cls.conf = _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None)
 
+        # Assert there is no leakage of env variables from this function
+        assert env == os.environ
+
     def test_invalid_path(self):
         invalid_path = "non_existing_file"
         with self.assertRaises(IOError):

docs/building.md

@@ -19,7 +19,7 @@ To build MongoDB, you will need:
       * `libcurl4-gnutls-dev`
     * On Ubuntu, the lzma library is required. Install `liblzma-dev`
     * On Amazon Linux, the xz-devel library is required. `yum install xz-devel`
-* Python 3.7.x and Pip modules:
+* Python 3.9.x and Pip modules:
   * See the section "Python Prerequisites" below.
 * About 13 GB of free disk space for the core binaries (`mongod`,
   `mongos`, and `mongo`) and about 600 GB for the install-all target.
@@ -43,7 +43,7 @@ The source for the tools is now available at
 Python Prerequisites
 ---------------
 
-In order to build MongoDB, Python 3.7+ is required, and several Python
+In order to build MongoDB, Python 3.9+ is required, and several Python
 modules must be installed. Python 3 is included in macOS 10.15 and later.
 For earlier macOS versions, Python 3 can be installed using Homebrew or
 MacPorts or similar.
@@ -131,7 +131,7 @@ Windows
 
 Build requirements:
 * Visual Studio 2022 version 17.0 or newer
-* Python 3.7
+* Python 3.9
 
 Or download a prebuilt binary for Windows at www.mongodb.org.
 

docs/golden_data_test_framework.md

@@ -180,7 +180,7 @@ buildscripts/golden_test.py setup
 Run buildscripts/golden_test.py setup utility.
 You may be asked for a password, when not running in "Run as administrator" shell.
 ```cmd
-c:\python\Python37\python.exe buildscripts/golden_test.py setup
+c:\python\python310\python.exe buildscripts/golden_test.py setup
 ```
 
 ### Manual Setup (Default config)

etc/evergreen.yml

@@ -168,7 +168,7 @@ variables:
       -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 ))
       --win-version-min=win10
     num_scons_link_jobs_available: 0.5
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     ext: zip
     scons_cache_scope: shared
     multiversion_platform: windows
@@ -551,7 +551,7 @@ buildvariants:
       -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.8")
       MONGO_DISTMOD=windows
     num_scons_link_jobs_available: 0.2
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     ext: zip
     scons_cache_scope: shared
     multiversion_platform: windows
@@ -631,7 +631,7 @@ buildvariants:
     burn_in_tests_build_variant: enterprise-windows-all-feature-flags-required
     exe: ".exe"
     content_type: application/zip
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     ext: zip
     multiversion_platform: windows
     multiversion_edition: enterprise
@@ -684,7 +684,7 @@ buildvariants:
     burn_in_tests_build_variant: enterprise-windows-all-feature-flags-required
     exe: ".exe"
     content_type: application/zip
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     ext: zip
     multiversion_platform: windows
     multiversion_edition: enterprise
@@ -773,7 +773,7 @@ buildvariants:
       -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5")
       --win-version-min=win10
     num_scons_link_jobs_available: 0.25
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     ext: zip
     resmoke_jobs_max: 1
     scons_cache_scope: shared
@@ -1890,7 +1890,7 @@ buildvariants:
       LIBPATH="c:/sasl/lib"
       -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 ))
       --win-version-min=win10
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
   tasks:
   - name: build_metrics_gen_TG
 
@@ -2908,7 +2908,7 @@ buildvariants:
   - windows-vsCurrent-small
   stepback: false
   expansions:
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
   tasks:
   - name: win_shared_scons_cache_pruning
 
@@ -2975,7 +2975,7 @@ buildvariants:
       LIBPATH="c:/sasl/lib"
       -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 ))
       --win-version-min=win10
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     ext: zip
     has_packages: false
     scons_cache_scope: shared

etc/evergreen_yml_components/variants/compile_static_analysis.yml

@@ -205,7 +205,7 @@ buildvariants:
       -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.8")
       --win-version-min=win10
     num_scons_link_jobs_available: 0.2
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     scons_cache_scope: shared
     compile_variant: *windows-compile-required
   tasks:

etc/evergreen_yml_components/variants/in_memory.yml

@@ -95,7 +95,7 @@ buildvariants:
       LIBPATH="c:/sasl/lib" -j$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) / 1.5")
       --win-version-min=win10
     num_scons_link_jobs_available: 0.25
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     test_flags: >-
       --storageEngine=inMemory
       --excludeWithAnyTags=requires_persistence,requires_journaling,incompatible_with_windows_tls

etc/evergreen_yml_components/variants/misc_release.yml

@@ -2514,7 +2514,7 @@ buildvariants:
       --win-version-min=win10
       --use-diagnostic-latches=off
     num_scons_link_jobs_available: 0.25
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     ext: zip
     scons_cache_scope: shared
     large_distro_name: windows-vsCurrent-large
@@ -2580,7 +2580,7 @@ buildvariants:
       --win-version-min=win10
       --use-diagnostic-latches=off
     num_scons_link_jobs_available: 0.25
-    python: '/cygdrive/c/python/python37/python.exe'
+    python: '/cygdrive/c/python/python39/python.exe'
     ext: zip
     scons_cache_scope: shared
     multiversion_platform: windows

etc/pip/components/external_auth.req

@@ -1,5 +1,5 @@
 # These are the dependencies of ldaptor
-passlib == 1.7.1
+passlib == 1.7.4
 pyOpenSSL == 19.0.0; platform_machine == "s390x" or platform_machine == "ppc64le" # Needed for pinned cryptography package - see SERVER-70845
 pyOpenSSL == 22.0.0; platform_machine != "s390x" and platform_machine != "ppc64le"
 pyparsing == 2.4.0

evergreen/prelude_python.sh

@@ -1,5 +1,5 @@
 if [ "Windows_NT" = "$OS" ]; then
-  python='/cygdrive/c/python/python37/python.exe'
+  python='/cygdrive/c/python/python39/python.exe'
 else
   if [ -f /opt/mongodbtoolchain/v4/bin/python3 ]; then
     python="/opt/mongodbtoolchain/v4/bin/python3"

jstests/free_mon/libs/free_mon.js

@@ -3,6 +3,8 @@
  * Control the Free Monitoring Mock Webserver.
  */
 
+load("jstests/libs/python.js");
+
 // These faults must match the list of faults in mock_http_server.py, see the
 // SUPPORTED_FAULT_TYPES list in mock_http_server.py
 const FAULT_FAIL_REGISTER = "fail_register";
@@ -23,14 +25,10 @@ class FreeMonWebServer {
      * @param {bool} disableFaultsOnStartup optionally disable fault on startup
      */
     constructor(fault_type, disableFaultsOnStartup) {
-        this.python = "python3";
+        this.python = getPython3Binary();
         this.disableFaultsOnStartup = disableFaultsOnStartup || false;
         this.fault_type = fault_type;
 
-        if (_isWindows()) {
-            this.python = "python.exe";
-        }
-
         print("Using python interpreter: " + this.python);
         this.web_server_py = "jstests/free_mon/libs/mock_http_server.py";
         this.control_py = "jstests/free_mon/libs/mock_http_control.py";

jstests/libs/python.js

@@ -3,25 +3,41 @@
 function getPython3Binary() {
     'use strict';
 
-    let cmd = '/opt/mongodbtoolchain/v4/bin/python3';
+    // On windows it is important to use python vs python3
-    if (_isWindows()) {
+    // or else we will pick up a python that is not in our venv
-        const paths = ["c:/python36/python.exe", "c:/python/python36/python.exe"];
+    clearRawMongoProgramOutput();
-        for (let p of paths) {
+    assert.eq(runNonMongoProgram("python", "--version"), 0);
-            if (fileExists(p)) {
+    const pythonVersion = rawMongoProgramOutput();  // Will look like "Python 3.10.4\n"
-                cmd = p;
+    const usingPython39 = /Python 3\.9/.exec(pythonVersion);
-                break;
+    const usingPython310 = /Python 3\.10/.exec(pythonVersion);
-            }
+    if (usingPython310) {
+        print(
+            "Found python 3.10 by default. Likely this is because we are using a virtual enviorment.");
+        return "python";
+    } else if (usingPython39) {
+        // TODO: SERVER-79172
+        // Once the above ticket is complete we should stop using python 3.9 on windows and upgrade
+        // to python 310 everywhere To solve: grep for python39 and fix instances of it
+        print(
+            "Found python 3.9 by default. Likely this is because we are using a windows virtual enviorment.");
+        return "python";
+    }
+
+    const paths = [
+        "/opt/mongodbtoolchain/v4/bin/python3",
+        "/cygdrive/c/python/python310/python.exe",
+        "c:/python/python310/python.exe"
+    ];
+    for (let p of paths) {
+        if (fileExists(p)) {
+            print("Found python3 in default location " + p);
+            return p;
         }
     }
 
-    if (fileExists(cmd)) {
-        return cmd;
-    }
-
-    clearRawMongoProgramOutput();
-    assert.eq(runNonMongoProgram("python", "--version"), 0);
-    const pythonVersion = rawMongoProgramOutput();
     assert(/Python 3/.exec(pythonVersion));
 
-    return "python";
+    // We are probs running on mac
+    print("Did not find python3 in a virtualenv or default location");
+    return "python3";
 }

jstests/noPassthrough/libs/backup_restore.js

@@ -19,6 +19,8 @@
  *   }
  */
 
+load("jstests/libs/python.js");
+
 var BackupRestoreTest = function(options) {
     "use strict";
 
@@ -123,7 +125,8 @@ var BackupRestoreTest = function(options) {
     function _fsmClient(host) {
         // Launch FSM client
         const suite = 'concurrency_replication_for_backup_restore';
-        const resmokeCmd = 'python buildscripts/resmoke.py run --shuffle --continueOnFailure' +
+        const resmokeCmd = getPython3Binary() +
+            ' buildscripts/resmoke.py run --shuffle --continueOnFailure' +
             ' --repeat=99999 --internalParam=is_inner_level --mongo=' +
             MongoRunner.getMongoShellPath() + ' --shellConnString=mongodb://' + host +
             ' --suites=' + suite;

jstests/noPassthrough/libs/configExpand/lib.js

@@ -88,9 +88,6 @@ function makeReflectionCmd(arg, opts = {}) {
 
         load('jstests/libs/python.js');
         let cmd = getPython3Binary();
-        if (_isWindows()) {
-            cmd = '"' + cmd + '"';
-        }
         cmd += ' jstests/noPassthrough/libs/configExpand/reflect.py';
 
         if (opts.sleep && (opts.sleep > 0)) {

jstests/ocsp/lib/mock_ocsp.js

@@ -3,6 +3,7 @@
  * OCSP certificate revocation.
  */
 load("jstests/ocsp/lib/ocsp_helpers.js");
+load("jstests/libs/python.js");
 
 // These are a list of faults to match the list of faults
 // in ocsp_mock.py.
@@ -46,13 +47,9 @@ class MockOCSPServer {
                 response_delay_secs = 0,
                 include_extraneous_status = false,
                 issuer_hash_algorithm = "") {
-        this.python = "python3";
+        this.python = getPython3Binary();
         this.fault_type = fault_type;
 
-        if (_isWindows()) {
-            this.python = "python.exe";
-        }
-
         this.ca_file = responder_certificate_set.cafile;
         this.ocsp_cert_file = responder_certificate_set.certfile;
         this.ocsp_cert_key = responder_certificate_set.keyfile;

jstests/resmoke_selftest/end2end/timeout/nested/top_level_timeout.js

@@ -1,6 +1,8 @@
 (function() {
 'use strict';
 
+load("jstests/libs/python.js");
+
 const rst = new ReplSetTest({nodes: 2});
 rst.startSet();
 rst.initiate();
@@ -10,7 +12,7 @@ function start() {
     // because when we pass --taskId into resmoke it thinks that it is being ran in evergreen
     // and cannot normally find an evergreen task associated with
     // buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_nested_timeout.yml
-    const resmokeCmd = 'python3 buildscripts/resmoke.py run ' +
+    const resmokeCmd = getPython3Binary() + ' buildscripts/resmoke.py run ' +
         '--storageEngineCacheSizeGB=1 --dbpathPrefix=/data/db/selftest_inner ' +
         '--internalParam=test_archival --taskId=123 ' +
         '--originSuite=resmoke_end2end_tests ' +

jstests/sharding/libs/proxy_protocol.js

@@ -1,16 +1,14 @@
 /**
  * Control the proxy protocol server.
  */
+
+load("jstests/libs/python.js");
 class ProxyProtocolServer {
     /**
      * Create a new proxy protocol server.
      */
     constructor(ingress_port, egress_port, version) {
-        this.python = "python3";
+        this.python = getPython3Binary();
-
-        if (_isWindows()) {
-            this.python = "python.exe";
-        }
 
         print("Using python interpreter: " + this.python);
         this.web_server_py = "jstests/sharding/libs/proxy_protocol_server.py";

jstests/ssl/ssl_ECDHE_suites.js

@@ -1,6 +1,7 @@
 // Test that the server supports ECDHE and DHE tls cipher suites.
 
 load('jstests/ssl/libs/ssl_helpers.js');
+load("jstests/libs/python.js");
 
 (function() {
 "use strict";
@@ -35,11 +36,7 @@ const x509_options = {
 const mongod = MongoRunner.runMongod(x509_options);
 
 // Use new toolchain python, if it exists
-let python_binary = '/opt/mongodbtoolchain/v4/bin/python3';
+let python_binary = getPython3Binary();
-if (runProgram('/bin/sh', '-c', 'ls ' + python_binary) !== 0) {
-    // We are running on mac
-    python_binary = 'python3';
-}
 
 // Run the tls cipher suite enumerator
 const python = '/usr/bin/env ' + python_binary;

jstests/watchdog/lib/charybdefs_lib.js

@@ -6,10 +6,13 @@ const EXIT_WATCHDOG = 61;
  *
  * @param {string} test_name unique name for test directories
  */
+
+load("jstests/libs/python.js");
+
 function CharybdefsControl(test_name) {
     'use strict';
 
-    const python = "python";
+    const python = getPython3Binary();
     let control_py = "/data/charybdefs/mongo/control.py";
 
     // Use the minimum watchdog period

src/mongo/shell/servers.js

@@ -1,3 +1,5 @@
+load("jstests/libs/python.js");
+
 var MongoRunner, _startMongod, startMongoProgram, runMongoProgram, startMongoProgramNoConnect,
     myPort;
 
@@ -169,8 +171,18 @@ function runHangAnalyzer(pids) {
     print(`Running hang analyzer for pids [${pids}]`);
 
     const scriptPath = pathJoin('.', 'buildscripts', 'resmoke.py');
-    const args =
+    const args = [
-        ['python', scriptPath, 'hang-analyzer', '-k', '-o', 'file', '-o', 'stdout', '-d', pids];
+        getPython3Binary(),
+        scriptPath,
+        'hang-analyzer',
+        '-k',
+        '-o',
+        'file',
+        '-o',
+        'stdout',
+        '-d',
+        pids
+    ];
 
     if (jsTest.options().evergreenTaskId) {
         args.push('-t', jsTest.options().evergreenTaskId);