SERVER-65672 upgrade python packages to support 3.10

.pydocstyle

@@ -9,6 +9,10 @@ inherit = false
 # D212 - Multi-line docstring summary should start at the first line
 # D213 - Multi-line docstring summary should start at the second line
 # D301 - Use r""" if any backslashes in a docstring
-ignore = D105,D202,D203,D212,D213,D301
+# D401 - First line should be in imperative mood
+# D407 - Missing dashed underline after section
+# D413 - Missing blank line after last section
+# D415 - First line should end with a period, question mark, or exclamation point
+ignore = D105,D202,D203,D212,D213,D301,D401,D407,D413,D415
 # Do not run on buildscripts/tests/
 match = ^((?!buildscripts[\\\/]tests[\\\/]).)*$

buildscripts/collect_resource_info.py

@@ -18,7 +18,7 @@ from buildscripts.resmokelib import utils  # pylint: disable=wrong-import-positi
 
 
 def main():
-    """Main."""
+    """Collect system resources."""
     usage = "usage: %prog [options]"
     parser = optparse.OptionParser(description=__doc__, usage=usage)
     parser.add_option(

buildscripts/errorcodes.py

@@ -239,14 +239,15 @@ def read_error_codes(src_root='src/mongo'):
 
 
 def replace_bad_codes(errors, next_code_generator):  # pylint: disable=too-many-locals
-    """Modify C++ source files to replace invalid assertion codes.
+    """
+    Modify C++ source files to replace invalid assertion codes.
 
     For now, we only modify zero codes.
 
-    Args:
-        errors: list of AssertLocation
-        next_code_generator: generator -> int, next non-conflicting assertion code
+    :param errors: list of AssertLocation
+    :param next_code_generator: generator -> int, next non-conflicting assertion code
     """
+
     zero_errors = [e for e in errors if int(e.code) == 0]
     skip_errors = [e for e in errors if int(e.code) != 0]
 
@@ -296,7 +297,7 @@ def coerce_to_number(ticket_value):
 
 
 def main():
-    """Main."""
+    """Validate error codes."""
     parser = OptionParser(description=__doc__.strip())
     parser.add_option("--fix", dest="replace", action="store_true", default=False,
                       help="Fix zero codes in source files [default: %default]")

buildscripts/gdb/mongo_printers.py

@@ -579,6 +579,7 @@ def find_match_brackets(search, opening='<', closing='>'):
     Example:
         'Foo<T>::iterator<U>''
         returns 5
+
     """
     index = search.find(opening)
     if index == -1:

buildscripts/idl/idl/errors.py

@@ -37,7 +37,6 @@ import inspect
 import os
 import sys
 from typing import List, Union
-from yaml import nodes
 import yaml
 
 from . import common

buildscripts/linter/git_base.py

@@ -236,6 +236,7 @@ class GitException(Exception):
         process_args: a list containing the git command and arguments (includes 'git' as its first
             element) that were run, if any.
         stderr: the error output of the git command.
+
     """
 
     def __init__(  # pylint: disable=too-many-arguments
@@ -258,6 +259,7 @@ class GitCommandResult(object):
         returncode: the return code.
         stdout: the output of the command.
         stderr: the error output of the command.
+
     """
 
     def __init__(  # pylint: disable=too-many-arguments

buildscripts/linter/parallel.py

@@ -9,7 +9,11 @@ from typing import Any, Callable, List
 
 def parallel_process(items, func):
     # type: (List[Any], Callable[[Any], bool]) -> bool
-    """Run a set of work items to completion and wait."""
+    """
+    Run a set of work items to completion and wait.
+
+    :returns whether all tasks were successful.
+    """
     try:
         cpus = cpu_count()
     except NotImplementedError:
@@ -17,20 +21,16 @@ def parallel_process(items, func):
 
     task_queue = queue.Queue()  # type: queue.Queue
 
-    # Use a list so that worker function will capture this variable
-    pp_event = threading.Event()
-    pp_result = [True]
-    pp_lock = threading.Lock()
+    has_failure_event = threading.Event()
 
     def worker():
         # type: () -> None
         """Worker thread to process work items in parallel."""
-        while not pp_event.is_set():
+        while True:
             try:
                 item = task_queue.get_nowait()
             except queue.Empty:
                 # if the queue is empty, exit the worker thread
-                pp_event.set()
                 return
 
             try:
@@ -39,13 +39,8 @@ def parallel_process(items, func):
                 # Tell the queue we finished with the item
                 task_queue.task_done()
 
-            # Return early if we fail, and signal we are done
             if not ret:
-                with pp_lock:
-                    pp_result[0] = False
-
-                pp_event.set()
-                return
+                has_failure_event.set()
 
     # Enqueue all the work we want to process
     for item in items:
@@ -62,10 +57,10 @@ def parallel_process(items, func):
 
     # Wait for the threads to finish
     # Loop with a timeout so that we can process Ctrl-C interrupts
-    while not pp_event.wait(1):
+    while not queue.Empty:
         time.sleep(1)
 
     for thread in threads:
         thread.join()
 
-    return pp_result[0]
+    return not has_failure_event.is_set()

buildscripts/linter/pydocstyle.py

@@ -11,7 +11,7 @@ class PyDocstyleLinter(base.LinterBase):
     def __init__(self):
         # type: () -> None
         """Create a pydocstyle linter."""
-        super(PyDocstyleLinter, self).__init__("pydocstyle", "2.1.1")
+        super(PyDocstyleLinter, self).__init__("pydocstyle", "6.1.1")
 
     def get_lint_version_cmd_args(self):
         # type: () -> List[str]

buildscripts/resmokelib/logging/buildlogger.py

@@ -27,7 +27,10 @@ _INCOMPLETE_LOG_OUTPUT = threading.Event()
 
 
 def is_log_output_incomplete():  # noqa: D205,D400
-    """Return true if we failed to write all of the log output to the buildlogger server, and return
+    """
+    Indicate whether the log output is incomplete.
+
+    Return true if we failed to write all of the log output to the buildlogger server, and return
     false otherwise.
     """
     return _INCOMPLETE_LOG_OUTPUT.is_set()

buildscripts/resmokelib/selector.py

@@ -422,6 +422,7 @@ class _Selector(object):
 
         Args:
             test_file_explorer: a TestFileExplorer instance.
+            tests_are_files: whether tests are files.
         """
         self._test_file_explorer = test_file_explorer
         self._tests_are_files = tests_are_files

buildscripts/resmokelib/testing/hooks/background_job.py

@@ -60,8 +60,10 @@ class _BackgroundJob(threading.Thread):  # pylint: disable=too-many-instance-att
         self.join()
 
     def pause(self):  # noqa: D205,D400
-        """Signal the background thread that it should stop executing 'self._hook_test_case', and
-        wait until the current execution has finished.
+        """
+        Signal the background thread that it should stop executing 'self._hook_test_case'.
+
+        Wait until the current execution has finished.
         """
         self._hook_test_case.signal_stop_test()
         with self._lock:

buildscripts/resmokelib/testing/hooks/stepdown.py

@@ -46,6 +46,9 @@ class ContinuousStepdown(interface.Hook):  # pylint: disable=too-many-instance-a
             use_stepdown_permitted_file: use a file to control if stepdown thread should do a stepdown.
             wait_for_mongos_retarget: whether to run validate on all mongoses for each collection
                 in each database, after pausing the stepdown thread.
+            auth_options: dictionary of auth options.
+            background_reconfig: whether to conduct reconfig in the background.
+            should_downgrade: whether dowgrades should be performed as part of the stepdown.
 
         Note that the "terminate" and "kill" arguments are named after the "SIGTERM" and
         "SIGKILL" signals that are used to stop the process. On Windows, there are no signals,

etc/pip/components/core.req

@@ -2,4 +2,5 @@
 psutil <= 5.8.0
 pymongo >= 3.9, < 4.0
 PyYAML >= 3.0.0, <= 6.0.0
+types-PyYAML ~= 6.0.5
 requests >= 2.0.0, <= 2.26.0

etc/pip/components/jiraclient.req

@@ -2,5 +2,4 @@
 jira <= 3.1.1
 requests-oauth <= 0.4.1
 PyJWT <= 2.3.0 # https://github.com/pycontribs/jira/issues/247
-cryptography == 2.3 # Needed for oauthlib to use RSAAlgorithm # Version locked - see SERVER-36618
 # We are omitting pycrypto based on https://github.com/pycontribs/jira/pull/629

etc/pip/components/lint.req

@@ -1,8 +1,8 @@
 # Linters
 # Note: These versions are checked by python modules in buildscripts/linter/
 GitPython ~= 3.1.7
-mypy < 0.900; python_version > "3.5"
-pydocstyle == 2.1.1
+mypy ~= 0.942
+pydocstyle == 6.1.1
 pylint == 2.7.2
 structlog ~= 19.2.0
 typing <= 3.7.4.3

etc/pip/components/platform.req

@@ -1,3 +1,5 @@
 # Platform-specific components
 pypiwin32>=223; sys_platform == "win32" and python_version > "3"
 pywin32>=225; sys_platform == "win32" and python_version > "3"
+
+cryptography == 2.3; platform_machine == "s390x" or platform_machine == "ppc64le" # Needed for oauthlib to use RSAAlgorithm # Version locked - see SERVER-36618