380 lines
20 KiB
Python
380 lines
20 KiB
Python
"""Standalone mongod fixture for executing JSTests against."""
|
|
|
|
import os
|
|
import os.path
|
|
import time
|
|
import shutil
|
|
import uuid
|
|
|
|
import yaml
|
|
|
|
import pymongo
|
|
import pymongo.errors
|
|
|
|
import buildscripts.resmokelib.testing.fixtures.interface as interface
|
|
|
|
|
|
class MongoDFixture(interface.Fixture):
|
|
"""Fixture which provides JSTests with a standalone mongod to run against."""
|
|
|
|
def __init__(self, logger, job_num, fixturelib, mongod_executable=None, mongod_options=None,
|
|
add_feature_flags=False, dbpath_prefix=None, preserve_dbpath=False, port=None):
|
|
"""Initialize MongoDFixture with different options for the mongod process."""
|
|
interface.Fixture.__init__(self, logger, job_num, fixturelib, dbpath_prefix=dbpath_prefix)
|
|
self.mongod_options = self.fixturelib.make_historic(
|
|
self.fixturelib.default_if_none(mongod_options, {}))
|
|
|
|
if "set_parameters" not in self.mongod_options:
|
|
self.mongod_options["set_parameters"] = {}
|
|
|
|
if add_feature_flags:
|
|
for ff in self.config.ENABLED_FEATURE_FLAGS:
|
|
self.mongod_options["set_parameters"][ff] = "true"
|
|
|
|
if "dbpath" in self.mongod_options and dbpath_prefix is not None:
|
|
raise ValueError("Cannot specify both mongod_options.dbpath and dbpath_prefix")
|
|
|
|
# Default to command line options if the YAML configuration is not passed in.
|
|
self.mongod_executable = self.fixturelib.default_if_none(mongod_executable,
|
|
self.config.MONGOD_EXECUTABLE)
|
|
|
|
# The dbpath in mongod_options takes precedence over other settings to make it easier for
|
|
# users to specify a dbpath containing data to test against.
|
|
if "dbpath" not in self.mongod_options:
|
|
self.mongod_options["dbpath"] = os.path.join(self._dbpath_prefix,
|
|
self.config.FIXTURE_SUBDIR)
|
|
self._dbpath = self.mongod_options["dbpath"]
|
|
|
|
if self.config.ALWAYS_USE_LOG_FILES:
|
|
self.mongod_options["logpath"] = self._dbpath + "/mongod.log"
|
|
self.mongod_options["logappend"] = ""
|
|
self.preserve_dbpath = True
|
|
else:
|
|
self.preserve_dbpath = preserve_dbpath
|
|
|
|
self.mongod = None
|
|
self.port = port or fixturelib.get_next_port(job_num)
|
|
self.mongod_options["port"] = self.port
|
|
|
|
# Always log backtraces to a file in the dbpath in our testing.
|
|
backtrace_log_file_name = os.path.join(self.get_dbpath_prefix(),
|
|
uuid.uuid4().hex + ".stacktrace")
|
|
self.mongod_options["set_parameters"]["backtraceLogFile"] = backtrace_log_file_name
|
|
|
|
def setup(self):
|
|
"""Set up the mongod."""
|
|
if not self.preserve_dbpath and os.path.lexists(self._dbpath):
|
|
shutil.rmtree(self._dbpath, ignore_errors=False)
|
|
|
|
os.makedirs(self._dbpath, exist_ok=True)
|
|
|
|
launcher = MongodLauncher(self.fixturelib)
|
|
# Second return val is the port, which we ignore because we explicitly created the port above.
|
|
# The port is used to set other mongod_option's here:
|
|
# https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136
|
|
mongod, _ = launcher.launch_mongod_program(self.logger, self.job_num,
|
|
executable=self.mongod_executable,
|
|
mongod_options=self.mongod_options)
|
|
try:
|
|
self.logger.info("Starting mongod on port %d...\n%s", self.port, mongod.as_command())
|
|
mongod.start()
|
|
self.logger.info("mongod started on port %d with pid %d.", self.port, mongod.pid)
|
|
except Exception as err:
|
|
msg = "Failed to start mongod on port {:d}: {}".format(self.port, err)
|
|
self.logger.exception(msg)
|
|
raise self.fixturelib.ServerFailure(msg)
|
|
|
|
self.mongod = mongod
|
|
|
|
def pids(self):
|
|
""":return: pids owned by this fixture if any."""
|
|
out = [x.pid for x in [self.mongod] if x is not None]
|
|
if not out:
|
|
self.logger.debug('Mongod not running when gathering standalone fixture pid.')
|
|
return out
|
|
|
|
def await_ready(self):
|
|
"""Block until the fixture can be used for testing."""
|
|
deadline = time.time() + MongoDFixture.AWAIT_READY_TIMEOUT_SECS
|
|
|
|
# Wait until the mongod is accepting connections. The retry logic is necessary to support
|
|
# versions of PyMongo <3.0 that immediately raise a ConnectionFailure if a connection cannot
|
|
# be established.
|
|
while True:
|
|
# Check whether the mongod exited for some reason.
|
|
exit_code = self.mongod.poll()
|
|
if exit_code is not None:
|
|
raise self.fixturelib.ServerFailure(
|
|
"Could not connect to mongod on port {}, process ended"
|
|
" unexpectedly with code {}.".format(self.port, exit_code))
|
|
|
|
try:
|
|
# Use a shorter connection timeout to more closely satisfy the requested deadline.
|
|
client = self.mongo_client(timeout_millis=500)
|
|
client.admin.command("ping")
|
|
break
|
|
except pymongo.errors.ConnectionFailure:
|
|
remaining = deadline - time.time()
|
|
if remaining <= 0.0:
|
|
raise self.fixturelib.ServerFailure(
|
|
"Failed to connect to mongod on port {} after {} seconds".format(
|
|
self.port, MongoDFixture.AWAIT_READY_TIMEOUT_SECS))
|
|
|
|
self.logger.info("Waiting to connect to mongod on port %d.", self.port)
|
|
time.sleep(0.1) # Wait a little bit before trying again.
|
|
|
|
self.logger.info("Successfully contacted the mongod on port %d.", self.port)
|
|
|
|
def _do_teardown(self, mode=None):
|
|
if self.mongod is None:
|
|
self.logger.warning("The mongod fixture has not been set up yet.")
|
|
return # Still a success even if nothing is running.
|
|
|
|
if mode == interface.TeardownMode.ABORT:
|
|
self.logger.info(
|
|
"Attempting to send SIGABRT from resmoke to mongod on port %d with pid %d...",
|
|
self.port, self.mongod.pid)
|
|
else:
|
|
self.logger.info("Stopping mongod on port %d with pid %d...", self.port,
|
|
self.mongod.pid)
|
|
if not self.is_running():
|
|
exit_code = self.mongod.poll()
|
|
msg = ("mongod on port {:d} was expected to be running, but wasn't. "
|
|
"Process exited with code {:d}.").format(self.port, exit_code)
|
|
self.logger.warning(msg)
|
|
raise self.fixturelib.ServerFailure(msg)
|
|
|
|
self.mongod.stop(mode)
|
|
exit_code = self.mongod.wait()
|
|
|
|
# Python's subprocess module returns negative versions of system calls.
|
|
if exit_code == 0 or (mode is not None and exit_code == -(mode.value)):
|
|
self.logger.info("Successfully stopped the mongod on port {:d}.".format(self.port))
|
|
else:
|
|
self.logger.warning("Stopped the mongod on port {:d}. "
|
|
"Process exited with code {:d}.".format(self.port, exit_code))
|
|
raise self.fixturelib.ServerFailure(
|
|
"mongod on port {:d} with pid {:d} exited with code {:d}".format(
|
|
self.port, self.mongod.pid, exit_code))
|
|
|
|
def is_running(self):
|
|
"""Return true if the mongod is still operating."""
|
|
return self.mongod is not None and self.mongod.poll() is None
|
|
|
|
def get_dbpath_prefix(self):
|
|
"""Return the _dbpath, as this is the root of the data directory."""
|
|
return self._dbpath
|
|
|
|
def get_node_info(self):
|
|
"""Return a list of NodeInfo objects."""
|
|
if self.mongod is None:
|
|
self.logger.warning("The mongod fixture has not been set up yet.")
|
|
return []
|
|
|
|
info = interface.NodeInfo(full_name=self.logger.full_name, name=self.logger.name,
|
|
port=self.port, pid=self.mongod.pid)
|
|
return [info]
|
|
|
|
def get_internal_connection_string(self):
|
|
"""Return the internal connection string."""
|
|
return "localhost:%d" % self.port
|
|
|
|
def get_driver_connection_url(self):
|
|
"""Return the driver connection URL."""
|
|
return "mongodb://" + self.get_internal_connection_string() + "/?directConnection=true"
|
|
|
|
|
|
# The below parameters define the default 'logComponentVerbosity' object passed to mongod processes
|
|
# started either directly via resmoke or those that will get started by the mongo shell. We allow
|
|
# this default to be different for tests run locally and tests run in Evergreen. This allows us, for
|
|
# example, to keep log verbosity high in Evergreen test runs without polluting the logs for
|
|
# developers running local tests.
|
|
|
|
# The default verbosity setting for any tests that are not started with an Evergreen task id. This
|
|
# will apply to any tests run locally.
|
|
DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY = {
|
|
"replication": {"rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2},
|
|
"transaction": 4, "tenantMigration": 4
|
|
}
|
|
|
|
# The default verbosity setting for any mongod processes running in Evergreen i.e. started with an
|
|
# Evergreen task id.
|
|
DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY = {
|
|
"replication": {"election": 4, "heartbeats": 2, "initialSync": 2,
|
|
"rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2},
|
|
"storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4
|
|
}
|
|
|
|
|
|
class MongodLauncher(object):
|
|
"""Class with utilities for launching a mongod."""
|
|
|
|
def __init__(self, fixturelib):
|
|
"""Initialize MongodLauncher."""
|
|
self.fixturelib = fixturelib
|
|
self.config = fixturelib.get_config()
|
|
|
|
def launch_mongod_program(self, logger, job_num, executable=None, process_kwargs=None,
|
|
mongod_options=None):
|
|
"""
|
|
Return a Process instance that starts mongod arguments constructed from 'mongod_options'.
|
|
|
|
@param logger - The logger to pass into the process.
|
|
@param executable - The mongod executable to run.
|
|
@param process_kwargs - A dict of key-value pairs to pass to the process.
|
|
@param mongod_options - A HistoryDict describing the various options to pass to the mongod.
|
|
"""
|
|
executable = self.fixturelib.default_if_none(executable,
|
|
self.config.DEFAULT_MONGOD_EXECUTABLE)
|
|
mongod_options = self.fixturelib.default_if_none(mongod_options, {}).copy()
|
|
|
|
# Apply the --setParameter command line argument. Command line options to resmoke.py override
|
|
# the YAML configuration.
|
|
# We leave the parameters attached for now so the top-level dict tracks its history.
|
|
suite_set_parameters = mongod_options.setdefault("set_parameters", {})
|
|
|
|
if self.config.MONGOD_SET_PARAMETERS is not None:
|
|
suite_set_parameters.update(yaml.safe_load(self.config.MONGOD_SET_PARAMETERS))
|
|
|
|
# Set default log verbosity levels if none were specified.
|
|
if "logComponentVerbosity" not in suite_set_parameters:
|
|
suite_set_parameters[
|
|
"logComponentVerbosity"] = self.get_default_log_component_verbosity_for_mongod()
|
|
|
|
# minNumChunksForSessionsCollection controls the minimum number of chunks the balancer will
|
|
# enforce for the sessions collection. If the actual number of chunks is less, the balancer will
|
|
# issue split commands to create more chunks. As a result, the balancer will also end up moving
|
|
# chunks for the sessions collection to balance the chunks across shards. Unless the suite is
|
|
# explicitly prepared to handle these background migrations, set the parameter to 1.
|
|
if "configsvr" in mongod_options and "minNumChunksForSessionsCollection" not in suite_set_parameters:
|
|
suite_set_parameters["minNumChunksForSessionsCollection"] = 1
|
|
|
|
# orphanCleanupDelaySecs controls an artificial delay before cleaning up an orphaned chunk
|
|
# that has migrated off of a shard, meant to allow most dependent queries on secondaries to
|
|
# complete first. It defaults to 900, or 15 minutes, which is prohibitively long for tests.
|
|
# Setting it in the .yml file overrides this.
|
|
if (("shardsvr" in mongod_options or "configsvr" in mongod_options)
|
|
and "orphanCleanupDelaySecs" not in suite_set_parameters):
|
|
suite_set_parameters["orphanCleanupDelaySecs"] = 1
|
|
|
|
# The LogicalSessionCache does automatic background refreshes in the server. This is
|
|
# race-y for tests, since tests trigger their own immediate refreshes instead. Turn off
|
|
# background refreshing for tests. Set in the .yml file to override this.
|
|
if "disableLogicalSessionCacheRefresh" not in suite_set_parameters:
|
|
suite_set_parameters["disableLogicalSessionCacheRefresh"] = True
|
|
|
|
# Set coordinateCommitReturnImmediatelyAfterPersistingDecision to false so that tests do
|
|
# not need to rely on causal consistency or explicitly wait for the transaction to finish
|
|
# committing.
|
|
if "coordinateCommitReturnImmediatelyAfterPersistingDecision" not in suite_set_parameters:
|
|
suite_set_parameters["coordinateCommitReturnImmediatelyAfterPersistingDecision"] = False
|
|
|
|
# There's a periodic background thread that checks for and aborts expired transactions.
|
|
# "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring
|
|
# and being aborted by the background thread. It defaults to 60 seconds, which is too short to
|
|
# be reliable for our tests. Setting it to 24 hours, so that it is longer than the Evergreen
|
|
# execution timeout.
|
|
if "transactionLifetimeLimitSeconds" not in suite_set_parameters:
|
|
suite_set_parameters["transactionLifetimeLimitSeconds"] = 24 * 60 * 60
|
|
|
|
# Hybrid index builds drain writes received during the build process in batches of 1000 writes
|
|
# by default. Not all tests perform enough writes to exercise the code path where multiple
|
|
# batches are applied, which means certain bugs are harder to encounter. Set this level lower
|
|
# so there are more opportunities to drain writes in multiple batches.
|
|
if "maxIndexBuildDrainBatchSize" not in suite_set_parameters:
|
|
suite_set_parameters["maxIndexBuildDrainBatchSize"] = 10
|
|
|
|
# The periodic no-op writer writes an oplog entry of type='n' once every 10 seconds. This has
|
|
# the potential to mask issues such as SERVER-31609 because it allows the operationTime of
|
|
# cluster to advance even if the client is blocked for other reasons. We should disable the
|
|
# periodic no-op writer. Set in the .yml file to override this.
|
|
if "replSet" in mongod_options and "writePeriodicNoops" not in suite_set_parameters:
|
|
suite_set_parameters["writePeriodicNoops"] = False
|
|
|
|
# The default time for stepdown and quiesce mode in response to SIGTERM is 15 seconds. Reduce
|
|
# this to 100ms for faster shutdown. On branches 4.4 and earlier, there is no quiesce mode, but
|
|
# the default time for stepdown is 10 seconds.
|
|
if (("replSet" in mongod_options or "serverless" in mongod_options)
|
|
and "shutdownTimeoutMillisForSignaledShutdown" not in suite_set_parameters):
|
|
suite_set_parameters["shutdownTimeoutMillisForSignaledShutdown"] = 100
|
|
|
|
if "enableFlowControl" not in suite_set_parameters and self.config.FLOW_CONTROL is not None:
|
|
suite_set_parameters["enableFlowControl"] = (self.config.FLOW_CONTROL == "on")
|
|
|
|
if ("failpoint.flowControlTicketOverride" not in suite_set_parameters
|
|
and self.config.FLOW_CONTROL_TICKETS is not None):
|
|
suite_set_parameters["failpoint.flowControlTicketOverride"] = {
|
|
"mode": "alwaysOn", "data": {"numTickets": self.config.FLOW_CONTROL_TICKETS}
|
|
}
|
|
|
|
_add_testing_set_parameters(suite_set_parameters)
|
|
|
|
shortcut_opts = {
|
|
"enableMajorityReadConcern": self.config.MAJORITY_READ_CONCERN,
|
|
"storageEngine": self.config.STORAGE_ENGINE,
|
|
"transportLayer": self.config.TRANSPORT_LAYER,
|
|
"wiredTigerCollectionConfigString": self.config.WT_COLL_CONFIG,
|
|
"wiredTigerEngineConfigString": self.config.WT_ENGINE_CONFIG,
|
|
"wiredTigerIndexConfigString": self.config.WT_INDEX_CONFIG,
|
|
}
|
|
|
|
if self.config.STORAGE_ENGINE == "inMemory":
|
|
shortcut_opts["inMemorySizeGB"] = self.config.STORAGE_ENGINE_CACHE_SIZE
|
|
elif self.config.STORAGE_ENGINE == "rocksdb":
|
|
shortcut_opts["rocksdbCacheSizeGB"] = self.config.STORAGE_ENGINE_CACHE_SIZE
|
|
elif self.config.STORAGE_ENGINE == "wiredTiger" or self.config.STORAGE_ENGINE is None:
|
|
shortcut_opts["wiredTigerCacheSizeGB"] = self.config.STORAGE_ENGINE_CACHE_SIZE
|
|
|
|
# These options are just flags, so they should not take a value.
|
|
opts_without_vals = ("logappend")
|
|
|
|
# Ensure that config servers run with journaling enabled.
|
|
if "configsvr" in mongod_options:
|
|
suite_set_parameters.setdefault("reshardingMinimumOperationDurationMillis", 5000)
|
|
suite_set_parameters.setdefault("reshardingCriticalSectionTimeoutMillis",
|
|
24 * 60 * 60) # 24 hours
|
|
|
|
# Command line options override the YAML configuration.
|
|
for opt_name in shortcut_opts:
|
|
opt_value = shortcut_opts[opt_name]
|
|
if opt_name in opts_without_vals:
|
|
# Options that are specified as --flag on the command line are represented by a boolean
|
|
# value where True indicates that the flag should be included in 'kwargs'.
|
|
if opt_value:
|
|
mongod_options[opt_name] = ""
|
|
else:
|
|
# Options that are specified as --key=value on the command line are represented by a
|
|
# value where None indicates that the key-value pair shouldn't be included in 'kwargs'.
|
|
if opt_value is not None:
|
|
mongod_options[opt_name] = opt_value
|
|
|
|
# Override the storage engine specified on the command line with "wiredTiger" if running a
|
|
# config server replica set.
|
|
if "replSet" in mongod_options and "configsvr" in mongod_options:
|
|
mongod_options["storageEngine"] = "wiredTiger"
|
|
|
|
return self.fixturelib.mongod_program(logger, job_num, executable, process_kwargs,
|
|
mongod_options)
|
|
|
|
def get_default_log_component_verbosity_for_mongod(self):
|
|
"""Return the default 'logComponentVerbosity' value to use for mongod processes."""
|
|
if self.config.EVERGREEN_TASK_ID:
|
|
return DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY
|
|
return DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY
|
|
|
|
|
|
def _add_testing_set_parameters(suite_set_parameters):
|
|
"""
|
|
Add certain behaviors should only be enabled for resmoke usage.
|
|
|
|
These are traditionally enable new commands, insecure access, and increased diagnostics.
|
|
"""
|
|
suite_set_parameters.setdefault("testingDiagnosticsEnabled", True)
|
|
suite_set_parameters.setdefault("enableTestCommands", True)
|
|
suite_set_parameters.setdefault("requireConfirmInSetFcv", False)
|
|
# The exact file location is on a per-process basis, so it'll have to be determined when the process gets spun up.
|
|
# Set it to true for now as a placeholder that will error if no further processing is done.
|
|
# The placeholder is needed so older versions don't have this option won't have this value set.
|
|
suite_set_parameters.setdefault("backtraceLogFile", True)
|
|
suite_set_parameters.setdefault("disableTransitionFromLatestToLastContinuous", False)
|