246 lines
9.3 KiB
Python
Executable File
246 lines
9.3 KiB
Python
Executable File
#!/usr/bin/env python
|
|
"""Command line utility for executing MongoDB tests of all kinds."""
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import os.path
|
|
import random
|
|
import sys
|
|
import time
|
|
|
|
# Get relative imports to work when the package is not installed on the PYTHONPATH.
|
|
if __name__ == "__main__" and __package__ is None:
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
# pylint: disable=wrong-import-position
|
|
from buildscripts.resmokelib import config
|
|
from buildscripts.resmokelib import errors
|
|
from buildscripts.resmokelib import logging
|
|
from buildscripts.resmokelib import parser
|
|
from buildscripts.resmokelib import reportfile
|
|
from buildscripts.resmokelib import sighandler
|
|
from buildscripts.resmokelib import suitesconfig
|
|
from buildscripts.resmokelib import testing
|
|
from buildscripts.resmokelib import utils
|
|
|
|
|
|
class Resmoke(object):
|
|
"""The main class to run tests with resmoke."""
|
|
|
|
def __init__(self):
|
|
"""Initialize the Resmoke instance."""
|
|
self.__start_time = time.time()
|
|
self._config = None
|
|
self._exec_logger = None
|
|
self._resmoke_logger = None
|
|
self._archive = None
|
|
self._interrupted = False
|
|
|
|
def configure_from_command_line(self):
|
|
"""Configure this instance using the command line arguments."""
|
|
self._config = parser.parse_command_line()
|
|
|
|
def _setup_logging(self):
|
|
logging.loggers.configure_loggers(self._config.logging_config)
|
|
logging.flush.start_thread()
|
|
self._exec_logger = logging.loggers.EXECUTOR_LOGGER
|
|
self._resmoke_logger = self._exec_logger.new_resmoke_logger()
|
|
|
|
def run(self):
|
|
"""Run resmoke."""
|
|
if self._config is None:
|
|
raise RuntimeError("Resmoke must be configured before calling run()")
|
|
self._setup_logging()
|
|
|
|
try:
|
|
if self._config.list_suites:
|
|
self.list_suites()
|
|
elif self._config.find_suites:
|
|
self.find_suites()
|
|
elif self._config.dry_run == "tests":
|
|
self.dry_run()
|
|
else:
|
|
self.run_tests()
|
|
finally:
|
|
if not self._interrupted:
|
|
logging.flush.stop_thread()
|
|
|
|
def list_suites(self):
|
|
"""List the suites that are available to execute."""
|
|
suite_names = suitesconfig.get_named_suites()
|
|
self._resmoke_logger.info("Suites available to execute:\n%s", "\n".join(suite_names))
|
|
|
|
def find_suites(self):
|
|
"""List the suites that run the specified tests."""
|
|
suites = self._get_suites()
|
|
suites_by_test = self._find_suites_by_test(suites)
|
|
for test in sorted(suites_by_test):
|
|
suite_names = suites_by_test[test]
|
|
self._resmoke_logger.info("%s will be run by the following suite(s): %s", test,
|
|
suite_names)
|
|
|
|
@staticmethod
|
|
def _find_suites_by_test(suites):
|
|
"""
|
|
Look up what other resmoke suites run the tests specified in the suites parameter.
|
|
|
|
Return a dict keyed by test name, value is array of suite names.
|
|
"""
|
|
memberships = {}
|
|
test_membership = suitesconfig.create_test_membership_map()
|
|
for suite in suites:
|
|
for test in suite.tests:
|
|
memberships[test] = test_membership[test]
|
|
return memberships
|
|
|
|
def dry_run(self):
|
|
"""List which tests would run and which tests would be excluded in a resmoke invocation."""
|
|
suites = self._get_suites()
|
|
for suite in suites:
|
|
self._shuffle_tests(suite)
|
|
sb = ["Tests that would be run in suite {}".format(suite.get_display_name())]
|
|
sb.extend(suite.tests or ["(no tests)"])
|
|
sb.append("Tests that would be excluded from suite {}".format(suite.get_display_name()))
|
|
sb.extend(suite.excluded or ["(no tests)"])
|
|
self._exec_logger.info("\n".join(sb))
|
|
|
|
def run_tests(self):
|
|
"""Run the suite and tests specified."""
|
|
self._resmoke_logger.info("resmoke.py invocation: %s", " ".join(sys.argv))
|
|
suites = None
|
|
try:
|
|
suites = self._get_suites()
|
|
|
|
self._setup_archival()
|
|
self._setup_signal_handler(suites)
|
|
|
|
for suite in suites:
|
|
self._interrupted = self._run_suite(suite)
|
|
if self._interrupted or (suite.options.fail_fast and suite.return_code != 0):
|
|
self._log_resmoke_summary(suites)
|
|
self.exit(suite.return_code)
|
|
|
|
self._log_resmoke_summary(suites)
|
|
|
|
# Exit with a nonzero code if any of the suites failed.
|
|
exit_code = max(suite.return_code for suite in suites)
|
|
self.exit(exit_code)
|
|
finally:
|
|
if not self._interrupted:
|
|
self._exit_archival()
|
|
if suites:
|
|
reportfile.write(suites)
|
|
|
|
def _run_suite(self, suite):
|
|
"""Run a test suite."""
|
|
self._log_suite_config(suite)
|
|
suite.record_suite_start()
|
|
interrupted = self._execute_suite(suite)
|
|
suite.record_suite_end()
|
|
self._log_suite_summary(suite)
|
|
return interrupted
|
|
|
|
def _log_resmoke_summary(self, suites):
|
|
"""Log a summary of the resmoke run."""
|
|
time_taken = time.time() - self.__start_time
|
|
if len(self._config.suite_files) > 1:
|
|
testing.suite.Suite.log_summaries(self._resmoke_logger, suites, time_taken)
|
|
|
|
def _log_suite_summary(self, suite):
|
|
"""Log a summary of the suite run."""
|
|
self._resmoke_logger.info("=" * 80)
|
|
self._resmoke_logger.info("Summary of %s suite: %s", suite.get_display_name(),
|
|
self._get_suite_summary(suite))
|
|
|
|
def _execute_suite(self, suite):
|
|
"""Execute a suite and return True if interrupted, False otherwise. """
|
|
self._shuffle_tests(suite)
|
|
if not suite.tests:
|
|
self._exec_logger.info("Skipping %s, no tests to run", suite.test_kind)
|
|
suite.return_code = 0
|
|
return False
|
|
executor_config = suite.get_executor_config()
|
|
executor = testing.executor.TestSuiteExecutor(
|
|
self._exec_logger, suite, archive_instance=self._archive, **executor_config)
|
|
try:
|
|
executor.run()
|
|
except errors.UserInterrupt:
|
|
suite.return_code = 130 # Simulate SIGINT as exit code.
|
|
return True
|
|
except IOError:
|
|
suite.return_code = 74 # Exit code for IOError on POSIX systems.
|
|
return True
|
|
except: # pylint: disable=bare-except
|
|
self._exec_logger.exception("Encountered an error when running %ss of suite %s.",
|
|
suite.test_kind, suite.get_display_name())
|
|
suite.return_code = 2
|
|
return False
|
|
return False
|
|
|
|
def _shuffle_tests(self, suite):
|
|
"""Shuffle the tests if the shuffle cli option was set."""
|
|
random.seed(config.RANDOM_SEED)
|
|
if not config.SHUFFLE:
|
|
return
|
|
self._exec_logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.",
|
|
suite.test_kind, suite.get_display_name(), config.RANDOM_SEED)
|
|
random.shuffle(suite.tests)
|
|
|
|
def _get_suites(self):
|
|
"""Return the list of suites for this resmoke invocation."""
|
|
try:
|
|
return suitesconfig.get_suites(self._config.suite_files, self._config.test_files)
|
|
except errors.SuiteNotFound as err:
|
|
self._resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err))
|
|
self.list_suites()
|
|
self.exit(1)
|
|
|
|
def _log_suite_config(self, suite):
|
|
sb = [
|
|
"YAML configuration of suite {}".format(suite.get_display_name()),
|
|
utils.dump_yaml({"test_kind": suite.get_test_kind_config()}), "",
|
|
utils.dump_yaml({"selector": suite.get_selector_config()}), "",
|
|
utils.dump_yaml({"executor": suite.get_executor_config()}), "",
|
|
utils.dump_yaml({"logging": self._config.logging_config})
|
|
]
|
|
self._resmoke_logger.info("\n".join(sb))
|
|
|
|
@staticmethod
|
|
def _get_suite_summary(suite):
|
|
"""Return a summary of the suite run."""
|
|
sb = []
|
|
suite.summarize(sb)
|
|
return "\n".join(sb)
|
|
|
|
def _setup_signal_handler(self, suites):
|
|
"""Set up a SIGUSR1 signal handler that logs a test result summary and a thread dump."""
|
|
sighandler.register(self._resmoke_logger, suites, self.__start_time)
|
|
|
|
def _setup_archival(self):
|
|
"""Set up the archival feature if enabled in the cli options."""
|
|
if config.ARCHIVE_FILE:
|
|
self._archive = utils.archival.Archival(
|
|
archival_json_file=config.ARCHIVE_FILE, limit_size_mb=config.ARCHIVE_LIMIT_MB,
|
|
limit_files=config.ARCHIVE_LIMIT_TESTS, logger=self._exec_logger)
|
|
|
|
def _exit_archival(self):
|
|
"""Finish up archival tasks before exit if enabled in the cli options."""
|
|
if self._archive:
|
|
self._archive.exit()
|
|
|
|
def exit(self, exit_code):
|
|
"""Exit with the provided exit code."""
|
|
self._resmoke_logger.info("Exiting with code: %d", exit_code)
|
|
sys.exit(exit_code)
|
|
|
|
|
|
def main():
|
|
"""Main function for resmoke."""
|
|
resmoke = Resmoke()
|
|
resmoke.configure_from_command_line()
|
|
resmoke.run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|