"""Unit tests for the resmokelib.testing.executor module.""" import logging import threading import unittest import mock from buildscripts.resmokelib import errors from buildscripts.resmokelib.testing import job from buildscripts.resmokelib.testing import queue_element from buildscripts.resmokelib.utils import queue as _queue # pylint: disable=missing-docstring,protected-access class TestJob(unittest.TestCase): TESTS = ["jstests/core/and.js", "jstests/core/or.js"] @staticmethod def mock_testcase(test_name): testcase = mock.Mock() testcase.test_name = test_name testcase.REGISTERED_NAME = "js_test" testcase.logger = logging.getLogger("job_unittest") return testcase @staticmethod def mock_interrupt_flag(): interrupt_flag = mock.Mock() interrupt_flag.is_set = lambda: False return interrupt_flag @staticmethod def get_suite_options(num_repeat_tests=None, time_repeat_tests_secs=None, num_repeat_tests_min=None, num_repeat_tests_max=None): suite_options = mock.Mock() suite_options.num_repeat_tests = num_repeat_tests suite_options.time_repeat_tests_secs = time_repeat_tests_secs suite_options.num_repeat_tests_min = num_repeat_tests_min suite_options.num_repeat_tests_max = num_repeat_tests_max return suite_options @staticmethod def queue_tests(tests, queue, queue_elem_type, suite_options): for test in tests: queue_elem = queue_elem_type(TestJob.mock_testcase(test), {}, suite_options) queue.put(queue_elem) @staticmethod def expected_run_num(time_repeat_tests_secs, test_time_secs): """Return the number of times a test is expected to run.""" return time_repeat_tests_secs / test_time_secs def test__run_num_repeat(self): num_repeat_tests = 1 queue = _queue.Queue() suite_options = self.get_suite_options(num_repeat_tests=num_repeat_tests) mock_time = MockTime(1) job_object = UnitJob(suite_options) self.queue_tests(self.TESTS, queue, queue_element.QueueElem, suite_options) job_object._get_time = mock_time.time job_object._run(queue, self.mock_interrupt_flag()) self.assertEqual(job_object.total_test_num, num_repeat_tests * len(self.TESTS)) for test in self.TESTS: self.assertEqual(job_object.tests[test], num_repeat_tests) def test__run_time_repeat_time_no_min_max(self): increment = 1 time_repeat_tests_secs = 10 expected_tests_run = self.expected_run_num(time_repeat_tests_secs, increment) queue = _queue.Queue() suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs) mock_time = MockTime(increment) job_object = UnitJob(suite_options) self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options) job_object._get_time = mock_time.time job_object._run(queue, self.mock_interrupt_flag()) self.assertEqual(job_object.total_test_num, expected_tests_run * len(self.TESTS)) for test in self.TESTS: self.assertEqual(job_object.tests[test], expected_tests_run) def test__run_time_repeat_time_no_min(self): increment = 1 time_repeat_tests_secs = 10 num_repeat_tests_max = 100 expected_tests_run = self.expected_run_num(time_repeat_tests_secs, increment) queue = _queue.Queue() suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs, num_repeat_tests_max=num_repeat_tests_max) mock_time = MockTime(increment) job_object = UnitJob(suite_options) self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options) job_object._get_time = mock_time.time job_object._run(queue, self.mock_interrupt_flag()) self.assertLess(job_object.total_test_num, num_repeat_tests_max * len(self.TESTS)) for test in self.TESTS: self.assertEqual(job_object.tests[test], expected_tests_run) def test__run_time_repeat_time_no_max(self): increment = 1 time_repeat_tests_secs = 10 num_repeat_tests_min = 1 expected_tests_run = self.expected_run_num(time_repeat_tests_secs, increment) queue = _queue.Queue() suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs, num_repeat_tests_min=num_repeat_tests_min) mock_time = MockTime(increment) job_object = UnitJob(suite_options) self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options) job_object._get_time = mock_time.time job_object._run(queue, self.mock_interrupt_flag()) self.assertGreater(job_object.total_test_num, num_repeat_tests_min * len(self.TESTS)) for test in self.TESTS: self.assertEqual(job_object.tests[test], expected_tests_run) def test__run_time_repeat_time(self): increment = 1 time_repeat_tests_secs = 10 num_repeat_tests_min = 1 num_repeat_tests_max = 100 expected_tests_run = self.expected_run_num(time_repeat_tests_secs, increment) queue = _queue.Queue() suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs, num_repeat_tests_min=num_repeat_tests_min, num_repeat_tests_max=num_repeat_tests_max) mock_time = MockTime(increment) job_object = UnitJob(suite_options) self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options) job_object._get_time = mock_time.time job_object._run(queue, self.mock_interrupt_flag()) self.assertGreater(job_object.total_test_num, num_repeat_tests_min * len(self.TESTS)) self.assertLess(job_object.total_test_num, num_repeat_tests_max * len(self.TESTS)) for test in self.TESTS: self.assertEqual(job_object.tests[test], expected_tests_run) def test__run_time_repeat_min(self): increment = 1 time_repeat_tests_secs = 2 num_repeat_tests_min = 3 num_repeat_tests_max = 100 queue = _queue.Queue() suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs, num_repeat_tests_min=num_repeat_tests_min, num_repeat_tests_max=num_repeat_tests_max) mock_time = MockTime(increment) job_object = UnitJob(suite_options) self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options) job_object._get_time = mock_time.time job_object._run(queue, self.mock_interrupt_flag()) self.assertEqual(job_object.total_test_num, num_repeat_tests_min * len(self.TESTS)) for test in self.TESTS: self.assertEqual(job_object.tests[test], num_repeat_tests_min) def test__run_time_repeat_max(self): increment = 1 time_repeat_tests_secs = 30 num_repeat_tests_min = 1 num_repeat_tests_max = 10 expected_time_repeat_tests = self.expected_run_num(time_repeat_tests_secs, increment) queue = _queue.Queue() suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs, num_repeat_tests_min=num_repeat_tests_min, num_repeat_tests_max=num_repeat_tests_max) mock_time = MockTime(increment) job_object = UnitJob(suite_options) self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options) job_object._get_time = mock_time.time job_object._run(queue, self.mock_interrupt_flag()) self.assertEqual(job_object.total_test_num, num_repeat_tests_max * len(self.TESTS)) for test in self.TESTS: self.assertEqual(job_object.tests[test], num_repeat_tests_max) self.assertLess(job_object.tests[test], expected_time_repeat_tests) class MockTime(object): """Class to mock time.time.""" def __init__(self, increment): """Initialize with an increment which simulates a time increment.""" self._time = 0 self._increment = increment def time(self): """Simulate time.time by incrementing for every invocation.""" cur_time = self._time self._time += self._increment return cur_time class UnitJob(job.Job): # pylint: disable=too-many-instance-attributes def __init__(self, suite_options): #pylint: disable=super-init-not-called self.job_num = 0 self.logger = logging.getLogger("job_unittest") self.fixture = None self.hooks = [] self.report = None self.archival = None self.suite_options = suite_options self.test_queue_logger = logging.getLogger("job_unittest") self.total_test_num = 0 self.tests = {} def _execute_test(self, test): self.total_test_num += 1 if test.test_name not in self.tests: self.tests[test.test_name] = 0 self.tests[test.test_name] += 1 class TestFixtureSetupAndTeardown(unittest.TestCase): """Test cases for error handling around setup_fixture() and teardown_fixture().""" def setUp(self): logger = logging.getLogger("job_unittest") self.__job_object = job.Job(job_num=0, logger=logger, fixture=None, hooks=[], report=None, archival=None, suite_options=None, test_queue_logger=logger) # Initialize the Job instance such that its setup_fixture() and teardown_fixture() methods # always indicate success. The settings for these mocked method will be changed in the # individual test cases below. self.__job_object.setup_fixture = mock.Mock(return_value=True) self.__job_object.teardown_fixture = mock.Mock(return_value=True) def __assert_when_run_tests(self, setup_succeeded=True, teardown_succeeded=True): queue = _queue.Queue() interrupt_flag = threading.Event() setup_flag = threading.Event() teardown_flag = threading.Event() self.__job_object(queue, interrupt_flag, setup_flag, teardown_flag) self.assertEqual(setup_succeeded, not interrupt_flag.is_set()) self.assertEqual(teardown_succeeded, not teardown_flag.is_set()) def test_setup_and_teardown_both_succeed(self): self.__assert_when_run_tests() def test_setup_returns_failure(self): self.__job_object.setup_fixture.return_value = False self.__assert_when_run_tests(setup_succeeded=False) def test_setup_raises_logging_config_exception(self): self.__job_object.setup_fixture.side_effect = errors.LoggerRuntimeConfigError( "Logging configuration error intentionally raised in unit test") self.__assert_when_run_tests(setup_succeeded=False) def test_setup_raises_unexpected_exception(self): self.__job_object.setup_fixture.side_effect = Exception( "Generic error intentionally raised in unit test") self.__assert_when_run_tests(setup_succeeded=False) def test_teardown_returns_failure(self): self.__job_object.teardown_fixture.return_value = False self.__assert_when_run_tests(teardown_succeeded=False) def test_teardown_raises_logging_config_exception(self): self.__job_object.teardown_fixture.side_effect = errors.LoggerRuntimeConfigError( "Logging configuration error intentionally raised in unit test") self.__assert_when_run_tests(teardown_succeeded=False) def test_teardown_raises_unexpected_exception(self): self.__job_object.teardown_fixture.side_effect = Exception( "Generic error intentionally raised in unit test") self.__assert_when_run_tests(teardown_succeeded=False)