366 lines
14 KiB
Python
366 lines
14 KiB
Python
"""Unit tests for buildscripts/burn_in_tests_multiversion.py."""
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import json
|
|
from datetime import datetime
|
|
import os
|
|
import sys
|
|
import unittest
|
|
from mock import MagicMock, patch
|
|
|
|
import inject
|
|
from shrub.v2 import BuildVariant, ShrubProject
|
|
from evergreen import EvergreenApi
|
|
|
|
import buildscripts.burn_in_tests_multiversion as under_test
|
|
from buildscripts.burn_in_tests import TaskInfo
|
|
from buildscripts.ciconfig.evergreen import parse_evergreen_file, EvergreenProjectConfig
|
|
import buildscripts.resmokelib.parser as _parser
|
|
import buildscripts.evergreen_gen_multiversion_tests as gen_multiversion
|
|
from buildscripts.evergreen_burn_in_tests import GenerateBurnInExecutor, EvergreenFileChangeDetector
|
|
from buildscripts.evergreen_generate_resmoke_tasks import GENERATE_CONFIG_FILE
|
|
from buildscripts.task_generation.gen_config import GenerationConfiguration
|
|
from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig
|
|
from buildscripts.task_generation.suite_split import SuiteSplitConfig
|
|
from buildscripts.task_generation.suite_split_strategies import greedy_division, SplitStrategy, \
|
|
FallbackStrategy, round_robin_fallback
|
|
from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions
|
|
|
|
_parser.set_run_options()
|
|
|
|
MONGO_4_2_HASH = "d94888c0d0a8065ca57d354ece33b3c2a1a5a6d6"
|
|
|
|
# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access,no-value-for-parameter
|
|
|
|
|
|
def create_tests_by_task_mock(n_tasks, n_tests, multiversion_values=None):
|
|
if multiversion_values is None:
|
|
multiversion_values = [None for _ in range(n_tasks)]
|
|
return {
|
|
f"task_{i}_gen": TaskInfo(
|
|
display_task_name=f"task_{i}",
|
|
resmoke_args=f"--suites=suite_{i}",
|
|
tests=[f"jstests/tests_{j}" for j in range(n_tests)],
|
|
require_multiversion=multiversion_values[i],
|
|
distro="",
|
|
)
|
|
for i in range(n_tasks)
|
|
}
|
|
|
|
|
|
MV_MOCK_SUITES = ["replica_sets_jscore_passthrough", "sharding_jscore_passthrough"]
|
|
MV_MOCK_TESTS = {
|
|
"replica_sets_jscore_passthrough": [
|
|
"core/all.js",
|
|
"core/andor.js",
|
|
"core/apitest_db.js",
|
|
"core/auth1.js",
|
|
"core/auth2.js",
|
|
], "sharding_jscore_passthrough": [
|
|
"core/basic8.js",
|
|
"core/batch_size.js",
|
|
"core/bson.js",
|
|
"core/bulk_insert.js",
|
|
"core/capped.js",
|
|
]
|
|
}
|
|
|
|
|
|
def create_multiversion_tests_by_task_mock(n_tasks, n_tests):
|
|
assert n_tasks <= len(MV_MOCK_SUITES)
|
|
assert n_tests <= len(MV_MOCK_TESTS[MV_MOCK_SUITES[0]])
|
|
return {
|
|
f"{MV_MOCK_SUITES[i]}": TaskInfo(
|
|
display_task_name=f"task_{i}",
|
|
resmoke_args=f"--suites=suite_{i}",
|
|
tests=[f"jstests/{MV_MOCK_TESTS[MV_MOCK_SUITES[i]][j]}" for j in range(n_tests)],
|
|
require_multiversion=None,
|
|
distro="",
|
|
)
|
|
for i in range(n_tasks)
|
|
}
|
|
|
|
|
|
_DATE = datetime(2018, 7, 15)
|
|
BURN_IN_TESTS = "buildscripts.burn_in_tests"
|
|
NUM_REPL_MIXED_VERSION_CONFIGS = len(gen_multiversion.REPL_MIXED_VERSION_CONFIGS)
|
|
NUM_SHARDED_MIXED_VERSION_CONFIGS = len(gen_multiversion.SHARDED_MIXED_VERSION_CONFIGS)
|
|
|
|
NS = "buildscripts.burn_in_tests_multiversion"
|
|
|
|
|
|
def ns(relative_name): # pylint: disable=invalid-name
|
|
"""Return a full name from a name relative to the test module"s name space."""
|
|
return NS + "." + relative_name
|
|
|
|
|
|
def bit_ns(relative_name):
|
|
return BURN_IN_TESTS + "." + relative_name
|
|
|
|
|
|
def mock_a_file(filename):
|
|
change = MagicMock(a_path=filename)
|
|
return change
|
|
|
|
|
|
def mock_git_diff(change_list):
|
|
diff = MagicMock()
|
|
diff.iter_change_type.return_value = change_list
|
|
return diff
|
|
|
|
|
|
def mock_changed_git_files(add_files):
|
|
repo = MagicMock()
|
|
repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files])
|
|
return repo
|
|
|
|
|
|
def get_evergreen_config(config_file_path):
|
|
evergreen_home = os.path.expanduser(os.path.join("~", "evergreen"))
|
|
if os.path.exists(evergreen_home):
|
|
return parse_evergreen_file(config_file_path, evergreen_home)
|
|
return parse_evergreen_file(config_file_path)
|
|
|
|
|
|
def create_variant_task_mock(task_name, suite_name, distro="distro"):
|
|
variant_task = MagicMock()
|
|
variant_task.name = task_name
|
|
variant_task.generated_task_name = task_name
|
|
variant_task.resmoke_suite = suite_name
|
|
variant_task.get_vars_suite_name.return_value = suite_name
|
|
variant_task.combined_resmoke_args = f"--suites={suite_name}"
|
|
variant_task.require_multiversion = None
|
|
variant_task.run_on = [distro]
|
|
return variant_task
|
|
|
|
|
|
def build_mock_gen_task_options():
|
|
return GenTaskOptions(
|
|
create_misc_suite=False,
|
|
is_patch=True,
|
|
generated_config_dir=under_test.DEFAULT_CONFIG_DIR,
|
|
use_default_timeouts=False,
|
|
)
|
|
|
|
|
|
def build_mock_split_task_config():
|
|
return SuiteSplitConfig(
|
|
evg_project="my project",
|
|
target_resmoke_time=60,
|
|
max_sub_suites=100,
|
|
max_tests_per_suite=1,
|
|
start_date=datetime.utcnow(),
|
|
end_date=datetime.utcnow(),
|
|
default_to_fallback=True,
|
|
)
|
|
|
|
|
|
def configure_dependencies(evg_api, split_config):
|
|
gen_task_options = build_mock_gen_task_options()
|
|
|
|
def dependencies(binder: inject.Binder) -> None:
|
|
binder.bind(SuiteSplitConfig, split_config)
|
|
binder.bind(SplitStrategy, greedy_division)
|
|
binder.bind(FallbackStrategy, round_robin_fallback)
|
|
binder.bind(GenTaskOptions, gen_task_options)
|
|
binder.bind(EvergreenApi, evg_api)
|
|
binder.bind(GenerationConfiguration,
|
|
GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE))
|
|
binder.bind(ResmokeProxyConfig,
|
|
ResmokeProxyConfig(resmoke_suite_dir=under_test.DEFAULT_TEST_SUITE_DIR))
|
|
binder.bind(EvergreenFileChangeDetector, None)
|
|
binder.bind(EvergreenProjectConfig, MagicMock())
|
|
binder.bind(
|
|
under_test.BurnInConfig,
|
|
under_test.BurnInConfig(build_id="build_id", build_variant="build variant",
|
|
revision="revision"))
|
|
|
|
inject.clear_and_configure(dependencies)
|
|
|
|
|
|
class TestCreateMultiversionGenerateTasksConfig(unittest.TestCase):
|
|
def tests_no_tasks_given(self):
|
|
target_file = "target_file.json"
|
|
mock_evg_api = MagicMock()
|
|
split_config = build_mock_split_task_config()
|
|
configure_dependencies(mock_evg_api, split_config)
|
|
|
|
orchestrator = under_test.MultiversionBurnInOrchestrator()
|
|
generated_config = orchestrator.generate_configuration({}, target_file, "build_variant")
|
|
|
|
evg_config = [
|
|
config for config in generated_config.file_list if config.file_name == target_file
|
|
]
|
|
self.assertEqual(1, len(evg_config))
|
|
evg_config = evg_config[0]
|
|
evg_config_dict = json.loads(evg_config.content)
|
|
|
|
self.assertEqual(0, len(evg_config_dict["tasks"]))
|
|
|
|
def test_tasks_not_in_multiversion_suites(self):
|
|
n_tasks = 1
|
|
n_tests = 1
|
|
target_file = "target_file.json"
|
|
mock_evg_api = MagicMock()
|
|
split_config = build_mock_split_task_config()
|
|
configure_dependencies(mock_evg_api, split_config)
|
|
tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
|
|
|
|
orchestrator = under_test.MultiversionBurnInOrchestrator()
|
|
generated_config = orchestrator.generate_configuration(tests_by_task, target_file,
|
|
"build_variant")
|
|
|
|
evg_config = [
|
|
config for config in generated_config.file_list if config.file_name == target_file
|
|
]
|
|
self.assertEqual(1, len(evg_config))
|
|
evg_config = evg_config[0]
|
|
evg_config_dict = json.loads(evg_config.content)
|
|
|
|
self.assertEqual(0, len(evg_config_dict["tasks"]))
|
|
|
|
@unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
|
|
@patch(
|
|
"buildscripts.evergreen_gen_multiversion_tests.get_backports_required_hash_for_shell_version"
|
|
)
|
|
def test_one_task_one_test(self, mock_hash):
|
|
mock_hash.return_value = MONGO_4_2_HASH
|
|
n_tasks = 1
|
|
n_tests = 1
|
|
target_file = "target_file.json"
|
|
mock_evg_api = MagicMock()
|
|
split_config = build_mock_split_task_config()
|
|
configure_dependencies(mock_evg_api, split_config)
|
|
tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests)
|
|
|
|
orchestrator = under_test.MultiversionBurnInOrchestrator()
|
|
generated_config = orchestrator.generate_configuration(tests_by_task, target_file,
|
|
"build_variant")
|
|
|
|
evg_config = [
|
|
config for config in generated_config.file_list if config.file_name == target_file
|
|
]
|
|
self.assertEqual(1, len(evg_config))
|
|
evg_config = evg_config[0]
|
|
evg_config_dict = json.loads(evg_config.content)
|
|
tasks = evg_config_dict["tasks"]
|
|
self.assertEqual(len(tasks), NUM_REPL_MIXED_VERSION_CONFIGS * n_tests)
|
|
|
|
@unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
|
|
@patch(
|
|
"buildscripts.evergreen_gen_multiversion_tests.get_backports_required_hash_for_shell_version"
|
|
)
|
|
def test_n_task_one_test(self, mock_hash):
|
|
mock_hash.return_value = MONGO_4_2_HASH
|
|
n_tasks = 2
|
|
n_tests = 1
|
|
target_file = "target_file.json"
|
|
mock_evg_api = MagicMock()
|
|
split_config = build_mock_split_task_config()
|
|
configure_dependencies(mock_evg_api, split_config)
|
|
tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests)
|
|
|
|
orchestrator = under_test.MultiversionBurnInOrchestrator()
|
|
generated_config = orchestrator.generate_configuration(tests_by_task, target_file,
|
|
"build_variant")
|
|
|
|
evg_config = [
|
|
config for config in generated_config.file_list if config.file_name == target_file
|
|
]
|
|
self.assertEqual(1, len(evg_config))
|
|
evg_config = evg_config[0]
|
|
evg_config_dict = json.loads(evg_config.content)
|
|
tasks = evg_config_dict["tasks"]
|
|
self.assertEqual(
|
|
len(tasks),
|
|
(NUM_REPL_MIXED_VERSION_CONFIGS + NUM_SHARDED_MIXED_VERSION_CONFIGS) * n_tests)
|
|
|
|
@unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
|
|
@patch(
|
|
"buildscripts.evergreen_gen_multiversion_tests.get_backports_required_hash_for_shell_version"
|
|
)
|
|
def test_one_task_n_test(self, mock_hash):
|
|
mock_hash.return_value = MONGO_4_2_HASH
|
|
n_tasks = 1
|
|
n_tests = 2
|
|
target_file = "target_file.json"
|
|
mock_evg_api = MagicMock()
|
|
split_config = build_mock_split_task_config()
|
|
configure_dependencies(mock_evg_api, split_config)
|
|
tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests)
|
|
|
|
orchestrator = under_test.MultiversionBurnInOrchestrator()
|
|
generated_config = orchestrator.generate_configuration(tests_by_task, target_file,
|
|
"build_variant")
|
|
|
|
evg_config = [
|
|
config for config in generated_config.file_list if config.file_name == target_file
|
|
]
|
|
self.assertEqual(1, len(evg_config))
|
|
evg_config = evg_config[0]
|
|
evg_config_dict = json.loads(evg_config.content)
|
|
tasks = evg_config_dict["tasks"]
|
|
self.assertEqual(len(tasks), NUM_REPL_MIXED_VERSION_CONFIGS * n_tests)
|
|
|
|
@unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
|
|
@patch(
|
|
"buildscripts.evergreen_gen_multiversion_tests.get_backports_required_hash_for_shell_version"
|
|
)
|
|
def test_n_task_m_test(self, mock_hash):
|
|
mock_hash.return_value = MONGO_4_2_HASH
|
|
n_tasks = 2
|
|
n_tests = 3
|
|
target_file = "target_file.json"
|
|
mock_evg_api = MagicMock()
|
|
split_config = build_mock_split_task_config()
|
|
configure_dependencies(mock_evg_api, split_config)
|
|
tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests)
|
|
|
|
orchestrator = under_test.MultiversionBurnInOrchestrator()
|
|
generated_config = orchestrator.generate_configuration(tests_by_task, target_file,
|
|
"build_variant")
|
|
|
|
evg_config = [
|
|
config for config in generated_config.file_list if config.file_name == target_file
|
|
]
|
|
self.assertEqual(1, len(evg_config))
|
|
evg_config = evg_config[0]
|
|
evg_config_dict = json.loads(evg_config.content)
|
|
tasks = evg_config_dict["tasks"]
|
|
self.assertEqual(
|
|
len(tasks),
|
|
(NUM_REPL_MIXED_VERSION_CONFIGS + NUM_SHARDED_MIXED_VERSION_CONFIGS) * n_tests)
|
|
|
|
|
|
class TestGenerateConfig(unittest.TestCase):
|
|
def test_validate_multiversion(self):
|
|
evg_conf_mock = MagicMock()
|
|
gen_config = under_test.GenerateConfig("build_variant", "project")
|
|
gen_config.validate(evg_conf_mock)
|
|
|
|
|
|
class TestGatherTaskInfo(unittest.TestCase):
|
|
def test_multiversion_task(self):
|
|
suite_name = "suite_1"
|
|
distro_name = "distro_1"
|
|
variant = "build_variant"
|
|
evg_conf_mock = MagicMock()
|
|
evg_conf_mock.get_task.return_value.is_generate_resmoke_task = False
|
|
task_mock = create_variant_task_mock("task 1", suite_name, distro_name)
|
|
task_mock.require_multiversion = True
|
|
test_list = [f"test{i}.js" for i in range(3)]
|
|
tests_by_suite = {
|
|
suite_name: test_list,
|
|
"suite 2": [f"test{i}.js" for i in range(1)],
|
|
"suite 3": [f"test{i}.js" for i in range(2)],
|
|
}
|
|
|
|
task_info = TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant)
|
|
|
|
self.assertIn(suite_name, task_info.resmoke_args)
|
|
for test in test_list:
|
|
self.assertIn(test, task_info.tests)
|
|
self.assertEqual(task_mock.require_multiversion, task_info.require_multiversion)
|
|
self.assertEqual(distro_name, task_info.distro)
|