1187 lines
53 KiB
Python
Executable File
1187 lines
53 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Test Failures module.
|
|
|
|
Update etc/test_lifecycle.yml to tag unreliable tests based on historic failure rates.
|
|
"""
|
|
|
|
import collections
|
|
import datetime
|
|
import itertools
|
|
import logging
|
|
import multiprocessing.dummy
|
|
import optparse
|
|
import os.path
|
|
import posixpath
|
|
import sys
|
|
import textwrap
|
|
|
|
import yaml
|
|
|
|
# Get relative imports to work when the package is not installed on the PYTHONPATH.
|
|
if __name__ == "__main__" and __package__ is None:
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
# pylint: disable=wrong-import-position
|
|
from buildscripts import git
|
|
from buildscripts import jiraclient
|
|
from buildscripts import resmokelib
|
|
from buildscripts.resmokelib import utils
|
|
from buildscripts.resmokelib.utils import globstar
|
|
from buildscripts.ciconfig import evergreen as ci_evergreen
|
|
from buildscripts.ciconfig import tags as ci_tags
|
|
from buildscripts.client import evergreen
|
|
from buildscripts.util import testname
|
|
# pylint: enable=wrong-import-position
|
|
|
|
# pylint: disable=too-many-lines
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
_NUMBER_TYPES = (int, float)
|
|
|
|
Rates = collections.namedtuple("Rates", ["acceptable", "unacceptable"])
|
|
|
|
Config = collections.namedtuple("Config", [
|
|
"test_fail_rates",
|
|
"task_fail_rates",
|
|
"variant_fail_rates",
|
|
"distro_fail_rates",
|
|
"reliable_min_runs",
|
|
"reliable_time_period",
|
|
"unreliable_min_runs",
|
|
"unreliable_time_period",
|
|
])
|
|
|
|
DEFAULT_CONFIG = Config(
|
|
test_fail_rates=Rates(acceptable=0.1, unacceptable=0.3),
|
|
task_fail_rates=Rates(acceptable=0.1, unacceptable=0.3),
|
|
variant_fail_rates=Rates(acceptable=0.2, unacceptable=0.4),
|
|
distro_fail_rates=Rates(acceptable=0.2, unacceptable=0.4),
|
|
reliable_min_runs=5,
|
|
reliable_time_period=datetime.timedelta(weeks=1),
|
|
unreliable_min_runs=20,
|
|
unreliable_time_period=datetime.timedelta(weeks=4)) # yapf: disable
|
|
|
|
DEFAULT_PROJECT = "mongodb-mongo-master"
|
|
|
|
DEFAULT_NUM_THREADS = 10
|
|
|
|
MAX_BATCH_SIZE = 50
|
|
|
|
|
|
def get_suite_tasks_membership(evg_conf):
|
|
"""Return a dictionary with keys of all suites and list of associated tasks."""
|
|
suite_membership = collections.defaultdict(list)
|
|
for task in evg_conf.tasks:
|
|
suite = task.resmoke_suite
|
|
if suite:
|
|
suite_membership[suite].append(task.name)
|
|
return suite_membership
|
|
|
|
|
|
def get_test_tasks_membership(evg_conf):
|
|
"""Return a dictionary with keys of all tests and list of associated tasks."""
|
|
test_suites_membership = resmokelib.suitesconfig.create_test_membership_map(test_kind="js_test")
|
|
suite_tasks_membership = get_suite_tasks_membership(evg_conf)
|
|
test_tasks_membership = collections.defaultdict(list)
|
|
for test in list(test_suites_membership.keys()):
|
|
for suite in test_suites_membership[test]:
|
|
test_tasks_membership[test].extend(suite_tasks_membership[suite])
|
|
return test_tasks_membership
|
|
|
|
|
|
def get_tests_from_tasks(tasks, test_tasks_membership):
|
|
"""Return a list of tests from list of specified tasks."""
|
|
tests = []
|
|
tasks_set = set(tasks)
|
|
for test in list(test_tasks_membership.keys()):
|
|
if not tasks_set.isdisjoint(test_tasks_membership[test]):
|
|
tests.append(test)
|
|
return tests
|
|
|
|
|
|
class TestCombination(object):
|
|
"""
|
|
Represent a combination of test, task, variant, and distro.
|
|
|
|
task, variant and distro may be None. If so the following fields must also be None.
|
|
"""
|
|
|
|
GROUP_BY_TEST = "test"
|
|
GROUP_BY_TASK = "task"
|
|
GROUP_BY_VARIANT = "variant"
|
|
GROUP_BY_DISTRO = "distro"
|
|
|
|
def __init__(self, test, task=None, variant=None, distro=None):
|
|
"""Initialize the TestCombination with the field values."""
|
|
self._test = test
|
|
self._task = task
|
|
self._variant = variant
|
|
self._distro = distro
|
|
self._group_by = self._validate()
|
|
|
|
def _validate(self):
|
|
if self._test is None:
|
|
raise ValueError("Test cannot be None in a combination")
|
|
group_by = self.GROUP_BY_DISTRO
|
|
if self._distro is None:
|
|
group_by = self.GROUP_BY_VARIANT
|
|
if self._variant is None:
|
|
group_by = self.GROUP_BY_TASK
|
|
if self._distro is not None:
|
|
raise ValueError("Distro cannot be set if variant is None")
|
|
if self._task is None:
|
|
group_by = self.GROUP_BY_TEST
|
|
if self._variant is not None:
|
|
raise ValueError("Variant cannot be set if task is None")
|
|
return group_by
|
|
|
|
@property
|
|
def test(self):
|
|
"""Return the test field value."""
|
|
return self._test
|
|
|
|
@property
|
|
def task(self):
|
|
"""Return the task field value. May be None."""
|
|
return self._task
|
|
|
|
@property
|
|
def variant(self):
|
|
"""Return the variant field value. May be None."""
|
|
return self._variant
|
|
|
|
@property
|
|
def distro(self):
|
|
"""Return the distro field value. May be None."""
|
|
return self._distro
|
|
|
|
@property
|
|
def group_by(self):
|
|
"""
|
|
Return how the test combination is grouped.
|
|
|
|
The possible values are `GROUP_BY_TEST`, `GROUP_BY_TASK`, `GROUP_BY_VARIANT`,
|
|
and `GROUP_BY_DISTRO`.
|
|
"""
|
|
return self._group_by
|
|
|
|
@property
|
|
def tag(self):
|
|
"""Return the unreliable tag matching this test combination."""
|
|
if self._group_by == self.GROUP_BY_TEST:
|
|
return "unreliable"
|
|
elif self._group_by == self.GROUP_BY_TASK:
|
|
return "unreliable|{}".format(self.task)
|
|
elif self._group_by == self.GROUP_BY_VARIANT:
|
|
return "unreliable|{}|{}".format(self.task, self.variant)
|
|
else:
|
|
return "unreliable|{}|{}|{}".format(self.task, self.variant, self.distro)
|
|
|
|
@staticmethod
|
|
def from_tag(test, tag):
|
|
"""Create a TestCombination from a test name and an unreliable tag."""
|
|
elements = _split_tag(tag)
|
|
return TestCombination(test=test, task=elements[0], variant=elements[1], distro=elements[2])
|
|
|
|
def as_tuple(self):
|
|
"""Return this combination as a (test, task, variant, distro) tuple."""
|
|
return (self.test, self.task, self.variant, self.distro)
|
|
|
|
def __eq__(self, other):
|
|
if type(other) is type(self):
|
|
return self.__dict__ == other.__dict__
|
|
return False
|
|
|
|
def __ne__(self, other):
|
|
return not self.__eq__(other)
|
|
|
|
def __hash__(self):
|
|
return hash(self.as_tuple())
|
|
|
|
def __repr__(self):
|
|
return "%s|%s|%s|%s" % (self.test, self.task, self.variant, self.distro)
|
|
|
|
|
|
class TestHistory(object):
|
|
"""Represent a test execution history over a reliable and unreliable periods."""
|
|
|
|
def __init__(self, test):
|
|
"""Initliaze the TestHistory with the test file name."""
|
|
self.test = test
|
|
self.reliable_stats = []
|
|
self.unreliable_stats = []
|
|
|
|
def get_reliable_period_rates(self, group_by=None):
|
|
"""
|
|
Return (TestCombination, failure rate, number of runs) tuples for the reliable period.
|
|
|
|
The test combinations are grouped according to the group_by parameter.
|
|
"""
|
|
return self._get_rates(self.reliable_stats, group_by)
|
|
|
|
def get_unreliable_period_rates(self, group_by=None):
|
|
"""
|
|
Return (TestCombination, failure rate, number of runs) tuples for the unreliable period.
|
|
|
|
The test combinations are grouped according to the group_by parameter.
|
|
"""
|
|
return self._get_rates(self.unreliable_stats, group_by)
|
|
|
|
def add_reliable_period_stats(self, test_stats_docs):
|
|
"""Add Evergreen test execution statistics documents that cover the reliable period."""
|
|
self.reliable_stats.extend(test_stats_docs)
|
|
|
|
def add_unreliable_period_stats(self, test_stats_docs):
|
|
"""Add Evergreen test execution statistics documents that cover the unreliable period."""
|
|
self.unreliable_stats.extend(test_stats_docs)
|
|
|
|
def _get_rates(self, stats, group_by=None):
|
|
results = []
|
|
if group_by == TestCombination.GROUP_BY_TEST:
|
|
keyfunc = self._group_by_test_key
|
|
elif group_by == TestCombination.GROUP_BY_TASK:
|
|
keyfunc = self._group_by_task_key
|
|
elif group_by == TestCombination.GROUP_BY_VARIANT:
|
|
keyfunc = self._group_by_variant_key
|
|
elif group_by == TestCombination.GROUP_BY_DISTRO or group_by is None:
|
|
keyfunc = self._group_by_distro_key
|
|
else:
|
|
raise ValueError("Invalid group_by value: {}".format(group_by))
|
|
sorted_stats = sorted(stats, key=keyfunc)
|
|
groups = itertools.groupby(sorted_stats, keyfunc)
|
|
for key, group in groups:
|
|
group = list(group)
|
|
results.append((key, self._get_failure_rate(group), self._get_num_run(group)))
|
|
return results
|
|
|
|
@staticmethod
|
|
def _get_failure_rate(test_stats):
|
|
num_pass = sum([d["num_pass"] for d in test_stats])
|
|
num_fail = sum([d["num_fail"] for d in test_stats])
|
|
if (num_pass + num_fail) == 0:
|
|
return 0
|
|
return float(num_fail) / (num_pass + num_fail)
|
|
|
|
@staticmethod
|
|
def _get_num_run(test_stats):
|
|
num_pass = sum([d["num_pass"] for d in test_stats])
|
|
num_fail = sum([d["num_fail"] for d in test_stats])
|
|
return num_pass + num_fail
|
|
|
|
@staticmethod
|
|
def _group_by_distro_key(test_stats_doc):
|
|
return TestCombination(
|
|
test=testname.normalize_test_file(test_stats_doc["test_file"]),
|
|
task=test_stats_doc["task_name"],
|
|
variant=test_stats_doc["variant"],
|
|
distro=test_stats_doc["distro"]) # yapf: disable
|
|
|
|
@staticmethod
|
|
def _group_by_variant_key(test_stats_doc):
|
|
return TestCombination(
|
|
test=testname.normalize_test_file(test_stats_doc["test_file"]),
|
|
task=test_stats_doc["task_name"],
|
|
variant=test_stats_doc["variant"],
|
|
distro=None) # yapf: disable
|
|
|
|
@staticmethod
|
|
def _group_by_task_key(test_stats_doc):
|
|
return TestCombination(
|
|
test=testname.normalize_test_file(test_stats_doc["test_file"]),
|
|
task=test_stats_doc["task_name"],
|
|
variant=None,
|
|
distro=None) # yapf: disable
|
|
|
|
@staticmethod
|
|
def _group_by_test_key(test_stats_doc):
|
|
return TestCombination(
|
|
test=testname.normalize_test_file(test_stats_doc["test_file"]),
|
|
task=None,
|
|
variant=None,
|
|
distro=None) # yapf: disable
|
|
|
|
|
|
class TestHistorySource(object): # pylint: disable=too-many-instance-attributes
|
|
"""A class used to parallelize requests to buildscripts.test_failures.TestHistory."""
|
|
|
|
def __init__( # pylint: disable=too-many-arguments
|
|
self, project, variants, distros, reliable_period, unreliable_period,
|
|
thread_pool_size=DEFAULT_NUM_THREADS):
|
|
"""Initialize the TestHistorySource.
|
|
|
|
Args:
|
|
project: the Evergreen project name.
|
|
variants: a list of variant names.
|
|
distros: a list of distro names.
|
|
reliable_period: the time delta for the reliable period.
|
|
unreliable_period: the time delta for the unreliable period.
|
|
thread_pool_size: the size of the thread pool used to make parallel requests.
|
|
"""
|
|
self._project = project
|
|
self._variants = variants
|
|
self._distros = distros
|
|
not_after = datetime.datetime.utcnow()
|
|
|
|
self._not_after = not_after.strftime(format="%Y-%m-%d")
|
|
self._reliable_not_before = (not_after - reliable_period).strftime(format="%Y-%m-%d")
|
|
|
|
self._reliable_group_num_days = reliable_period.days
|
|
self._unreliable_not_before = (not_after - unreliable_period).strftime(format="%Y-%m-%d")
|
|
self._unreliable_group_num_days = unreliable_period.days
|
|
|
|
self._thread_pool = multiprocessing.dummy.Pool(thread_pool_size)
|
|
self._evg_api = evergreen.get_evergreen_apiv2(num_retries=3)
|
|
|
|
def get_history(self, test_tasks_list):
|
|
"""Return an iterator with the TestHistory for each of the tests provided.
|
|
|
|
Args:
|
|
test_tasks_list: a list of (test, task list) tuples.
|
|
"""
|
|
return self._thread_pool.imap_unordered(self._get_history, test_tasks_list)
|
|
|
|
def _get_history(self, test_tasks):
|
|
try:
|
|
test, tasks = test_tasks
|
|
history = TestHistory(test)
|
|
tests = testname.denormalize_test_file(test)
|
|
# Get stats for the reliable period.
|
|
history.add_reliable_period_stats(
|
|
self._get_test_stats(self._reliable_not_before, self._reliable_group_num_days,
|
|
tests, tasks))
|
|
# Get stats for the unreliable period.
|
|
history.add_unreliable_period_stats(
|
|
self._get_test_stats(self._unreliable_not_before, self._unreliable_group_num_days,
|
|
tests, tasks))
|
|
return history
|
|
except Exception:
|
|
LOGGER.exception("An error occurred while getting test history")
|
|
raise
|
|
|
|
def _get_test_stats(self, not_before, group_num_days, tests, tasks):
|
|
return self._evg_api.test_stats(self._project, not_before, self._not_after, group_num_days,
|
|
tests=tests, tasks=tasks, variants=self._variants,
|
|
distros=self._distros)
|
|
|
|
|
|
def unreliable_test(test_fr, unacceptable_fr, test_runs, min_run):
|
|
"""Check for an unreliable test.
|
|
|
|
A test should be added to the set of tests believed not to run reliably when it has more
|
|
than min_run executions with a failure percentage greater than unacceptable_fr.
|
|
"""
|
|
return test_runs >= min_run and test_fr >= unacceptable_fr
|
|
|
|
|
|
def reliable_test(test_fr, acceptable_fr, test_runs, min_run):
|
|
"""Check for a reliable test.
|
|
|
|
A test should then removed from the set of tests believed not to run reliably when it has
|
|
less than min_run executions or has a failure percentage less than acceptable_fr.
|
|
"""
|
|
return test_runs < min_run or test_fr <= acceptable_fr
|
|
|
|
|
|
def update_lifecycle( # pylint: disable=too-many-arguments
|
|
lifecycle_tags_file, rates, method_test, add_tags, threshold, min_run):
|
|
"""Update the lifecycle object based on the test_method.
|
|
|
|
The test_method checks unreliable or reliable fail_rates.
|
|
"""
|
|
for combination, fail_rate, num_run in rates:
|
|
if method_test(fail_rate, threshold, num_run, min_run):
|
|
update_tag = combination.tag
|
|
if add_tags:
|
|
lifecycle_tags_file.add_tag("js_test", combination.test, update_tag, fail_rate)
|
|
else:
|
|
lifecycle_tags_file.remove_tag("js_test", combination.test, update_tag, fail_rate)
|
|
|
|
|
|
def compare_tags(tag_a, tag_b):
|
|
"""Return 1, -1 or 0 if 'tag_a' is superior, inferior or equal to 'tag_b'."""
|
|
a_split = tag_a.split("|")
|
|
b_split = tag_b.split("|")
|
|
return (a_split > b_split) - (a_split < b_split)
|
|
|
|
|
|
def validate_config(config): # pylint: disable=too-many-branches
|
|
"""Raise a TypeError or ValueError exception if 'config' isn't a valid model."""
|
|
|
|
for (name, fail_rates) in (("test", config.test_fail_rates), ("task", config.task_fail_rates),
|
|
("variant", config.variant_fail_rates), ("distro",
|
|
config.distro_fail_rates)):
|
|
if not isinstance(fail_rates.acceptable, _NUMBER_TYPES):
|
|
raise TypeError("The acceptable {} failure rate must be a number, but got {}".format(
|
|
name, fail_rates.acceptable))
|
|
elif fail_rates.acceptable < 0 or fail_rates.acceptable > 1:
|
|
raise ValueError(("The acceptable {} failure rate must be between 0 and 1 (inclusive),"
|
|
" but got {}").format(name, fail_rates.acceptable))
|
|
elif not isinstance(fail_rates.unacceptable, _NUMBER_TYPES):
|
|
raise TypeError("The unacceptable {} failure rate must be a number, but got {}".format(
|
|
name, fail_rates.unacceptable))
|
|
elif fail_rates.unacceptable < 0 or fail_rates.unacceptable > 1:
|
|
raise ValueError(("The unacceptable {} failure rate must be between 0 and 1"
|
|
" (inclusive), but got {}").format(name, fail_rates.unacceptable))
|
|
elif fail_rates.acceptable > fail_rates.unacceptable:
|
|
raise ValueError(
|
|
("The acceptable {0} failure rate ({1}) must be no larger than unacceptable {0}"
|
|
" failure rate ({2})").format(name, fail_rates.acceptable,
|
|
fail_rates.unacceptable))
|
|
|
|
for (name, min_runs) in (("reliable", config.reliable_min_runs), ("unreliable",
|
|
config.unreliable_min_runs)):
|
|
if not isinstance(min_runs, _NUMBER_TYPES):
|
|
raise TypeError(("The minimum number of runs for considering a test {} must be a"
|
|
" number, but got {}").format(name, min_runs))
|
|
elif min_runs <= 0:
|
|
raise ValueError(("The minimum number of runs for considering a test {} must be a"
|
|
" positive integer, but got {}").format(name, min_runs))
|
|
elif isinstance(min_runs, float) and not min_runs.is_integer():
|
|
raise ValueError(("The minimum number of runs for considering a test {} must be an"
|
|
" integer, but got {}").format(name, min_runs))
|
|
|
|
for (name, time_period) in (("reliable", config.reliable_time_period),
|
|
("unreliable", config.unreliable_time_period)):
|
|
if not isinstance(time_period, datetime.timedelta):
|
|
raise TypeError(
|
|
"The {} time period must be a datetime.timedelta instance, but got {}".format(
|
|
name, time_period))
|
|
elif time_period.days <= 0:
|
|
raise ValueError(
|
|
"The {} time period must be a positive number of days, but got {}".format(
|
|
name, time_period))
|
|
elif time_period - datetime.timedelta(days=time_period.days) > datetime.timedelta():
|
|
raise ValueError(
|
|
"The {} time period must be an integral number of days, but got {}".format(
|
|
name, time_period))
|
|
|
|
|
|
def update_tags(lifecycle_tags, config, test_history): # pylint: disable=too-many-locals
|
|
"""Update the tags in 'lifecycle_tags'.
|
|
|
|
This is based on the historical test failures of tests 'tests'
|
|
mentioned in 'report' according to the model described by 'config'.
|
|
"""
|
|
|
|
# yapf: disable
|
|
for (group_by, rates) in [(TestCombination.GROUP_BY_TEST, config.test_fail_rates),
|
|
(TestCombination.GROUP_BY_TASK, config.task_fail_rates),
|
|
(TestCombination.GROUP_BY_VARIANT, config.variant_fail_rates),
|
|
(TestCombination.GROUP_BY_DISTRO, config.distro_fail_rates)]:
|
|
|
|
reliable_rates = test_history.get_reliable_period_rates(group_by)
|
|
update_lifecycle(lifecycle_tags, reliable_rates, reliable_test, False,
|
|
rates.acceptable, config.unreliable_min_runs)
|
|
|
|
unreliable_rates = test_history.get_unreliable_period_rates(group_by)
|
|
update_lifecycle(lifecycle_tags, unreliable_rates, unreliable_test, True,
|
|
rates.unacceptable, config.unreliable_min_runs)
|
|
|
|
# Remove the tags that correspond to combinations that have not run during the reliable
|
|
# period.
|
|
test = test_history.test
|
|
reliable_combinations = {r[0] for r in reliable_rates}
|
|
tags = lifecycle_tags.lifecycle.get_tags("js_test", test)
|
|
for tag in tags[:]:
|
|
tag_combination = TestCombination.from_tag(test, tag)
|
|
if tag_combination.group_by != group_by:
|
|
continue
|
|
if tag_combination not in reliable_combinations:
|
|
LOGGER.info("Removing tag '%s' of test '%s' because the combination did not run"
|
|
" during the reliable period", tag, test)
|
|
lifecycle_tags.remove_tag("js_test", test, tag, failure_rate=0)
|
|
|
|
|
|
def _split_tag(tag):
|
|
"""Split a tag into its components.
|
|
|
|
Return a tuple containing task, variant, distro. The values are None if absent from the tag.
|
|
If the tag is invalid, the return value is (None, None, None).
|
|
"""
|
|
elements = tag.split("|")
|
|
length = len(elements)
|
|
if elements[0] != "unreliable" or length < 2 or length > 4:
|
|
return None, None, None
|
|
# Fill out the array.
|
|
elements.extend([None] * (4 - length))
|
|
# Return as a tuple.
|
|
return tuple(elements[1:])
|
|
|
|
|
|
def _is_tag_still_relevant(evg_conf, tag):
|
|
"""Indicate if a tag still corresponds to a valid task/variant/distro combination."""
|
|
if tag == "unreliable":
|
|
return True
|
|
task, variant, distro = _split_tag(tag)
|
|
if not task or task not in evg_conf.task_names:
|
|
return False
|
|
if variant:
|
|
variant_conf = evg_conf.get_variant(variant)
|
|
if not variant_conf or task not in variant_conf.task_names:
|
|
return False
|
|
if distro and distro not in variant_conf.distro_names:
|
|
return False
|
|
return True
|
|
|
|
|
|
def clean_up_tags(lifecycle_tags, evg_conf):
|
|
"""Remove the tags that do not correspond to a valid test/task/variant/distro combination."""
|
|
lifecycle = lifecycle_tags.lifecycle
|
|
for test_kind in lifecycle.get_test_kinds():
|
|
for test_pattern in lifecycle.get_test_patterns(test_kind):
|
|
if not globstar.glob(test_pattern):
|
|
# The pattern does not match any file in the repository.
|
|
lifecycle_tags.clean_up_test(test_kind, test_pattern)
|
|
continue
|
|
for tag in lifecycle.get_tags(test_kind, test_pattern):
|
|
if not _is_tag_still_relevant(evg_conf, tag):
|
|
lifecycle_tags.clean_up_tag(test_kind, test_pattern, tag)
|
|
|
|
|
|
def _config_as_options(config):
|
|
return ("--reliableTestMinRuns {} "
|
|
"--reliableDays {} "
|
|
"--unreliableTestMinRuns {} "
|
|
"--unreliableDays {} "
|
|
"--testFailRates {} {} "
|
|
"--taskFailRates {} {} "
|
|
"--variantFailRates {} {} "
|
|
"--distroFailRates {} {}").format(
|
|
config.reliable_min_runs, config.reliable_time_period.days,
|
|
config.unreliable_min_runs, config.unreliable_time_period.days,
|
|
config.test_fail_rates.acceptable, config.test_fail_rates.unacceptable,
|
|
config.task_fail_rates.acceptable, config.task_fail_rates.unacceptable,
|
|
config.variant_fail_rates.acceptable, config.variant_fail_rates.unacceptable,
|
|
config.distro_fail_rates.acceptable, config.distro_fail_rates.unacceptable)
|
|
|
|
|
|
class TagsConfigWithChangelog(object):
|
|
"""A wrapper around TagsConfig to update a tags file and record the modifications made."""
|
|
|
|
def __init__(self, lifecycle):
|
|
"""Initialize the TagsConfigWithChangelog with the lifecycle TagsConfig."""
|
|
self.lifecycle = lifecycle
|
|
self.added = {}
|
|
self.removed = {}
|
|
self.cleaned_up = {}
|
|
|
|
@staticmethod
|
|
def _cancel_tag_log(log_dict, test_kind, test, tag):
|
|
"""Remove a tag from a changelog dictionary.
|
|
|
|
Used to remove a tag from the 'added' or 'removed' attribute.
|
|
"""
|
|
kind_dict = log_dict[test_kind]
|
|
test_dict = kind_dict[test]
|
|
del test_dict[tag]
|
|
if not test_dict:
|
|
del kind_dict[test]
|
|
if not kind_dict:
|
|
del log_dict[test_kind]
|
|
|
|
def add_tag(self, test_kind, test, tag, failure_rate):
|
|
"""Add a tag."""
|
|
if self.lifecycle.add_tag(test_kind, test, tag):
|
|
if tag in self.removed.get(test_kind, {}).get(test, {}):
|
|
# The tag has just been removed.
|
|
self._cancel_tag_log(self.removed, test_kind, test, tag)
|
|
else:
|
|
self.added.setdefault(test_kind, {}).setdefault(test, {})[tag] = failure_rate
|
|
|
|
def remove_tag(self, test_kind, test, tag, failure_rate):
|
|
"""Remove a tag."""
|
|
if self.lifecycle.remove_tag(test_kind, test, tag):
|
|
if tag in self.added.get(test_kind, {}).get(test, {}):
|
|
# The tag has just been added.
|
|
self._cancel_tag_log(self.added, test_kind, test, tag)
|
|
else:
|
|
self.removed.setdefault(test_kind, {}).setdefault(test, {})[tag] = failure_rate
|
|
|
|
def clean_up_tag(self, test_kind, test, tag):
|
|
"""Clean up an invalid tag."""
|
|
self.lifecycle.remove_tag(test_kind, test, tag)
|
|
self.cleaned_up.setdefault(test_kind, {}).setdefault(test, []).append(tag)
|
|
|
|
def clean_up_test(self, test_kind, test):
|
|
"""Clean up an invalid test."""
|
|
self.lifecycle.remove_test_pattern(test_kind, test)
|
|
self.cleaned_up.setdefault(test_kind, {})[test] = []
|
|
|
|
|
|
class JiraIssueCreator(object):
|
|
"""JiraIssueCreator class."""
|
|
|
|
_LABEL = "test-lifecycle"
|
|
_PROJECT = "TIGBOT"
|
|
_MAX_DESCRIPTION_SIZE = 32767
|
|
|
|
def __init__( # pylint: disable=too-many-arguments
|
|
self, server=None, username=None, password=None, access_token=None,
|
|
access_token_secret=None, consumer_key=None, key_cert=None):
|
|
"""Initialize JiraIssueCreator."""
|
|
self._client = jiraclient.JiraClient(
|
|
server=server, username=username, password=password, access_token=access_token,
|
|
access_token_secret=access_token_secret, consumer_key=consumer_key, key_cert=key_cert)
|
|
|
|
def create_issue( # pylint: disable=too-many-arguments
|
|
self, evg_project, mongo_revision, model_config, added, removed, cleaned_up):
|
|
"""Create a JIRA issue for the test lifecycle tag update."""
|
|
summary = self._get_jira_summary(evg_project)
|
|
description = self._get_jira_description(evg_project, mongo_revision, model_config, added,
|
|
removed, cleaned_up)
|
|
issue_key = self._client.create_issue(self._PROJECT, summary, description, [self._LABEL])
|
|
return issue_key
|
|
|
|
def close_fix_issue(self, issue_key):
|
|
"""Close the issue with the "Fixed" resolution."""
|
|
LOGGER.info("Closing issue '%s' as FIXED.", issue_key)
|
|
self._client.close_issue(issue_key, self._client.FIXED_RESOLUTION_NAME)
|
|
|
|
def close_wontfix_issue(self, issue_key):
|
|
"""Close the issue the with "Won't Fix" resolution."""
|
|
LOGGER.info("Closing issue '%s' as WON'T FIX.", issue_key)
|
|
self._client.close_issue(issue_key, self._client.WONT_FIX_RESOLUTION_NAME)
|
|
|
|
@staticmethod
|
|
def _get_jira_summary(project):
|
|
return "Update of test lifecycle tags for {}".format(project)
|
|
|
|
@staticmethod
|
|
def _monospace(text):
|
|
"""Transform a text into a monospace JIRA text."""
|
|
return "{{" + text + "}}"
|
|
|
|
@staticmethod
|
|
def _truncate_description(desc):
|
|
max_size = JiraIssueCreator._MAX_DESCRIPTION_SIZE
|
|
if len(desc) > max_size:
|
|
warning = ("\nDescription truncated: "
|
|
"exceeded max size of {} characters.").format(max_size)
|
|
truncated_length = max_size - len(warning)
|
|
desc = desc[:truncated_length] + warning
|
|
return desc
|
|
|
|
@staticmethod
|
|
def _get_jira_description( # pylint: disable=too-many-arguments
|
|
project, mongo_revision, model_config, added, removed, cleaned_up):
|
|
mono = JiraIssueCreator._monospace
|
|
config_desc = _config_as_options(model_config)
|
|
added_desc = JiraIssueCreator._make_updated_tags_description(added)
|
|
removed_desc = JiraIssueCreator._make_updated_tags_description(removed)
|
|
cleaned_up_desc = JiraIssueCreator._make_tags_cleaned_up_description(cleaned_up)
|
|
project_link = "[{0}|https://evergreen.mongodb.com/waterfall/{1}]".format(
|
|
mono(project), project)
|
|
revision_link = "[{0}|https://github.com/mongodb/mongo/commit/{1}]".format(
|
|
mono(mongo_revision), mongo_revision)
|
|
full_desc = ("h3. Automatic update of the test lifecycle tags\n"
|
|
"Evergreen Project: {0}\n"
|
|
"Revision: {1}\n\n"
|
|
"{{{{update_test_lifecycle.py}}}} options:\n{2}\n\n"
|
|
"h5. Tags added\n{3}\n\n"
|
|
"h5. Tags removed\n{4}\n\n"
|
|
"h5. Tags cleaned up (no longer relevant)\n{5}\n").format(
|
|
project_link, revision_link, mono(config_desc), added_desc, removed_desc,
|
|
cleaned_up_desc)
|
|
|
|
return JiraIssueCreator._truncate_description(full_desc)
|
|
|
|
@staticmethod
|
|
def _make_updated_tags_description(data):
|
|
mono = JiraIssueCreator._monospace
|
|
tags_lines = []
|
|
for test_kind in sorted(data.keys()):
|
|
tests = data[test_kind]
|
|
tags_lines.append("- *{0}*".format(test_kind))
|
|
for test in sorted(tests.keys()):
|
|
tags = tests[test]
|
|
tags_lines.append("-- {0}".format(mono(test)))
|
|
for tag in sorted(tags.keys()):
|
|
coefficient = tags[tag]
|
|
tags_lines.append("--- {0} ({1:.2f})".format(mono(tag), coefficient))
|
|
if tags_lines:
|
|
return "\n".join(tags_lines)
|
|
return "_None_"
|
|
|
|
@staticmethod
|
|
def _make_tags_cleaned_up_description(cleaned_up):
|
|
mono = JiraIssueCreator._monospace
|
|
tags_cleaned_up_lines = []
|
|
for test_kind in sorted(cleaned_up.keys()):
|
|
test_tags = cleaned_up[test_kind]
|
|
tags_cleaned_up_lines.append("- *{0}*".format(test_kind))
|
|
for test in sorted(test_tags.keys()):
|
|
tags = test_tags[test]
|
|
tags_cleaned_up_lines.append("-- {0}".format(mono(test)))
|
|
if not tags:
|
|
tags_cleaned_up_lines.append("--- ALL (test file removed or renamed as part of"
|
|
" an earlier commit)")
|
|
else:
|
|
for tag in sorted(tags):
|
|
tags_cleaned_up_lines.append("--- {0}".format(mono(tag)))
|
|
if tags_cleaned_up_lines:
|
|
return "\n".join(tags_cleaned_up_lines)
|
|
return "_None_"
|
|
|
|
|
|
class LifecycleTagsFile(object): # pylint: disable=too-many-instance-attributes
|
|
"""Represent a test lifecycle tags file that can be written and committed."""
|
|
|
|
def __init__( # pylint: disable=too-many-arguments
|
|
self, project, lifecycle_file, metadata_repo_url=None, references_file=None,
|
|
jira_issue_creator=None, git_info=None,
|
|
model_config=None): # noqa: D214,D401,D405,D406,D407,D411,D413
|
|
"""Initalize the LifecycleTagsFile.
|
|
|
|
Arguments:
|
|
project: The Evergreen project name, e.g. "mongodb-mongo-master".
|
|
lifecycle_file: The path to the lifecycle tags file. If 'metadata_repo_url' is
|
|
specified, this path must be relative to the root of the metadata repository.
|
|
metadata_repo_url: The URL of the metadat repository that contains the test lifecycle
|
|
tags file.
|
|
references_file: The path to the references file in the metadata repository.
|
|
jira_issue_creator: A JiraIssueCreator instance.
|
|
git_info: A tuple containing the git user's name and email to set before committing.
|
|
model_config: The model configuration as a Config instance.
|
|
"""
|
|
self.project = project
|
|
self.mongo_repo = git.Repository(os.getcwd())
|
|
self.mongo_revision = self.mongo_repo.get_current_revision()
|
|
# The branch name is the same on both repositories.
|
|
self.mongo_branch = self.mongo_repo.get_branch_name()
|
|
self.metadata_branch = project
|
|
|
|
if metadata_repo_url:
|
|
# The file can be found in another repository. We clone it.
|
|
self.metadata_repo = self._clone_repository(metadata_repo_url, self.project)
|
|
self.relative_lifecycle_file = lifecycle_file
|
|
self.lifecycle_file = os.path.join(self.metadata_repo.directory, lifecycle_file)
|
|
self.relative_references_file = references_file
|
|
self.references_file = os.path.join(self.metadata_repo.directory, references_file)
|
|
if git_info:
|
|
self.metadata_repo.configure("user.name", git_info[0])
|
|
self.metadata_repo.configure("user.email", git_info[1])
|
|
else:
|
|
self.metadata_repo = None
|
|
self.relative_lifecycle_file = lifecycle_file
|
|
self.lifecycle_file = lifecycle_file
|
|
self.relative_references_file = None
|
|
self.references_file = None
|
|
self.metadata_repo_url = metadata_repo_url
|
|
self.lifecycle = ci_tags.TagsConfig.from_file(self.lifecycle_file, cmp_func=compare_tags)
|
|
self.jira_issue_creator = jira_issue_creator
|
|
self.model_config = model_config
|
|
self.changelog_lifecycle = TagsConfigWithChangelog(self.lifecycle)
|
|
|
|
@staticmethod
|
|
def _clone_repository(metadata_repo_url, branch):
|
|
directory_name = posixpath.splitext(posixpath.basename(metadata_repo_url))[0]
|
|
LOGGER.info("Cloning the repository %s into the directory %s", metadata_repo_url,
|
|
directory_name)
|
|
return git.Repository.clone(metadata_repo_url, directory_name, branch)
|
|
|
|
def is_modified(self):
|
|
"""Indicate if the tags have been modified."""
|
|
return self.lifecycle.is_modified()
|
|
|
|
def _create_issue(self):
|
|
LOGGER.info("Creating a JIRA issue")
|
|
issue_key = self.jira_issue_creator.create_issue(
|
|
self.project, self.mongo_revision, self.model_config, self.changelog_lifecycle.added,
|
|
self.changelog_lifecycle.removed, self.changelog_lifecycle.cleaned_up)
|
|
LOGGER.info("JIRA issue created: %s", issue_key)
|
|
return issue_key
|
|
|
|
def write(self):
|
|
"""Write the test lifecycle tag file."""
|
|
LOGGER.info("Writing the tag file to '%s'", self.lifecycle_file)
|
|
comment = ("This file was generated by {} and shouldn't be edited by hand. It was"
|
|
" generated against commit {} with the following options: {}.").format(
|
|
sys.argv[0], self.mongo_repo.get_current_revision(),
|
|
_config_as_options(self.model_config))
|
|
self.lifecycle.write_file(self.lifecycle_file, comment)
|
|
|
|
def _ready_for_commit(self, ref_branch, references):
|
|
# Check that the test lifecycle tags file has changed.
|
|
diff = self.metadata_repo.git_diff(
|
|
["--name-only", ref_branch, self.relative_lifecycle_file])
|
|
if not diff:
|
|
LOGGER.info("The local lifecycle file is identical to the the one on branch '%s'",
|
|
ref_branch)
|
|
return False
|
|
# Check that the lifecycle file has not been updated after the current mongo revision.
|
|
update_revision = references.get("test-lifecycle", {}).get(self.project)
|
|
if update_revision and not self.mongo_repo.is_ancestor(update_revision,
|
|
self.mongo_revision):
|
|
LOGGER.warning(("The existing lifecycle file is based on revision '%s' which is not a"
|
|
" parent revision of the current revision '%s'"), update_revision,
|
|
self.mongo_revision)
|
|
return False
|
|
return True
|
|
|
|
def _read_references(self, metadata_branch=None):
|
|
branch = metadata_branch if metadata_branch is not None else ""
|
|
references_content = self.metadata_repo.git_cat_file(
|
|
["blob", "{0}:{1}".format(branch, self.relative_references_file)])
|
|
return yaml.safe_load(references_content)
|
|
|
|
def _update_and_write_references(self, references):
|
|
LOGGER.info("Writing the references file to '%s'", self.references_file)
|
|
references.setdefault("test-lifecycle", {})[self.project] = self.mongo_revision
|
|
with open(self.references_file, "w") as fstream:
|
|
yaml.safe_dump(references, fstream, default_flow_style=False)
|
|
|
|
def _commit_locally(self, issue_key):
|
|
self.metadata_repo.git_add([self.relative_lifecycle_file])
|
|
self.metadata_repo.git_add([self.relative_references_file])
|
|
commit_message = "{} Update {}".format(issue_key, self.relative_lifecycle_file)
|
|
self.metadata_repo.commit_with_message(commit_message)
|
|
LOGGER.info("Change committed with message: %s", commit_message)
|
|
|
|
def commit(self, nb_retries=10):
|
|
"""Commit the test lifecycle tag file.
|
|
|
|
Args:
|
|
nb_retries: the number of times the script will reset, fetch, recommit and retry when
|
|
the push fails.
|
|
"""
|
|
references = self._read_references()
|
|
# Verify we are ready to commit.
|
|
if not self._ready_for_commit(self.metadata_branch, references):
|
|
return True
|
|
|
|
# Write the references file.
|
|
self._update_and_write_references(references)
|
|
|
|
# Create the issue.
|
|
issue_key = self._create_issue()
|
|
|
|
# Commit the change.
|
|
self._commit_locally(issue_key)
|
|
|
|
# Push the change.
|
|
tries = 0
|
|
pushed = False
|
|
upstream = "origin/{0}".format(self.metadata_branch)
|
|
while tries < nb_retries:
|
|
try:
|
|
self.metadata_repo.push_to_remote_branch("origin", self.metadata_branch)
|
|
pushed = True
|
|
break
|
|
except git.GitException:
|
|
LOGGER.warning("git push command failed, fetching and retrying.")
|
|
# Fetch upstream branch.
|
|
LOGGER.info("Fetching branch %s of %s", self.metadata_branch,
|
|
self.metadata_repo_url)
|
|
self.metadata_repo.fetch_remote_branch("origin", self.metadata_branch)
|
|
# Resetting the current branch to the origin branch
|
|
LOGGER.info("Resetting branch %s to %s", self.metadata_branch, upstream)
|
|
self.metadata_repo.git_reset(["--hard", upstream])
|
|
# Rewrite the test lifecycle tags file
|
|
self.write()
|
|
# Rewrite the references file
|
|
references = self._read_references()
|
|
self._update_and_write_references(references)
|
|
# Checking if we can still commit
|
|
if not self._ready_for_commit(upstream, references):
|
|
LOGGER.warning("Aborting.")
|
|
break
|
|
# Committing
|
|
self._commit_locally(issue_key)
|
|
tries += 1
|
|
if pushed:
|
|
self.jira_issue_creator.close_fix_issue(issue_key)
|
|
return True
|
|
self.jira_issue_creator.close_wontfix_issue(issue_key)
|
|
return False
|
|
|
|
|
|
def make_lifecycle_tags_file(options, model_config):
|
|
"""Create a LifecycleTagsFile based on the script options."""
|
|
if options.commit:
|
|
if not options.jira_config:
|
|
LOGGER.error("JIRA configuration file is required when specifying --commit.")
|
|
return None
|
|
if not (options.git_user_name or options.git_user_email):
|
|
LOGGER.error("Git configuration parameters are required when specifying --commit.")
|
|
return None
|
|
jira_issue_creator = JiraIssueCreator(**utils.load_yaml_file(options.jira_config))
|
|
git_config = (options.git_user_name, options.git_user_email)
|
|
else:
|
|
jira_issue_creator = None
|
|
git_config = None
|
|
|
|
lifecycle_tags_file = LifecycleTagsFile(options.project, options.tag_file,
|
|
options.metadata_repo_url, options.references_file,
|
|
jira_issue_creator, git_config, model_config)
|
|
|
|
return lifecycle_tags_file
|
|
|
|
|
|
def main(): # pylint: disable=too-many-branches,too-many-locals,too-many-statements
|
|
"""Exexcute utility to update a resmoke.py tag file.
|
|
|
|
This is based on computing test failure rates from the Evergreen API.
|
|
"""
|
|
|
|
parser = optparse.OptionParser(
|
|
description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] [test1 test2 ...]")
|
|
|
|
data_options = optparse.OptionGroup(
|
|
parser, title="Data options",
|
|
description=("Options used to configure what historical test failure data to retrieve from"
|
|
" Evergreen."))
|
|
parser.add_option_group(data_options)
|
|
|
|
data_options.add_option("--project", dest="project", metavar="<project-name>",
|
|
default=DEFAULT_PROJECT,
|
|
help="The Evergreen project to analyze. Defaults to '%default'.")
|
|
|
|
data_options.add_option(
|
|
"--tasks", dest="tasks", metavar="<task1,task2,...>",
|
|
help=("The Evergreen tasks to analyze for tagging unreliable tests. If specified in"
|
|
" additional to having test positional arguments, then only tests that run under the"
|
|
" specified Evergreen tasks will be analyzed. If omitted, then the list of tasks"
|
|
" defaults to the non-excluded list of tasks from the specified"
|
|
" --evergreenProjectConfig file."))
|
|
|
|
data_options.add_option(
|
|
"--variants", dest="variants", metavar="<variant1,variant2,...>", default="",
|
|
help="The Evergreen build variants to analyze for tagging unreliable tests.")
|
|
|
|
data_options.add_option("--distros", dest="distros", metavar="<distro1,distro2,...>",
|
|
default="",
|
|
help="The Evergreen distros to analyze for tagging unreliable tests.")
|
|
|
|
data_options.add_option(
|
|
"--evergreenProjectConfig", dest="evergreen_project_config",
|
|
metavar="<project-config-file>", default="etc/evergreen.yml",
|
|
help=("The Evergreen project configuration file used to get the list of tasks if --tasks is"
|
|
" omitted. Defaults to '%default'."))
|
|
|
|
model_options = optparse.OptionGroup(
|
|
parser, title="Model options",
|
|
description=("Options used to configure whether (test,), (test, task),"
|
|
" (test, task, variant), and (test, task, variant, distro) combinations are"
|
|
" considered unreliable."))
|
|
parser.add_option_group(model_options)
|
|
|
|
model_options.add_option(
|
|
"--reliableTestMinRuns", type="int", dest="reliable_test_min_runs",
|
|
metavar="<reliable-min-runs>", default=DEFAULT_CONFIG.reliable_min_runs,
|
|
help=("The minimum number of test executions required for a test's failure rate to"
|
|
" determine whether the test is considered reliable. If a test has fewer than"
|
|
" <reliable-min-runs> executions, then it cannot be considered unreliable."))
|
|
|
|
model_options.add_option(
|
|
"--unreliableTestMinRuns", type="int", dest="unreliable_test_min_runs",
|
|
metavar="<unreliable-min-runs>", default=DEFAULT_CONFIG.unreliable_min_runs,
|
|
help=("The minimum number of test executions required for a test's failure rate to"
|
|
" determine whether the test is considered unreliable. If a test has fewer than"
|
|
" <unreliable-min-runs> executions, then it cannot be considered unreliable."))
|
|
|
|
model_options.add_option(
|
|
"--testFailRates", type="float", nargs=2, dest="test_fail_rates",
|
|
metavar="<test-acceptable-fail-rate> <test-unacceptable-fail-rate>",
|
|
default=DEFAULT_CONFIG.test_fail_rates,
|
|
help=("Controls how readily a test is considered unreliable. Each failure rate must be a"
|
|
" number between 0 and 1 (inclusive) with"
|
|
" <test-unacceptable-fail-rate> >= <test-acceptable-fail-rate>. If a test fails no"
|
|
" more than <test-acceptable-fail-rate> in <reliable-days> time, then it is"
|
|
" considered reliable. Otherwise, if a test fails at least as much as"
|
|
" <test-unacceptable-fail-rate> in <test-unreliable-days> time, then it is considered"
|
|
" unreliable. Defaults to %default."))
|
|
|
|
model_options.add_option(
|
|
"--taskFailRates", type="float", nargs=2, dest="task_fail_rates",
|
|
metavar="<task-acceptable-fail-rate> <task-unacceptable-fail-rate>",
|
|
default=DEFAULT_CONFIG.task_fail_rates,
|
|
help=("Controls how readily a (test, task) combination is considered unreliable. Each"
|
|
" failure rate must be a number between 0 and 1 (inclusive) with"
|
|
" <task-unacceptable-fail-rate> >= <task-acceptable-fail-rate>. If a (test, task)"
|
|
" combination fails no more than <task-acceptable-fail-rate> in <reliable-days> time,"
|
|
" then it is considered reliable. Otherwise, if a test fails at least as much as"
|
|
" <task-unacceptable-fail-rate> in <unreliable-days> time, then it is considered"
|
|
" unreliable. Defaults to %default."))
|
|
|
|
model_options.add_option(
|
|
"--variantFailRates", type="float", nargs=2, dest="variant_fail_rates",
|
|
metavar="<variant-acceptable-fail-rate> <variant-unacceptable-fail-rate>",
|
|
default=DEFAULT_CONFIG.variant_fail_rates,
|
|
help=("Controls how readily a (test, task, variant) combination is considered unreliable."
|
|
" Each failure rate must be a number between 0 and 1 (inclusive) with"
|
|
" <variant-unacceptable-fail-rate> >= <variant-acceptable-fail-rate>. If a"
|
|
" (test, task, variant) combination fails no more than <variant-acceptable-fail-rate>"
|
|
" in <reliable-days> time, then it is considered reliable. Otherwise, if a test fails"
|
|
" at least as much as <variant-unacceptable-fail-rate> in <unreliable-days> time,"
|
|
" then it is considered unreliable. Defaults to %default."))
|
|
|
|
model_options.add_option(
|
|
"--distroFailRates", type="float", nargs=2, dest="distro_fail_rates",
|
|
metavar="<distro-acceptable-fail-rate> <distro-unacceptable-fail-rate>",
|
|
default=DEFAULT_CONFIG.distro_fail_rates,
|
|
help=("Controls how readily a (test, task, variant, distro) combination is considered"
|
|
" unreliable. Each failure rate must be a number between 0 and 1 (inclusive) with"
|
|
" <distro-unacceptable-fail-rate> >= <distro-acceptable-fail-rate>. If a"
|
|
" (test, task, variant, distro) combination fails no more than"
|
|
" <distro-acceptable-fail-rate> in <reliable-days> time, then it is considered"
|
|
" reliable. Otherwise, if a test fails at least as much as"
|
|
" <distro-unacceptable-fail-rate> in <unreliable-days> time, then it is considered"
|
|
" unreliable. Defaults to %default."))
|
|
|
|
model_options.add_option(
|
|
"--reliableDays", type="int", dest="reliable_days", metavar="<ndays>",
|
|
default=DEFAULT_CONFIG.reliable_time_period.days,
|
|
help=("The time period to analyze when determining if a test has become reliable. Defaults"
|
|
" to %default day(s)."))
|
|
|
|
model_options.add_option(
|
|
"--unreliableDays", type="int", dest="unreliable_days", metavar="<ndays>",
|
|
default=DEFAULT_CONFIG.unreliable_time_period.days,
|
|
help=("The time period to analyze when determining if a test has become unreliable."
|
|
" Defaults to %default day(s)."))
|
|
|
|
parser.add_option("--resmokeTagFile", dest="tag_file", metavar="<tagfile>",
|
|
default="etc/test_lifecycle.yml",
|
|
help=("The resmoke.py tag file to update. If --metadataRepo is specified, it"
|
|
" is the relative path in the metadata repository, otherwise it can be"
|
|
" an absolute path or a relative path from the current directory."
|
|
" Defaults to '%default'."))
|
|
|
|
parser.add_option("--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>",
|
|
default="git@github.com:mongodb/mongo-test-metadata.git",
|
|
help=("The repository that contains the lifecycle file. "
|
|
"It will be cloned in the current working directory. "
|
|
"Defaults to '%default'."))
|
|
|
|
parser.add_option("--referencesFile", dest="references_file", metavar="<references-file>",
|
|
default="references.yml",
|
|
help=("The YAML file in the metadata repository that contains the revision "
|
|
"mappings. Defaults to '%default'."))
|
|
|
|
parser.add_option("--requestThreads", type="int", dest="num_request_threads",
|
|
metavar="<num-request-threads>", default=DEFAULT_NUM_THREADS,
|
|
help=("The maximum number of threads to use when querying the Evergreen API."
|
|
" Batches are processed sequentially but the test history is queried in"
|
|
" parallel for each task. Defaults to %default."))
|
|
|
|
commit_options = optparse.OptionGroup(
|
|
parser, title="Commit options",
|
|
description=("Options used to configure whether and how to commit the updated test"
|
|
" lifecycle tags."))
|
|
parser.add_option_group(commit_options)
|
|
|
|
commit_options.add_option("--commit", action="store_true", dest="commit", default=False,
|
|
help="Indicates that the updated tag file should be committed.")
|
|
|
|
commit_options.add_option(
|
|
"--jiraConfig", dest="jira_config", metavar="<jira-config>", default=None,
|
|
help=("The YAML file containing the JIRA access configuration ('user', 'password',"
|
|
"'server')."))
|
|
|
|
commit_options.add_option(
|
|
"--gitUserName", dest="git_user_name", metavar="<git-user-name>", default="Test Lifecycle",
|
|
help=("The git user name that will be set before committing to the metadata repository."
|
|
" Defaults to '%default'."))
|
|
|
|
commit_options.add_option(
|
|
"--gitUserEmail", dest="git_user_email", metavar="<git-user-email>",
|
|
default="buil+testlifecycle@mongodb.com",
|
|
help=("The git user email address that will be set before committing to the metadata"
|
|
" repository. Defaults to '%default'."))
|
|
|
|
logging_options = optparse.OptionGroup(
|
|
parser, title="Logging options",
|
|
description="Options used to configure the logging output of the script.")
|
|
parser.add_option_group(logging_options)
|
|
|
|
logging_options.add_option("--logLevel", dest="log_level", metavar="<log-level>", choices=[
|
|
"DEBUG", "INFO", "WARNING", "ERROR"
|
|
], default="INFO", help=("The log level. Accepted values are: DEBUG, INFO, WARNING and ERROR."
|
|
" Defaults to '%default'."))
|
|
|
|
logging_options.add_option(
|
|
"--logFile", dest="log_file", metavar="<log-file>", default=None,
|
|
help="The destination file for the logs output. Defaults to the standard output.")
|
|
|
|
(options, tests) = parser.parse_args()
|
|
|
|
logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=options.log_level,
|
|
filename=options.log_file)
|
|
evg_conf = ci_evergreen.parse_evergreen_file(options.evergreen_project_config)
|
|
use_test_tasks_membership = False
|
|
|
|
tasks = options.tasks.split(",") if options.tasks else []
|
|
if not tasks:
|
|
# If no tasks are specified, then the list of tasks is all.
|
|
tasks = evg_conf.lifecycle_task_names
|
|
use_test_tasks_membership = True
|
|
|
|
variants = options.variants.split(",") if options.variants else []
|
|
|
|
distros = options.distros.split(",") if options.distros else []
|
|
|
|
config = Config(
|
|
test_fail_rates=Rates(*options.test_fail_rates),
|
|
task_fail_rates=Rates(*options.task_fail_rates),
|
|
variant_fail_rates=Rates(*options.variant_fail_rates),
|
|
distro_fail_rates=Rates(*options.distro_fail_rates),
|
|
reliable_min_runs=options.reliable_test_min_runs,
|
|
reliable_time_period=datetime.timedelta(days=options.reliable_days),
|
|
unreliable_min_runs=options.unreliable_test_min_runs,
|
|
unreliable_time_period=datetime.timedelta(days=options.unreliable_days))
|
|
validate_config(config)
|
|
|
|
lifecycle_tags_file = make_lifecycle_tags_file(options, config)
|
|
if not lifecycle_tags_file:
|
|
sys.exit(1)
|
|
|
|
test_tasks_membership = get_test_tasks_membership(evg_conf)
|
|
# If no tests are specified then the list of tests is generated from the list of tasks.
|
|
if not tests:
|
|
tests = get_tests_from_tasks(tasks, test_tasks_membership)
|
|
if not options.tasks:
|
|
use_test_tasks_membership = True
|
|
|
|
test_history_source = TestHistorySource(
|
|
options.project, variants, distros, config.reliable_time_period,
|
|
config.unreliable_time_period, options.num_request_threads)
|
|
test_tasks_list = []
|
|
for test in tests:
|
|
# Find all associated tasks for the test if tasks or tests were not specified.
|
|
if use_test_tasks_membership:
|
|
tasks = test_tasks_membership[test]
|
|
if not tasks:
|
|
LOGGER.warning("No tasks found for tests %s, skipping this group.", tests)
|
|
continue
|
|
test_tasks_list.append((test, tasks))
|
|
results = test_history_source.get_history(test_tasks_list)
|
|
|
|
LOGGER.info("Updating the tags")
|
|
nb_tests = len(test_tasks_list)
|
|
count = 0
|
|
for test_history in results:
|
|
progress = round(100 * count / nb_tests, 2)
|
|
LOGGER.info("Progress: %s %%", progress)
|
|
update_tags(lifecycle_tags_file.changelog_lifecycle, config, test_history)
|
|
count += 1
|
|
|
|
# Remove tags that are no longer relevant
|
|
clean_up_tags(lifecycle_tags_file.changelog_lifecycle, evg_conf)
|
|
|
|
# We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there
|
|
# have been changes to the tags. In particular, we avoid modifying the file when only the header
|
|
# comment for the YAML file would change.
|
|
if lifecycle_tags_file.is_modified():
|
|
lifecycle_tags_file.write()
|
|
|
|
if options.commit:
|
|
commit_ok = lifecycle_tags_file.commit()
|
|
if not commit_ok:
|
|
sys.exit(1)
|
|
else:
|
|
LOGGER.info("The tags have not been modified.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|