816 lines
31 KiB
Python
816 lines
31 KiB
Python
"""Unit tests for buildscripts/metrics/burn_in_tests.py."""
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import copy
|
|
import datetime
|
|
import unittest
|
|
|
|
import requests
|
|
|
|
from mock import Mock, MagicMock, patch
|
|
|
|
import buildscripts.metrics.burn_in_tests as burn_in
|
|
import buildscripts.client.evergreen as evg_client
|
|
|
|
# pylint: disable=missing-docstring,protected-access
|
|
|
|
BURN_IN = "buildscripts.metrics.burn_in_tests"
|
|
EVERGREEN = "buildscripts.client.evergreen"
|
|
|
|
PROJECT_PATCHES = {
|
|
"myproject": [
|
|
{
|
|
"patch_id": "patch1",
|
|
"create_time": "2019-01-01T00:00:00.000Z",
|
|
"status": "failed"
|
|
},
|
|
{
|
|
"patch_id": "patch2",
|
|
"create_time": "2019-03-01T00:00:00.000Z",
|
|
"status": "failed"
|
|
},
|
|
{
|
|
"patch_id": "patch_only_burn_failure",
|
|
"create_time": "2019-04-01T00:00:00.000Z",
|
|
"status": "failed"
|
|
},
|
|
{
|
|
"patch_id": "patch4",
|
|
"create_time": "2019-04-01T00:00:00.000Z",
|
|
"status": "created"
|
|
},
|
|
{
|
|
"patch_id": "patch5",
|
|
"create_time": "2019-04-01T00:00:00.000Z",
|
|
"status": "succeeded"
|
|
}
|
|
],
|
|
"another_project": [
|
|
{
|
|
"patch_id": "patch1",
|
|
"create_time": "2019-01-01T00:00:00.000Z",
|
|
"status": "failed"
|
|
},
|
|
{
|
|
"patch_id": "patch2",
|
|
"create_time": "2019-03-01T00:00:00.000Z",
|
|
"status": "failed"
|
|
},
|
|
{
|
|
"patch_id": "apatch3",
|
|
"create_time": "2019-04-01T00:00:00.000Z",
|
|
"status": "failed"
|
|
},
|
|
{
|
|
"patch_id": "patch5",
|
|
"create_time": "1900-04-01T00:00:00.000Z",
|
|
"status": "succeeded"
|
|
}
|
|
]
|
|
} # yapf: disable
|
|
|
|
VERSION_BUILD1 = {
|
|
"_id": "build1",
|
|
"version": "patch1",
|
|
"create_time": "2019-01-01T00:00:00.000Z",
|
|
"status": "failed",
|
|
"status_counts": {"succeeded": 2, "failed": 2},
|
|
"build_variant": "variant1",
|
|
"tasks": ["build1mytask1a", "build1mytask2a", "build1myburn_in_tests_here", "build1mytask3a"]
|
|
} # yapf: disable
|
|
VERSION_BUILD2 = {
|
|
"_id": "build2",
|
|
"version": "patch1",
|
|
"create_time": "2019-02-01T00:00:00.000Z",
|
|
"status": "failed",
|
|
"status_counts": {"succeeded": 1, "failed": 2},
|
|
"build_variant": "variant2",
|
|
"tasks": ["build2mytask1b", "build2mytask2b", "build2mytask3b"]
|
|
} # yapf: disable
|
|
VERSION_BUILD3 = {
|
|
"_id": "build3",
|
|
"version": "patch2",
|
|
"create_time": "2019-03-01T00:00:00.000Z",
|
|
"status": "failed",
|
|
"status_counts": {"succeeded": 3, "failed": 1},
|
|
"build_variant": "variant1",
|
|
"tasks": [
|
|
"build3mytask1c",
|
|
"build3myburn_in_tests_yes",
|
|
"build3myburn_in_tests_yes2",
|
|
"build3mytask3c"
|
|
]
|
|
} # yapf: disable
|
|
VERSION_BUILD4 = {
|
|
"_id": "build4",
|
|
"version": "patch2",
|
|
"create_time": "2019-04-01T00:00:00.000Z",
|
|
"status": "failed",
|
|
"status_counts": {"succeeded": 3, "failed": 1},
|
|
"build_variant": "variant3",
|
|
"tasks": ["build4mytask1c", "build4mytask2c", "build4myburn_in_tests_1", "build4mytask3c"],
|
|
} # yapf: disable
|
|
VERSION_BUILD5 = {
|
|
"_id": "build5",
|
|
"version": "patch_only_burn_failure",
|
|
"create_time": "2019-04-01T00:00:00.000Z",
|
|
"status": "failed",
|
|
"status_counts": {"succeeded": 1, "failed": 0},
|
|
"build_variant": "variant1",
|
|
"tasks": ["build5task"]
|
|
} # yapf: disable
|
|
VERSION_BUILD6 = {
|
|
"_id": "build6",
|
|
"version": "patch_only_burn_failure",
|
|
"create_time": "2019-04-01T00:00:00.000Z",
|
|
"status": "failed",
|
|
"status_counts": {"succeeded": 2, "failed": 1},
|
|
"build_variant": "variant3",
|
|
"tasks": ["build6task1", "build6task2", "build6anotherburn_in_tests"]
|
|
} # yapf: disable
|
|
VERSION_BUILD7 = {
|
|
"_id": "build7",
|
|
"version": "patch5",
|
|
"create_time": "2019-04-01T00:00:00.000Z",
|
|
"status": "success",
|
|
"status_counts": {"succeeded": 2, "failed": 0},
|
|
"build_variant": "variant3",
|
|
"tasks": ["build7anotherburn_in_tests", "build7task"]
|
|
} # yapf: disable
|
|
|
|
VERSION_BUILDS = {
|
|
"patch1": [VERSION_BUILD1, VERSION_BUILD2],
|
|
"patch2": [VERSION_BUILD3, VERSION_BUILD4],
|
|
"patch_only_burn_failure": [VERSION_BUILD5, VERSION_BUILD6],
|
|
"patch4": [],
|
|
"patch5": [VERSION_BUILD7]
|
|
} # yapf: disable
|
|
|
|
BUILDS_WITH_BURN_IN = [{"_id": "build1"}, {"_id": "build3"}, {"_id": "build4"}, {"_id": "build5"},
|
|
{"_id": "build6"}]
|
|
BUILDS = BUILDS_WITH_BURN_IN + [{"_id": "build2"}]
|
|
|
|
BUILD1_TASKS = [
|
|
{"task_id": "build1mytask1a",
|
|
"execution": 0,
|
|
"display_name": "task1",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build1mytask2a",
|
|
"execution": 0,
|
|
"display_name": "task2",
|
|
"status": "failed",
|
|
"time_taken_ms": 200
|
|
},
|
|
{"task_id": "build1myburn_in_tests_here",
|
|
"execution": 0,
|
|
"display_name": burn_in.BURN_IN_GENERATED_TASK_PREFIX + "thistask",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 300
|
|
},
|
|
{"task_id": "build1mytask3a",
|
|
"execution": 0,
|
|
"display_name": "task3",
|
|
"status": "failed",
|
|
"time_taken_ms": 100
|
|
}
|
|
] # yapf: disable
|
|
BUILD2_TASKS = [
|
|
{"task_id": "build2mytask1b",
|
|
"execution": 0,
|
|
"display_name": "task1",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build2mytask2b",
|
|
"execution": 0,
|
|
"display_name": "task2",
|
|
"status": "failed",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build2mytask3b",
|
|
"execution": 0,
|
|
"display_name": "task3",
|
|
"status": "failed",
|
|
"time_taken_ms": 100
|
|
}
|
|
] # yapf: disable
|
|
BUILD3_TASKS = [
|
|
{"task_id": "build3mytask1c",
|
|
"execution": 0,
|
|
"display_name": "task1",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build3myburn_in_tests_yes",
|
|
"execution": 0,
|
|
"display_name": burn_in.BURN_IN_GENERATED_TASK_PREFIX + "anothertask1",
|
|
"status": "succeeded",
|
|
"time_taken_ms": burn_in.BURN_IN_TIME_MS - 1
|
|
},
|
|
{"task_id": "build3myburn_in_tests_yes2",
|
|
"execution": 0,
|
|
"display_name": burn_in.BURN_IN_GENERATED_TASK_PREFIX + "anothertask2",
|
|
"status": "failed",
|
|
"time_taken_ms": burn_in.BURN_IN_TIME_MS + 1
|
|
},
|
|
{"task_id": "build3mytask3c",
|
|
"execution": 0,
|
|
"display_name": "burn_in_tests_not",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
}
|
|
] # yapf: disable
|
|
BUILD4_TASKS = [
|
|
{"task_id": "build4mytask1c",
|
|
"execution": 0,
|
|
"display_name": "task1",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build4mytask2c",
|
|
"execution": 0,
|
|
"display_name": "task2",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build4myburn_in_tests_1",
|
|
"execution": 1,
|
|
"display_name": burn_in.BURN_IN_GENERATED_TASK_PREFIX + "anothertask",
|
|
"status": "failed",
|
|
"time_taken_ms": 200
|
|
},
|
|
{"task_id": "build4mytask3c",
|
|
"execution": 0,
|
|
"display_name": "burn_in_tests_not",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
}
|
|
] # yapf: disable
|
|
BUILD5_TASKS = [
|
|
{"task_id": "build5task",
|
|
"execution": 0,
|
|
"display_name": "task4",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
] # yapf: disable
|
|
BUILD6_TASKS = [
|
|
{"task_id": "build6task1",
|
|
"execution": 0,
|
|
"display_name": "task5",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build6task2",
|
|
"execution": 0,
|
|
"display_name": "task6",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build6anotherburn_in_tests",
|
|
"execution": 1,
|
|
"display_name": burn_in.BURN_IN_GENERATED_TASK_PREFIX + "anothertask6",
|
|
"status": "failed",
|
|
"time_taken_ms": burn_in.BURN_IN_TIME_MS + 1
|
|
}
|
|
] # yapf: disable
|
|
BUILD7_TASKS = [
|
|
{"task_id": "build7task",
|
|
"execution": 0,
|
|
"display_name": "task1",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 100
|
|
},
|
|
{"task_id": "build7anotherburn_in_tests",
|
|
"execution": 1,
|
|
"display_name": burn_in.BURN_IN_GENERATED_TASK_PREFIX + "anothertask7",
|
|
"status": "succeeded",
|
|
"time_taken_ms": 200
|
|
}
|
|
] # yapf: disable
|
|
|
|
BUILD_TASKS_WITH_BURN_IN_LIST = BUILD1_TASKS + BUILD3_TASKS + BUILD4_TASKS + BUILD5_TASKS + BUILD6_TASKS + BUILD7_TASKS
|
|
|
|
BUILD_TASKS_WITH_BURN_IN = [
|
|
{"_id": "build1", "tasks": BUILD1_TASKS},
|
|
{"_id": "build3", "tasks": BUILD3_TASKS},
|
|
{"_id": "build4", "tasks": BUILD4_TASKS},
|
|
{"_id": "build5", "tasks": BUILD5_TASKS},
|
|
{"_id": "build6", "tasks": BUILD6_TASKS},
|
|
{"_id": "build7", "tasks": BUILD7_TASKS},
|
|
] # yapf: disable
|
|
|
|
BUILD_TASKS = BUILD_TASKS_WITH_BURN_IN + [
|
|
{"_id": "build2", "tasks": BUILD2_TASKS}
|
|
] # yapf: disable
|
|
|
|
TASKS_TESTS = [
|
|
{"task_id": "build1myburn_in_tests_here", "execution": 0, "tests": ["test1.js", "test2.js"]},
|
|
{"task_id": "build3myburn_in_tests_yes", "execution": 0, "tests": []},
|
|
{"task_id": "build3myburn_in_tests_yes2", "execution": 0, "tests": ["test1.js"]},
|
|
{"task_id": "build4myburn_in_tests_1", "execution": 1, "tests": ["test1.js", "test2.js"]},
|
|
{"task_id": "build6anotherburn_in_tests", "execution": 3, "tests": ["test1.js"]},
|
|
{"task_id": "build7anotherburn_in_tests", "execution": 7, "tests": ["test1.js"]},
|
|
] # yapf: disable
|
|
|
|
|
|
def raise_http_error(code):
|
|
response = requests.Response()
|
|
response.status_code = code
|
|
response.raise_for_status()
|
|
|
|
|
|
class TestParseCommandLine(unittest.TestCase):
|
|
def test_parse_command_line(self):
|
|
options = burn_in.parse_command_line().parse_args([])
|
|
self.assertEqual(options.days, burn_in.DEFAULT_DAYS)
|
|
self.assertEqual(options.project, burn_in.DEFAULT_PROJECT)
|
|
self.assertEqual(options.report_file, burn_in.DEFAULT_REPORT_FILE)
|
|
self.assertIsNone(options.log_level)
|
|
self.assertIsNone(options.evg_client_log_level)
|
|
|
|
def test_parse_command_line_partial_args(self):
|
|
days = 5
|
|
project = "myproject"
|
|
arg_list = ["--days", str(days), "--project", project]
|
|
options = burn_in.parse_command_line().parse_args(arg_list)
|
|
self.assertEqual(options.days, days)
|
|
self.assertEqual(options.project, project)
|
|
self.assertEqual(options.report_file, burn_in.DEFAULT_REPORT_FILE)
|
|
self.assertIsNone(options.log_level)
|
|
self.assertIsNone(options.evg_client_log_level)
|
|
|
|
def test_parse_command_line_all_args(self):
|
|
days = 5
|
|
project = "myproject"
|
|
report_file = "myreport.json"
|
|
log_level = "INFO"
|
|
evg_client_log_level = "DEBUG"
|
|
arg_list = [
|
|
"--days", str(days),
|
|
"--project", project,
|
|
"--report", report_file,
|
|
"--log", log_level,
|
|
"--evgClientLog", evg_client_log_level
|
|
] # yapf: disable
|
|
parser_args = burn_in.parse_command_line().parse_args(arg_list)
|
|
self.assertEqual(parser_args.days, days)
|
|
self.assertEqual(parser_args.project, project)
|
|
self.assertEqual(parser_args.report_file, report_file)
|
|
self.assertEqual(parser_args.log_level, log_level)
|
|
self.assertEqual(parser_args.evg_client_log_level, evg_client_log_level)
|
|
|
|
|
|
class TestConfigureLogging(unittest.TestCase):
|
|
def test_configure_logging(self):
|
|
log_level = 15
|
|
evg_client_log_level = 20
|
|
burn_in.configure_logging(log_level, evg_client_log_level)
|
|
self.assertEqual(burn_in.LOGGER.getEffectiveLevel(), log_level)
|
|
self.assertEqual(evg_client.LOGGER.getEffectiveLevel(), evg_client_log_level)
|
|
|
|
def test_configure_logging_log_level(self):
|
|
log_level = 15
|
|
evg_client_log_level = evg_client.LOGGER.getEffectiveLevel()
|
|
burn_in.configure_logging(log_level, None)
|
|
self.assertEqual(burn_in.LOGGER.getEffectiveLevel(), log_level)
|
|
self.assertEqual(evg_client.LOGGER.getEffectiveLevel(), evg_client_log_level)
|
|
|
|
def test_configure_logging_evg_client_log_level(self):
|
|
log_level = burn_in.LOGGER.getEffectiveLevel()
|
|
evg_client_log_level = 30
|
|
burn_in.configure_logging(None, evg_client_log_level)
|
|
self.assertEqual(burn_in.LOGGER.getEffectiveLevel(), log_level)
|
|
self.assertEqual(evg_client.LOGGER.getEffectiveLevel(), evg_client_log_level)
|
|
|
|
def test_configure_logging_default(self):
|
|
log_level = burn_in.LOGGER.getEffectiveLevel()
|
|
evg_client_log_level = evg_client.LOGGER.getEffectiveLevel()
|
|
burn_in.configure_logging(None, None)
|
|
self.assertEqual(burn_in.LOGGER.getEffectiveLevel(), log_level)
|
|
self.assertEqual(evg_client.LOGGER.getEffectiveLevel(), evg_client_log_level)
|
|
|
|
|
|
class TestWriteJsonFile(unittest.TestCase):
|
|
def test_write_json_file(self):
|
|
my_data = {"key1": "val1", "key_list": ["l1", "l2"]}
|
|
path = "myfile"
|
|
with patch("builtins.open") as mock_file,\
|
|
patch("json.dump") as mock_json_dump:
|
|
burn_in.write_json_file(my_data, path)
|
|
mock_file.assert_called_once_with("myfile", "w")
|
|
mock_json_dump.assert_called_once()
|
|
self.assertDictEqual(mock_json_dump.call_args_list[0][0][0], my_data)
|
|
|
|
|
|
class TestStrToDatetime(unittest.TestCase):
|
|
def test_str_to_datetime(self):
|
|
date_str = "2019-01-01T10:03:33.190Z"
|
|
self.assertEqual(
|
|
burn_in.str_to_datetime(date_str), datetime.datetime(2019, 1, 1, 10, 3, 33, 190000))
|
|
self.assertNotEqual(burn_in.str_to_datetime(date_str), datetime.datetime(2019, 1, 1, 10))
|
|
|
|
|
|
class TestStrToDatimeDate(unittest.TestCase):
|
|
def test_datetime_date(self):
|
|
date_str = "2019-01-01T10:01:22.857Z"
|
|
self.assertEqual(
|
|
burn_in.str_to_datetime_date(date_str),
|
|
datetime.datetime(2019, 1, 1).date())
|
|
|
|
|
|
class Projects(object):
|
|
def __init__(self, patches):
|
|
self.patches = patches
|
|
|
|
def _project_patches_gen(self, project):
|
|
if project not in self.patches:
|
|
raise requests.exceptions.HTTPError
|
|
for project_patch in self.patches[project]:
|
|
yield project_patch
|
|
|
|
|
|
class VersionBuilds(object):
|
|
def __init__(self, version_builds):
|
|
self.version_builds = version_builds
|
|
|
|
def _version_builds(self, patch_id):
|
|
if patch_id in self.version_builds:
|
|
return self.version_builds[patch_id]
|
|
return []
|
|
|
|
|
|
class BuildTasks(object):
|
|
def __init__(self, build_tasks):
|
|
self.build_tasks = build_tasks
|
|
|
|
def _tasks_by_build_id(self, build_id):
|
|
for build_task in self.build_tasks:
|
|
if build_id == build_task["_id"]:
|
|
return build_task["tasks"]
|
|
return []
|
|
|
|
|
|
class TaskTests(object):
|
|
def __init__(self, tasks_tests, http_error_code=404):
|
|
self.tasks_tests = tasks_tests
|
|
self.http_error_code = http_error_code
|
|
self.http_errors = 0
|
|
|
|
def _tests_by_task(self, task_id, execution):
|
|
for task in self.tasks_tests:
|
|
if task["task_id"] == task_id and task["execution"] == execution:
|
|
return task["tests"]
|
|
self.http_errors += 1
|
|
raise_http_error(self.http_error_code)
|
|
return []
|
|
|
|
|
|
class TestGetBurnInBuilds(unittest.TestCase):
|
|
def test_get_burn_in_builds(self):
|
|
projects = Projects(PROJECT_PATCHES)
|
|
version_builds = VersionBuilds(VERSION_BUILDS)
|
|
evg_api = Mock()
|
|
evg_api.project_patches_gen = projects._project_patches_gen
|
|
evg_api.version_builds = version_builds._version_builds
|
|
project = "myproject"
|
|
days = 30000
|
|
burn_in_builds = burn_in.get_burn_in_builds(evg_api, project, days)
|
|
self.assertEqual(5, len(burn_in_builds))
|
|
|
|
def test_get_burn_in_builds_partial_patches(self):
|
|
projects = Projects(PROJECT_PATCHES)
|
|
version_builds = VersionBuilds(VERSION_BUILDS)
|
|
evg_api = Mock()
|
|
evg_api.project_patches_gen = projects._project_patches_gen
|
|
evg_api.version_builds = version_builds._version_builds
|
|
project = "another_project"
|
|
days = 30000
|
|
burn_in_builds = burn_in.get_burn_in_builds(evg_api, project, days)
|
|
self.assertEqual(3, len(burn_in_builds))
|
|
days = 300000 # Go further back in time to pull in more patch builds.
|
|
burn_in_builds = burn_in.get_burn_in_builds(evg_api, project, days)
|
|
self.assertEqual(4, len(burn_in_builds))
|
|
|
|
def test_get_burn_in_builds_no_patches(self):
|
|
version_builds = VersionBuilds(VERSION_BUILDS)
|
|
evg_api = Mock()
|
|
evg_api.project_patches_gen = lambda _: []
|
|
evg_api.version_builds = version_builds._version_builds
|
|
project = "myproject"
|
|
days = 30000
|
|
burn_in_builds = burn_in.get_burn_in_builds(evg_api, project, days)
|
|
self.assertEqual(0, len(burn_in_builds))
|
|
|
|
def test_get_burn_in_builds_no_patches_days(self):
|
|
projects = Projects(PROJECT_PATCHES)
|
|
version_builds = VersionBuilds(VERSION_BUILDS)
|
|
evg_api = Mock()
|
|
evg_api.project_patches_gen = projects._project_patches_gen
|
|
evg_api.version_builds = version_builds._version_builds
|
|
project = "myproject"
|
|
days = 0
|
|
burn_in_builds = burn_in.get_burn_in_builds(evg_api, project, days)
|
|
self.assertEqual(0, len(burn_in_builds))
|
|
|
|
def test_get_burn_in_builds_missing_patch(self):
|
|
projects = Projects(PROJECT_PATCHES)
|
|
version_builds = VersionBuilds(VERSION_BUILDS)
|
|
evg_api = Mock()
|
|
evg_api.project_patches_gen = projects._project_patches_gen
|
|
evg_api.version_builds = version_builds._version_builds
|
|
project = "another_project"
|
|
days = 30000
|
|
burn_in_builds = burn_in.get_burn_in_builds(evg_api, project, days)
|
|
self.assertEqual(3, len(burn_in_builds))
|
|
|
|
|
|
class TestIsBurnInDisplayTask(unittest.TestCase):
|
|
def test_is_burn_in_display_task(self):
|
|
self.assertTrue(burn_in.is_burn_in_display_task("burn_in_tests"))
|
|
self.assertTrue(burn_in.is_burn_in_display_task("burn_in:mytask"))
|
|
self.assertFalse(burn_in.is_burn_in_display_task("burn_in_test"))
|
|
|
|
|
|
class TestGetBurnInTasks(unittest.TestCase):
|
|
def test_get_burn_in_tasks(self):
|
|
builds = BUILDS_WITH_BURN_IN
|
|
build_tasks = BuildTasks(BUILD_TASKS_WITH_BURN_IN)
|
|
evg_api = Mock()
|
|
evg_api.tasks_by_build_id = build_tasks._tasks_by_build_id
|
|
burn_in_tasks = burn_in.get_burn_in_tasks(evg_api, builds)
|
|
self.assertEqual(5, len(burn_in_tasks))
|
|
for task in burn_in_tasks:
|
|
self.assertTrue(task["display_name"].startswith(burn_in.BURN_IN_GENERATED_TASK_PREFIX))
|
|
|
|
def test_get_burn_in_tasks_no_builds(self):
|
|
build_tasks = BuildTasks(BUILD_TASKS_WITH_BURN_IN)
|
|
evg_api = Mock()
|
|
evg_api.tasks_by_build_id = build_tasks._tasks_by_build_id
|
|
burn_in_tasks = burn_in.get_burn_in_tasks(evg_api, [])
|
|
self.assertEqual(0, len(burn_in_tasks))
|
|
|
|
def test_get_burn_in_tasks_missing_build(self):
|
|
builds = BUILDS
|
|
build_tasks = BuildTasks(BUILD_TASKS_WITH_BURN_IN)
|
|
evg_api = Mock()
|
|
evg_api.tasks_by_build_id = build_tasks._tasks_by_build_id
|
|
burn_in_tasks = burn_in.get_burn_in_tasks(evg_api, builds)
|
|
self.assertEqual(5, len(burn_in_tasks))
|
|
|
|
|
|
class TestGetTestsFromTasks(unittest.TestCase):
|
|
def test_get_tests(self):
|
|
test_tasks = TaskTests(TASKS_TESTS)
|
|
evg_api = Mock()
|
|
evg_api.tests_by_task = test_tasks._tests_by_task
|
|
tests = burn_in.get_tests_from_tasks(evg_api, BUILD_TASKS_WITH_BURN_IN_LIST)
|
|
self.assertEqual(5, len(tests))
|
|
for test in tests:
|
|
self.assertIn(test, ["test1.js", "test2.js"])
|
|
|
|
def test_get_tests_no_tasks(self):
|
|
test_tasks = TaskTests(TASKS_TESTS)
|
|
evg_api = Mock()
|
|
evg_api.tests_by_task = test_tasks._tests_by_task
|
|
tests = burn_in.get_tests_from_tasks(evg_api, [])
|
|
self.assertEqual(0, len(tests))
|
|
|
|
def test_get_tests_missing_task(self):
|
|
test_tasks = TaskTests(TASKS_TESTS)
|
|
evg_api = Mock()
|
|
evg_api.tests_by_task = test_tasks._tests_by_task
|
|
tests = burn_in.get_tests_from_tasks(evg_api, BUILD2_TASKS)
|
|
self.assertEqual(0, len(tests))
|
|
|
|
def test_get_tests_uncaught_http_error(self):
|
|
http_error_code = 401
|
|
test_tasks = TaskTests(TASKS_TESTS, http_error_code=http_error_code)
|
|
evg_api = Mock()
|
|
evg_api.tests_by_task = test_tasks._tests_by_task
|
|
with self.assertRaises(requests.exceptions.HTTPError) as err:
|
|
burn_in.get_tests_from_tasks(evg_api, BUILD4_TASKS)
|
|
self.assertEqual(1, test_tasks.http_errors)
|
|
self.assertEqual(http_error_code, err.response.status_code)
|
|
|
|
|
|
class TestReport(unittest.TestCase):
|
|
@staticmethod
|
|
def _get_burn_in_builds():
|
|
return [
|
|
VERSION_BUILD1, VERSION_BUILD3, VERSION_BUILD4, VERSION_BUILD5, VERSION_BUILD6,
|
|
VERSION_BUILD7
|
|
]
|
|
|
|
@staticmethod
|
|
def _get_burn_in_tasks():
|
|
return [
|
|
task for task in BUILD_TASKS_WITH_BURN_IN_LIST
|
|
if task["display_name"].startswith(burn_in.BURN_IN_GENERATED_TASK_PREFIX)
|
|
]
|
|
|
|
def test__init_burn_in_patch_builds(self):
|
|
burn_in_patch_builds = burn_in.Report._init_burn_in_patch_builds(
|
|
[VERSION_BUILD1, VERSION_BUILD3])
|
|
self.assertEqual(2, len(burn_in_patch_builds))
|
|
self.assertIn("patch1", burn_in_patch_builds)
|
|
self.assertEqual(1, len(burn_in_patch_builds["patch1"]["builds"]))
|
|
self.assertDictEqual(VERSION_BUILD1, burn_in_patch_builds["patch1"]["builds"][0])
|
|
self.assertEqual(1, len(burn_in_patch_builds["patch2"]["builds"]))
|
|
self.assertDictEqual(VERSION_BUILD3, burn_in_patch_builds["patch2"]["builds"][0])
|
|
|
|
def test__init_burn_in_patch_multiple_builds(self):
|
|
burn_in_patch_builds = burn_in.Report._init_burn_in_patch_builds(
|
|
[VERSION_BUILD3, VERSION_BUILD4])
|
|
self.assertEqual(1, len(burn_in_patch_builds))
|
|
self.assertIn("patch2", burn_in_patch_builds)
|
|
self.assertEqual(2, len(burn_in_patch_builds["patch2"]["builds"]))
|
|
self.assertDictEqual(VERSION_BUILD3, burn_in_patch_builds["patch2"]["builds"][0])
|
|
self.assertDictEqual(VERSION_BUILD4, burn_in_patch_builds["patch2"]["builds"][1])
|
|
|
|
def test__init_burn_in_patch_no_builds(self):
|
|
burn_in_patch_builds = burn_in.Report._init_burn_in_patch_builds([])
|
|
self.assertDictEqual({}, burn_in_patch_builds)
|
|
|
|
def test__init_burn_in_tasks(self):
|
|
tasks = [
|
|
{"task_id": "task1", "tests": ["a", "b"]},
|
|
{"task_id": "task2", "tests": []},
|
|
{"task_id": "task3", "tests": ["x", "y"]},
|
|
]
|
|
burn_in_tasks = burn_in.Report._init_burn_in_tasks(tasks)
|
|
self.assertEqual(3, len(burn_in_tasks))
|
|
for task in tasks:
|
|
self.assertIn(task["task_id"], burn_in_tasks)
|
|
self.assertDictEqual(task, burn_in_tasks[task["task_id"]])
|
|
|
|
def test__init_burn_in_tasks_no_tasks(self):
|
|
tasks = []
|
|
burn_in_tasks = burn_in.Report._init_burn_in_tasks(tasks)
|
|
self.assertDictEqual({}, burn_in_tasks)
|
|
|
|
def test__init_report_fields(self):
|
|
num_patch_builds = 3
|
|
num_burn_in_tasks = 1
|
|
num_tests = 4
|
|
comment = "my comment"
|
|
report = burn_in.Report._init_report_fields(num_patch_builds, num_burn_in_tasks, num_tests,
|
|
comment)
|
|
self.assertEqual(len(burn_in.REPORT_FIELDS) + 1, len(report))
|
|
for field in burn_in.REPORT_FIELDS:
|
|
self.assertIn(field, report)
|
|
for field in burn_in.REPORT_TIME_FIELDS:
|
|
self.assertIsNone(report[field])
|
|
self.assertEqual(0, report["tasks"])
|
|
self.assertEqual(0, report["tasks_succeeded"])
|
|
self.assertEqual(0, report["tasks_failed"])
|
|
self.assertEqual(0, report["tasks_failed_burn_in"])
|
|
self.assertEqual(0, report["tasks_failed_only_burn_in"])
|
|
self.assertEqual(0, report[burn_in.BURN_IN_TASKS_EXCEED])
|
|
self.assertEqual(num_burn_in_tasks, report["burn_in_generated_tasks"])
|
|
self.assertEqual(num_patch_builds, report["patch_builds_with_burn_in_task"])
|
|
self.assertEqual(num_tests, report["burn_in_tests"])
|
|
self.assertIn(burn_in.REPORT_COMMENT_FIELD, report)
|
|
self.assertEqual(comment, report[burn_in.REPORT_COMMENT_FIELD])
|
|
|
|
def test__init_report_fields_no_comment(self):
|
|
num_patch_builds = 13
|
|
num_burn_in_tasks = 11
|
|
num_tests = 14
|
|
report = burn_in.Report._init_report_fields(num_patch_builds, num_burn_in_tasks, num_tests)
|
|
self.assertEqual(len(burn_in.REPORT_FIELDS), len(report))
|
|
for field in burn_in.REPORT_FIELDS:
|
|
self.assertIn(field, report)
|
|
for field in burn_in.REPORT_TIME_FIELDS:
|
|
self.assertIsNone(report[field])
|
|
self.assertEqual(0, report["tasks"])
|
|
self.assertEqual(0, report["tasks_succeeded"])
|
|
self.assertEqual(0, report["tasks_failed"])
|
|
self.assertEqual(0, report["tasks_failed_burn_in"])
|
|
self.assertEqual(0, report["tasks_failed_only_burn_in"])
|
|
self.assertEqual(0, report[burn_in.BURN_IN_TASKS_EXCEED])
|
|
self.assertEqual(num_burn_in_tasks, report["burn_in_generated_tasks"])
|
|
self.assertEqual(num_patch_builds, report["patch_builds_with_burn_in_task"])
|
|
self.assertEqual(num_tests, report["burn_in_tests"])
|
|
self.assertNotIn(burn_in.REPORT_COMMENT_FIELD, report)
|
|
|
|
def test_generate_report(self):
|
|
builds = self._get_burn_in_builds()
|
|
tasks = self._get_burn_in_tasks()
|
|
tests = ["test1.js", "test2.js", "test3.js", "test4.js", "test5.js"]
|
|
burn_in_report = burn_in.Report(builds, tasks, tests)
|
|
report = burn_in_report.generate_report()
|
|
self.assertEqual(len(burn_in.REPORT_FIELDS), len(report))
|
|
for field in burn_in.REPORT_FIELDS:
|
|
self.assertIn(field, report)
|
|
self.assertEqual(18, report["tasks"])
|
|
self.assertEqual(13, report["tasks_succeeded"])
|
|
self.assertEqual(5, report["tasks_failed"])
|
|
self.assertEqual(3, report["tasks_failed_burn_in"])
|
|
self.assertEqual(2, report["tasks_failed_only_burn_in"])
|
|
self.assertEqual(6, report["burn_in_generated_tasks"])
|
|
self.assertEqual(4, report["patch_builds_with_burn_in_task"])
|
|
self.assertEqual(len(tests), report["burn_in_tests"])
|
|
self.assertEqual(2, report[burn_in.BURN_IN_TASKS_EXCEED])
|
|
self.assertEqual("2019-01-01T00:00:00.000Z", report["report_start_time"])
|
|
self.assertEqual("2019-04-01T00:00:00.000Z", report["report_end_time"])
|
|
|
|
def test_generate_report_with_comment(self):
|
|
builds = self._get_burn_in_builds()
|
|
tasks = self._get_burn_in_tasks()
|
|
tests = ["test1.js", "test2.js", "test3.js", "test4.js", "test5.js"]
|
|
comment = "my_comment"
|
|
burn_in_report = burn_in.Report(builds, tasks, tests, comment=comment)
|
|
report = burn_in_report.generate_report()
|
|
self.assertEqual(len(burn_in.REPORT_FIELDS) + 1, len(report))
|
|
for field in burn_in.REPORT_FIELDS:
|
|
self.assertIn(field, report)
|
|
self.assertIn(burn_in.REPORT_COMMENT_FIELD, report)
|
|
self.assertEqual(comment, report[burn_in.REPORT_COMMENT_FIELD])
|
|
|
|
def test___update_report_time_no_time(self):
|
|
builds = self._get_burn_in_builds()
|
|
tasks = self._get_burn_in_tasks()
|
|
tests = ["test1.js", "test2.js", "test3.js", "test4.js", "test5.js"]
|
|
burn_in_report = burn_in.Report(builds, tasks, tests)
|
|
burn_in_report.report["report_start_time"] = None
|
|
create_time = "2019-01-01T00:00:00.000Z"
|
|
burn_in_report._update_report_time(create_time)
|
|
self.assertEqual(burn_in_report.report["report_start_time"], create_time)
|
|
self.assertEqual(burn_in_report.report["report_end_time"], create_time)
|
|
|
|
def test___update_report_time_no_endtime(self):
|
|
builds = self._get_burn_in_builds()
|
|
tasks = self._get_burn_in_tasks()
|
|
tests = ["test1.js", "test2.js", "test3.js", "test4.js", "test5.js"]
|
|
burn_in_report = burn_in.Report(builds, tasks, tests)
|
|
start_time = "2019-01-01T00:00:00.000Z"
|
|
burn_in_report.report["report_start_time"] = start_time
|
|
create_time = "2019-02-01T00:00:00.000Z"
|
|
burn_in_report._update_report_time(create_time)
|
|
self.assertEqual(burn_in_report.report["report_start_time"], start_time)
|
|
self.assertEqual(burn_in_report.report["report_end_time"], create_time)
|
|
|
|
def test___update_report_time_no_update(self):
|
|
builds = self._get_burn_in_builds()
|
|
tasks = self._get_burn_in_tasks()
|
|
tests = ["test1.js", "test2.js", "test3.js", "test4.js", "test5.js"]
|
|
burn_in_report = burn_in.Report(builds, tasks, tests)
|
|
start_time = "2019-01-01T00:00:00.000Z"
|
|
end_time = "2019-03-01T00:00:00.000Z"
|
|
burn_in_report.report["report_start_time"] = start_time
|
|
burn_in_report.report["report_end_time"] = end_time
|
|
create_time = "2019-02-01T00:00:00.000Z"
|
|
burn_in_report._update_report_time(create_time)
|
|
self.assertEqual(burn_in_report.report["report_start_time"], start_time)
|
|
self.assertEqual(burn_in_report.report["report_end_time"], end_time)
|
|
|
|
def test___update_report_burn_in(self):
|
|
builds = self._get_burn_in_builds()
|
|
tasks = self._get_burn_in_tasks()
|
|
tests = ["test1.js", "test2.js", "test3.js", "test4.js", "test5.js"]
|
|
burn_in_report = burn_in.Report(builds, tasks, tests)
|
|
burn_in_report._update_report_burn_in(
|
|
burn_in_report.burn_in_patch_builds["patch_only_burn_failure"]["builds"], 1)
|
|
self.assertEqual(burn_in_report.report["tasks_failed_burn_in"], 1)
|
|
self.assertEqual(burn_in_report.report["tasks_failed_only_burn_in"], 1)
|
|
self.assertEqual(burn_in_report.report[burn_in.BURN_IN_TASKS_EXCEED], 1)
|
|
|
|
def test___update_report_burn_in_no_task(self):
|
|
builds = self._get_burn_in_builds()
|
|
tasks = self._get_burn_in_tasks()
|
|
tests = ["test1.js", "test2.js", "test3.js", "test4.js", "test5.js"]
|
|
burn_in_report = burn_in.Report(builds, tasks, tests)
|
|
burn_in_report._update_report_burn_in([], 0)
|
|
self.assertEqual(burn_in_report.report["tasks_failed_burn_in"], 0)
|
|
self.assertEqual(burn_in_report.report["tasks_failed_only_burn_in"], 0)
|
|
self.assertEqual(burn_in_report.report[burn_in.BURN_IN_TASKS_EXCEED], 0)
|
|
|
|
def test___update_report_status(self):
|
|
builds = self._get_burn_in_builds()
|
|
tasks = self._get_burn_in_tasks()
|
|
tests = ["test1.js", "test2.js", "test3.js", "test4.js", "test5.js"]
|
|
burn_in_report = burn_in.Report(builds, tasks, tests)
|
|
build = {
|
|
"status_counts": {"succeeded": 2, "failed": 2},
|
|
"tasks": ["t1", "t2", "t3", "t4"],
|
|
}
|
|
burn_in_report._update_report_status(build)
|
|
self.assertEqual(burn_in_report.report["tasks"], 4)
|
|
self.assertEqual(burn_in_report.report["tasks_succeeded"], 2)
|
|
self.assertEqual(burn_in_report.report["tasks_failed"], 2)
|
|
|
|
def test__is_patch_build_completed(self):
|
|
builds = [{"status": "failed"}, {"status": "success"}]
|
|
self.assertTrue(burn_in.Report._is_patch_build_completed(builds))
|
|
|
|
def test__is_patch_build_completed_incomplete(self):
|
|
builds = [{"status": "failed"}, {"status": "started"}]
|
|
self.assertFalse(burn_in.Report._is_patch_build_completed(builds))
|
|
|
|
def test__is_patch_build_completed_no_builds(self):
|
|
self.assertTrue(burn_in.Report._is_patch_build_completed([]))
|