xuemingqiang/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "SERVER-77559, SERVER-77560: Implement Evergreen file system log handler for resmoke and update resmoke test results (#21826)"

Browse Source 
This reverts commit 7f15b1145e41081994fe250dba9e1e0347794aae.
This commit is contained in:
Dylan Richardson
2025-12-17 20:15:59 +00:00
committed by Andrew Bui
parent c326c9a2b2
commit 722bdbc7cc
8 changed files with 13 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
buildscripts/resmokeconfig/loggers/evg.yml
View File

@@ -1,27 +0,0 @@
logging:
executor:
format: '[%(name)s] %(asctime)s %(message)s'
handlers:
- class: logging.StreamHandler
fixture:
format: '[%(name)s] %(message)s'
handlers:
- class: evergreen
tests:
format: '[%(name)s] %(message)s'
handlers:
- class: evergreen
shorten_logger_name:
remove:
- MongoDFixture
- ReplicaSetFixture
- ShardedClusterFixture
replace:
primary: prim
secondary: sec
node: n
shard: s
configsvr: c
mongos: s
job: j

4
buildscripts/resmokelib/config.py
View File

@@ -111,7 +111,6 @@ DEFAULTS = {
"task_name": None,
"variant_name": None,
"version_id": None,
"work_dir": None,
# WiredTiger options.
"wt_coll_config": None,
@@ -287,9 +286,6 @@ EVERGREEN_VARIANT_NAME = None
# the commit hash.
EVERGREEN_VERSION_ID = None
# The Evergreen task's working directory.
EVERGREEN_WORK_DIR = None
# If set, then any jstests that have any of the specified tags will be excluded from the suite(s).
EXCLUDE_WITH_ANY_TAGS = None

1
buildscripts/resmokelib/configure_resmoke.py
View File

@@ -240,7 +240,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py
_config.EVERGREEN_TASK_NAME = config.pop("task_name")
_config.EVERGREEN_VARIANT_NAME = config.pop("variant_name")
_config.EVERGREEN_VERSION_ID = config.pop("version_id")
_config.EVERGREEN_WORK_DIR = config.pop("work_dir")
# Archival options. Archival is enabled only when running on evergreen.
if not _config.EVERGREEN_TASK_ID:

14
buildscripts/resmokelib/logging/formatters.py
View File

@@ -41,17 +41,3 @@ class ISO8601Formatter(logging.Formatter):
utc_offset_mins = (utc_offset_secs / 60) % 60
utc_offset_hours = utc_offset_secs / 3600
return "%s%02d%02d" % (utc_offset_prefix, utc_offset_hours, utc_offset_mins)
class EvergreenLogFormatter(logging.Formatter):
"""Log line formatter for Evergreen log messages.
See `https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Task-Output-Directory#test-logs`
for more info.
"""
def format(self, record):
"""Return the Evergreen formatted record."""
ts = int(record.created * 1e9)
return "\n".join([f"{ts} {line}" for line in super().format(record).split("\n")])

87
buildscripts/resmokelib/logging/loggers.py
View File

@@ -1,13 +1,10 @@
"""Module to hold the logger instances themselves."""
import logging
import os
import sys
import yaml
from . import buildlogger
from . import formatters
from .. import config
from .. import errors
_DEFAULT_FORMAT = "[%(name)s] %(message)s"
@@ -47,13 +44,6 @@ def configure_loggers(logging_config):
EXECUTOR_LOGGER = ExecutorRootLogger(logging_config, build_logger_server, fixture_logger,
tests_logger)
_write_evergreen_log_spec()
def get_evergreen_log_name(job_num, test_id=None):
"""Return the log name, relative to the reserved test log directory, on the Evergreen task host."""
return f"job{job_num}/" + (f"{test_id}.log" if test_id else "global.log")
class BaseLogger(logging.Logger):
"""Base class for the custom loggers used in this library.
@@ -137,8 +127,6 @@ class RootLogger(BaseLogger):
handler = logging.StreamHandler(sys.stdout)
elif handler_class == "buildlogger":
return # Buildlogger handlers are applied when creating specific child loggers
elif handler_class == "evergreen":
return # Evergreen handlers are applied when creating specific child loggers
else:
raise ValueError("Unknown handler class '%s'" % handler_class)
handler.setFormatter(formatter)
@@ -204,46 +192,42 @@ class JobLogger(BaseLogger):
"""Create a new fixture logger that will be a child of the "fixture" root logger."""
return FixtureLogger(fixture_class, self.job_num, self.build_id, self.fixture_root_logger)
def new_test_logger(self, test_id, test_shortname, test_basename, command, parent): # pylint: disable=too-many-arguments
def new_test_logger(self, test_shortname, test_basename, command, parent):
"""Create a new test logger that will be a child of the given parent."""
if self.build_id:
# If we're configured to log messages to the buildlogger server, then request a new
# test id for this test.
build_test_id = self.build_logger_server.new_test_id(self.build_id, test_basename,
command)
if not build_test_id:
# test_id for this test.
test_id = self.build_logger_server.new_test_id(self.build_id, test_basename, command)
if not test_id:
buildlogger.set_log_output_incomplete()
raise errors.LoggerRuntimeConfigError(
"Encountered an error configuring buildlogger for test {}: Failed to get a new"
" test_id".format(test_basename))
url = self.build_logger_server.get_test_log_url(self.build_id, build_test_id)
url = self.build_logger_server.get_test_log_url(self.build_id, test_id)
self.info("Writing output of %s to %s.", test_basename, url)
return TestLogger(test_shortname, parent, self.job_num, self.build_id, build_test_id,
url)
return TestLogger(test_shortname, parent, self.build_id, test_id, url)
return TestLogger(test_shortname, parent, job_num=self.job_num, test_id=test_id)
return TestLogger(test_shortname, parent)
class TestLogger(BaseLogger):
"""TestLogger class."""
def __init__( # pylint: disable=too-many-arguments
self, test_name, parent, job_num=None, build_id=None, test_id=None, url=None):
self, test_name, parent, build_id=None, test_id=None, url=None):
"""Initialize a TestLogger.
:param test_name: the test name.
:param parent: the parent logger.
:param job_num: the number of the job the test is running on.
:param build_id: the build logger build id.
:param test_id: the test id for the logger handler.
:param test_id: the build logger test id.
:param url: the build logger URL endpoint for the test.
"""
name = "%s:%s" % (parent.name, test_name)
BaseLogger.__init__(self, name, parent=parent)
self.url_endpoint = url
self._add_build_logger_handler(build_id, test_id)
_add_evergreen_handler(self, self.logging_config, job_num, test_id)
def _add_build_logger_handler(self, build_id, test_id):
logger_info = self.logging_config[TESTS_LOGGER_NAME]
@@ -285,7 +269,6 @@ class FixtureLogger(BaseLogger):
self.fixture_class = fixture_class
self.job_num = job_num
self._add_build_logger_handler(build_id)
_add_evergreen_handler(self, self.logging_config, job_num)
def _add_build_logger_handler(self, build_id):
logger_info = self.logging_config[FIXTURE_LOGGER_NAME]
@@ -387,55 +370,3 @@ def _get_buildlogger_handler_info(logger_info):
if handler_info.pop("class") == "buildlogger":
return handler_info
return None
# Utility functions for Evergreen file system logging.
# See `https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Task-Output-Directory#test-logs`
# for more information.
def _write_evergreen_log_spec():
"""Configure file system logging for Evergreen tasks."""
if not config.EVERGREEN_WORK_DIR:
return
fp = f"{_get_evergreen_log_dirname()}/log_spec.yaml"
os.makedirs(os.path.dirname(fp), exist_ok=True)
EXECUTOR_LOGGER.info("Writing Evergreen test log spec to %s.", fp)
log_spec = {
"version": 0,
"format": "text-timestamp",
}
with open(fp, "w") as fd:
yaml.dump(log_spec, fd)
def _add_evergreen_handler(logger, logging_config, job_num, test_id=None):
"""Add a new evergreen handler to a logger."""
logger_info = logging_config[TESTS_LOGGER_NAME]
evergreen_handler_info = None
for handler_info in logger_info["handlers"]:
if handler_info["class"] == "evergreen":
evergreen_handler_info = handler_info
break
if evergreen_handler_info:
fp = f"{_get_evergreen_log_dirname()}/{get_evergreen_log_name(job_num, test_id)}"
os.makedirs(os.path.dirname(fp), exist_ok=True)
handler = logging.FileHandler(filename=fp, mode="a")
handler.setFormatter(
formatters.EvergreenLogFormatter(fmt=logger_info.get("format", _DEFAULT_FORMAT)))
logger.addHandler(handler)
if test_id:
EXECUTOR_LOGGER.info("Writing output of %s to %s.", test_id, fp)
else:
EXECUTOR_LOGGER.info("Writing output of job #%d to %s.", job_num, fp)
def _get_evergreen_log_dirname():
"""Return the reserved directory for test logs on the Evergreen task host."""
return f"{config.EVERGREEN_WORK_DIR}/build/TestLogs"

3
buildscripts/resmokelib/parser.py
View File

@@ -385,9 +385,6 @@ def _add_run(subparsers): # pylint: disable=too-many-statements
evergreen_options.add_argument("--versionId", dest="version_id", metavar="VERSION_ID",
help="Sets the version ID of the task.")
evergreen_options.add_argument("--taskWorkDir", dest="work_dir", metavar="TASK_WORK_DIR",
help="Sets the working directory of the task.")
benchmark_options = parser.add_argument_group(
title=_BENCHMARK_ARGUMENT_TITLE, description="Options for running Benchmark/Benchrun tests")

23
buildscripts/resmokelib/testing/report.py
View File

@@ -4,7 +4,6 @@ This is used to support additional test status and timing information for the re
"""
import copy
import os
import threading
import time
import unittest
@@ -99,7 +98,6 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr
unittest.TestResult.startTest(self, test)
test_info = _TestInfo(test.id(), test.test_name, test.dynamic)
test_info.group_id = f"job{self.job_logger.job_num}"
basename = test.basename()
command = test.as_command()
@@ -114,14 +112,10 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr
self.num_dynamic += 1
# Set up the test-specific logger.
test_logger = self.job_logger.new_test_logger(test.id(), test.short_name(), test.basename(),
command, test.logger)
test_logger = self.job_logger.new_test_logger(test.short_name(), test.basename(), command,
test.logger)
test_info.url_endpoint = test_logger.url_endpoint
test_info.log_info = {
"log_name": logging.loggers.get_evergreen_log_name(self.job_logger.job_num, test.id()),
"logs_to_merge": [logging.loggers.get_evergreen_log_name(self.job_logger.job_num)],
"rendering_type": "resmoke", "version": 0
}
test.override_logger(test_logger)
test_info.start_time = time.time()
@@ -268,18 +262,13 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr
for test_info in self.test_infos:
result = {
"test_file": test_info.test_file,
"group_id": test_info.group_id,
"status": test_info.evergreen_status,
"exit_code": test_info.return_code,
"start": test_info.start_time,
"end": test_info.end_time,
"elapsed": test_info.end_time - test_info.start_time,
"log_info": test_info.log_info,
}
if test_info.display_test_name is not None:
result["display_test_name"] = test_info.display_test_name
if test_info.url_endpoint is not None:
result["url"] = test_info.url_endpoint
result["url_raw"] = test_info.url_endpoint + "?raw=1"
@@ -306,9 +295,6 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr
# Using test_file as the test id is ok here since the test id only needs to be unique
# during suite execution.
test_info = _TestInfo(test_file, test_file, is_dynamic)
test_info.display_test_name = result.get("display_test_name")
test_info.group_id = result.get("group_id")
test_info.log_info = result.get("log_info")
test_info.url_endpoint = result.get("url")
test_info.status = result["status"]
test_info.evergreen_status = test_info.status
@@ -362,8 +348,6 @@ class _TestInfo(object): # pylint: disable=too-many-instance-attributes
self.test_id = test_id
self.test_file = test_file
self.group_id = None
self.display_test_name = None
self.dynamic = dynamic
self.start_time = None
@@ -371,7 +355,6 @@ class _TestInfo(object): # pylint: disable=too-many-instance-attributes
self.status = None
self.evergreen_status = None
self.return_code = None
self.log_info = None
self.url_endpoint = None

3
etc/evergreen.yml
View File

@@ -1387,7 +1387,7 @@ functions:
${san_options} \
${snmp_config_path} \
${resmoke_wrapper} \
$python buildscripts/resmoke.py run \
$python buildscripts/resmoke.py run \
${resmoke_args} \
$extra_args \
${test_flags} \
@@ -1404,7 +1404,6 @@ functions:
--taskName=${task_name} \
--variantName=${build_variant} \
--versionId=${version_id} \
--taskWorkDir='${workdir}' \
--archiveFile=archive.json \
--reportFile=report.json \
--perfReportFile=perf.json