128 lines
4.7 KiB
Python
128 lines
4.7 KiB
Python
"""Save various diagnostics info from the remote powercycle instance."""
|
|
|
|
from buildscripts.resmokelib.powercycle import powercycle_constants
|
|
from buildscripts.resmokelib.powercycle.lib import PowercycleCommand
|
|
from buildscripts.resmokelib.powercycle.lib.remote_operations import SSHOperation
|
|
|
|
|
|
class TarEC2Artifacts(PowercycleCommand):
|
|
"""Tar EC2 artifacts."""
|
|
|
|
COMMAND = "tarEC2Artifacts"
|
|
|
|
def execute(self) -> None:
|
|
""":return: None."""
|
|
if "ec2_ssh_failure" in self.expansions:
|
|
return
|
|
tar_cmd = "tar" if "tar" not in self.expansions else self.expansions["tar"]
|
|
ec2_artifacts = powercycle_constants.LOG_PATH
|
|
# On test success, we only archive mongod.log.
|
|
if self.expansions.get("exit_code", "1") != "0":
|
|
ec2_artifacts = f"{ec2_artifacts} {powercycle_constants.DB_PATH}"
|
|
ec2_artifacts = f"{ec2_artifacts} {powercycle_constants.BACKUP_ARTIFACTS}"
|
|
if self.is_windows():
|
|
ec2_artifacts = f"{ec2_artifacts} {powercycle_constants.EVENT_LOGPATH}"
|
|
|
|
cmd = f"{tar_cmd} czf ec2_artifacts.tgz {ec2_artifacts}"
|
|
|
|
self.remote_op.operation(SSHOperation.SHELL, cmd, None)
|
|
|
|
|
|
class CopyEC2Artifacts(PowercycleCommand):
|
|
"""Copy EC2 artifacts."""
|
|
|
|
COMMAND = "copyEC2Artifacts"
|
|
|
|
def execute(self) -> None:
|
|
""":return: None."""
|
|
if "ec2_ssh_failure" in self.expansions:
|
|
return
|
|
|
|
self.remote_op.operation(SSHOperation.COPY_FROM, "ec2_artifacts.tgz", None)
|
|
|
|
|
|
class GatherRemoteEventLogs(PowercycleCommand):
|
|
"""
|
|
Gather remote event logs.
|
|
|
|
The event logs on Windows are a useful diagnostic to have when determining if something bad
|
|
happened to the remote machine after it was repeatedly crashed during powercycle testing. For
|
|
example, the Application and System event logs have previously revealed that the mongod.exe
|
|
process abruptly exited due to not being able to open a file despite the process successfully
|
|
being restarted and responding to network requests.
|
|
"""
|
|
|
|
COMMAND = "gatherRemoteEventLogs"
|
|
|
|
def execute(self) -> None:
|
|
""":return: None."""
|
|
if not self.is_windows() or self.expansions.get("ec2_ssh_failure", ""):
|
|
return
|
|
|
|
event_logpath = powercycle_constants.EVENT_LOGPATH
|
|
cmds = f"mkdir -p {event_logpath}"
|
|
cmds = f"{cmds}; wevtutil qe Application /c:10000 /rd:true /f:Text > {event_logpath}/application.log"
|
|
cmds = f"{cmds}; wevtutil qe Security /c:10000 /rd:true /f:Text > {event_logpath}/security.log"
|
|
cmds = f"{cmds}; wevtutil qe System /c:10000 /rd:true /f:Text > {event_logpath}/system.log"
|
|
|
|
self.remote_op.operation(SSHOperation.SHELL, cmds, None)
|
|
|
|
|
|
class GatherRemoteMongoCoredumps(PowercycleCommand):
|
|
"""Gather Remote Mongo Coredumps."""
|
|
|
|
COMMAND = "gatherRemoteMongoCoredumps"
|
|
|
|
def execute(self) -> None:
|
|
""":return: None."""
|
|
if "ec2_ssh_failure" in self.expansions:
|
|
return
|
|
|
|
remote_dir = powercycle_constants.REMOTE_DIR
|
|
# Find all core files and move to $remote_dir
|
|
cmds = (
|
|
"core_files=$(/usr/bin/find -H . \\( -name '*.core' -o -name '*.mdmp' \\) 2> /dev/null)"
|
|
)
|
|
cmds = f'{cmds}; if [ -z "$core_files" ]; then exit 0; fi'
|
|
cmds = f"{cmds}; echo Found remote core files $core_files, moving to $(pwd)"
|
|
cmds = f"{cmds}; for core_file in $core_files"
|
|
cmds = f"{cmds}; do base_name=$(echo $core_file | sed 's/.*///')"
|
|
cmds = f"{cmds}; if [ ! -f $base_name ]; then mv $core_file .; fi"
|
|
cmds = f"{cmds}; done"
|
|
|
|
self.remote_op.operation(SSHOperation.SHELL, cmds, remote_dir)
|
|
|
|
|
|
class CopyRemoteMongoCoredumps(PowercycleCommand):
|
|
"""Copy Remote Mongo Coredumps."""
|
|
|
|
COMMAND = "copyRemoteMongoCoredumps"
|
|
|
|
def execute(self) -> None:
|
|
""":return: None."""
|
|
if self.expansions.get("ec2_ssh_failure", ""):
|
|
return
|
|
|
|
if self.is_windows():
|
|
core_suffix = "mdmp"
|
|
else:
|
|
core_suffix = "core"
|
|
|
|
remote_dir = powercycle_constants.REMOTE_DIR
|
|
# Core file may not exist so we ignore the return code.
|
|
self.remote_op.operation(SSHOperation.SHELL, f"{remote_dir}/*.{core_suffix}", None, True)
|
|
|
|
|
|
class CopyEC2MonitorFiles(PowercycleCommand):
|
|
"""Copy EC2 monitor files."""
|
|
|
|
COMMAND = "copyEC2MonitorFiles"
|
|
|
|
def execute(self) -> None:
|
|
""":return: None."""
|
|
tar_cmd = "tar" if "tar" not in self.expansions else self.expansions["tar"]
|
|
cmd = f"{tar_cmd} czf ec2_monitor_files.tgz {powercycle_constants.EC2_MONITOR_FILES}"
|
|
|
|
self.remote_op.operation(SSHOperation.SHELL, cmd, None)
|
|
self.remote_op.operation(SSHOperation.COPY_FROM, "ec2_monitor_files.tgz", None)
|