diff --git a/MODULE.bazel b/MODULE.bazel index 9fcc4a4d489..2cfa1c98db5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -286,6 +286,12 @@ setup_bolt_data = use_repo_rule("//bazel/repository_rules:bolt_data.bzl", "setup setup_bolt_data(name = "bolt_data") +mothra_repository = use_repo_rule("//bazel/repository_rules:mothra.bzl", "mothra_repository") + +# This repository is created in CI setup or by manually cloning the 10gen/mothra repo to ./mothra. +# If the directory doesn't exist, a stub with empty teams will be created. +mothra_repository(name = "mothra") + setup_mongo_windows_toolchains_extension = use_extension("//bazel/toolchains/cc/mongo_windows:mongo_toolchain.bzl", "setup_mongo_windows_toolchain_extension") use_repo(setup_mongo_windows_toolchains_extension, "mongo_windows_toolchain") diff --git a/bazel/repository_rules/mothra.bzl b/bazel/repository_rules/mothra.bzl new file mode 100644 index 00000000000..b1da0ed63f1 --- /dev/null +++ b/bazel/repository_rules/mothra.bzl @@ -0,0 +1,52 @@ +"""Repository rule for optionally loading mothra teams data.""" + +def _mothra_repo(ctx): + """Creates a repository for mothra teams data. + + If the mothra directory exists, it will be used. + Otherwise, a stub with empty teams will be created. + """ + mothra_path = ctx.path(ctx.workspace_root).get_child("mothra") + + if mothra_path.exists: + # Mothra directory exists, symlink to it + for item in mothra_path.readdir(): + ctx.symlink(item, item.basename) + + ctx.file( + "BUILD.bazel", + """ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "teams", + srcs = glob(["mothra/teams/*.yaml"]), +) +""", + ) + else: + # Create a stub team file to satisfy runfiles resolution + ctx.file( + "mothra/teams/devprod.yaml", + """# This is an intentionally empty stub. The real mothra repo was not cloned. For the real team mappings to be used, clone 10gen/mothra into mothra. +teams: [] +""", + ) + + ctx.file( + "BUILD.bazel", + """ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "teams", + srcs = glob(["mothra/teams/*.yaml"]), +) +""", + ) + +mothra_repository = repository_rule( + implementation = _mothra_repo, + local = True, + configure = True, +) diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index 0fae6c6f522..81d4faf65cb 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -303,6 +303,10 @@ py_binary( py_binary( name = "generate_result_tasks", srcs = ["generate_result_tasks.py"], + data = [ + "@codeowners_binary//:codeowners", + "@mothra//:teams", + ], visibility = ["//visibility:public"], deps = [ dependency( @@ -313,6 +317,10 @@ py_binary( "shrub-py", group = "testing", ), + dependency( + "bazel-runfiles", + group = "testing", + ), ], ) diff --git a/buildscripts/generate_result_tasks.py b/buildscripts/generate_result_tasks.py index dcfbf5c3c9d..c80d442020a 100644 --- a/buildscripts/generate_result_tasks.py +++ b/buildscripts/generate_result_tasks.py @@ -14,12 +14,17 @@ Options: --outfile File path for the generated task config. """ +import glob import json import os import subprocess -from typing import List +import sys +from functools import cache +from typing import List, Optional +import runfiles import typer +import yaml from shrub.v2 import FunctionCall, Task from typing_extensions import Annotated @@ -33,7 +38,97 @@ def make_results_task(target: str) -> Task: FunctionCall("fetch remote test results", {"test_label": target}), ] - return Task(target, commands) + task = Task(target, commands).as_dict() + + tag = get_assignment_tag(target) + if tag: + task["tags"] = [tag] + + return task + + +def get_assignment_tag(target: str) -> Optional[str]: + # Format is like "assigned_to_jira_team_devprod_build". + # See also docs/evergreen-testing/yaml_configuration/task_ownership_tags.md + + assignment_tags = resolve_assignment_tags() + tags = set() + for codeowner in get_codeowners(target): + if codeowner in assignment_tags: + tags.add(assignment_tags[codeowner]) + if len(tags) > 1: + print( + f"Target {target} has {len(tags)} possible assignment tags based on it's codeowner: {tags}. Picking the first encountered.", + file=sys.stderr, + ) + return list(tags)[0] if tags else None + + +def get_codeowners(target: str) -> list[str]: + package = target.split(":", 1)[0] + return resolve_codeowners().get(package) + + +@cache +def resolve_assignment_tags() -> dict[str, str]: + try: + # Find the teams directory in the runfiles. Unfortunately, resolving the + # directory requires resolving a specific file within the runfiles, so + # an arbitrary team's YAML is used. + r = runfiles.Create() + teams_dir = os.path.dirname(r.Rlocation("mothra/mothra/teams/devprod.yaml")) + + teams = [] + for file in glob.glob(teams_dir + "/*.yaml"): + with open(file, "rt") as f: + teams += yaml.safe_load(f).get("teams", []) + + assignment_tags = {} + for team in teams: + evergreen_tag_name = team.get("evergreen_tag_name") + github_teams = team.get("code_owners", {}).get("github_teams", []) + for github_team in github_teams: + name = github_team.get("team_name") + if name and evergreen_tag_name: + assignment_tags[name] = "assigned_to_jira_team_" + evergreen_tag_name + return assignment_tags + except Exception as e: + # Conservatively except any exception here. In the worst case, the contents/format of the + # Mothra repo could change out from under us, and it should not completely fail + # task generation. + print(f"Failed to resolve assignment tags: {e}", file=sys.stderr) + return {} + + +@cache +def resolve_codeowners() -> dict[str, list[str]]: + try: + result = subprocess.run( + 'find * -name "BUILD.bazel" | xargs bazel run @codeowners_binary//:codeowners --', + shell=True, + capture_output=True, + text=True, + check=True, + ) + + codeowners_map = {} + for line in result.stdout.strip().split("\n"): + if not line.strip(): + continue + # Each line is formatted like: "./buildscripts/BUILD.bazel @owner1 @owner2 ..." + words = line.split() + package = "//" + words[0].removeprefix("./").removesuffix("/BUILD.bazel") + # Remove teams that don't provide a meaningful mapping to a real owner. + owners = set(words[1:]) + owners.difference_update({"@svc-auto-approve-bot", "@10gen/mongo-default-approvers"}) + + codeowners_map[package] = [owner.removeprefix("@") for owner in owners] + return codeowners_map + except subprocess.CalledProcessError as e: + print(f"Failed to resolve codeowners: {e.returncode}", file=sys.stderr) + print(f"STDOUT:\n{e.stdout}", file=sys.stderr) + print(f"STDERR:\n{e.stderr}", file=sys.stderr) + return {} def query_targets() -> List[str]: @@ -61,7 +156,7 @@ def main(outfile: Annotated[str, typer.Option()]): test_targets = query_targets() tasks = [make_results_task(target) for target in test_targets] - project = {"tasks": [task.as_dict() for task in tasks]} + project = {"tasks": [task for task in tasks]} with open(outfile, "w") as f: f.write(json.dumps(project, indent=4)) diff --git a/docs/evergreen-testing/yaml_configuration/task_ownership_tags.md b/docs/evergreen-testing/yaml_configuration/task_ownership_tags.md index 7229011b8b8..6662413784d 100644 --- a/docs/evergreen-testing/yaml_configuration/task_ownership_tags.md +++ b/docs/evergreen-testing/yaml_configuration/task_ownership_tags.md @@ -12,3 +12,5 @@ If the linter configuration is missing your team: 1. Make sure that your team configuration exists or add it in mothra 2. Make sure that your team configuration in mothra has `evergreen_tag_name` 3. Update the tag list with `assigned_to_jira_team_{evergreen_tag_name}` tag for your team + +Dynamically generated tasks for resmoke suites (i.e. the ones named like `//buildscripts/resmokeconfig:core`) will set the ownership tag based on a best effort lookup from the codeowner of the test's definition to a team name from mothra, picking the first encountered in case of multiple possible assignments. diff --git a/etc/evergreen_yml_components/configuration.yml b/etc/evergreen_yml_components/configuration.yml index 52e3ccdb317..1d8f6b6d117 100644 --- a/etc/evergreen_yml_components/configuration.yml +++ b/etc/evergreen_yml_components/configuration.yml @@ -104,6 +104,11 @@ modules: branch: master ref: v0.1.1 auto_update: true + - name: mothra + owner: 10gen + repo: mothra + branch: main + prefix: ${workdir}/src # Pre task steps pre: diff --git a/etc/evergreen_yml_components/variants/misc/task_generation.yml b/etc/evergreen_yml_components/variants/misc/task_generation.yml index 7749f31492f..26594656f15 100644 --- a/etc/evergreen_yml_components/variants/misc/task_generation.yml +++ b/etc/evergreen_yml_components/variants/misc/task_generation.yml @@ -11,6 +11,8 @@ buildvariants: activate: true run_on: - rhel8.8-medium + modules: + - mothra tasks: - name: version_gen - name: version_burn_in_gen diff --git a/poetry.lock b/poetry.lock index 672eb4b41a5..283365792ac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -99,6 +99,18 @@ files = [ docs = ["furo", "jaraco.packaging (>=9.3)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["jaraco.test", "pytest (!=8.0.*)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)"] +[[package]] +name = "bazel-runfiles" +version = "1.8.3" +description = "" +optional = false +python-versions = ">=3.7" +groups = ["testing"] +markers = "platform_machine != \"s390x\" and platform_machine != \"ppc64le\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\"" +files = [ + {file = "bazel_runfiles-1.8.3-py3-none-any.whl", hash = "sha256:57a2cc04e0b924606e8dd70fc8b31d157db43680a300f546dc60207b5ce7ca82"}, +] + [[package]] name = "blinker" version = "1.9.0" @@ -5884,4 +5896,4 @@ libdeps = ["cxxfilt", "eventlet", "flask", "flask-cors", "gevent", "lxml", "prog [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "c4a064d90a866bf24bdc2d77d8f10147a1a6cf2d94f193e064cf8405250449cd" +content-hash = "33a897a6d5a0482f8d4d9378582c1f3a732d74fae6068e9b80a9ffb6cc7a3873" diff --git a/pyproject.toml b/pyproject.toml index adf38690c14..81b302c17ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,6 +142,7 @@ python-msilib = { version = "^0.5.0", markers = "sys_platform == 'win32' and pyt cryptography = "^44.0.2" [tool.poetry.group.testing.dependencies] +bazel-runfiles = "^1.8.3" curatorbin = "^1.2.4" PyKMIP = {git = "https://github.com/mongodb-forks/PyKMIP.git", rev = "c48cb01635819e478b573e3245ef840a11d78865"} kafka-python = "^2.0.2"