From b0035c299ff5a2bcf3bbdffefac418d45da5d6ae Mon Sep 17 00:00:00 2001 From: Trevor Guidry Date: Wed, 18 Dec 2024 15:03:09 -0600 Subject: [PATCH] SERVER-98555 Switch Jepsen and Antithesis to https cloning (#30435) GitOrigin-RevId: d7c45fdd00f35bfc02ad12816bef806ba2558a30 --- .../testing/docker_cluster_image_builder.py | 52 +++++++++---------- buildscripts/util/BUILD.bazel | 1 + buildscripts/util/expansions.py | 27 ++++++++++ etc/evergreen_yml_components/definitions.yml | 40 +++++++++++++- evergreen/jepsen_docker/setup.sh | 4 +- 5 files changed, 92 insertions(+), 32 deletions(-) create mode 100644 buildscripts/util/expansions.py diff --git a/buildscripts/resmokelib/testing/docker_cluster_image_builder.py b/buildscripts/resmokelib/testing/docker_cluster_image_builder.py index afa6e5f4ac8..1d1a04f5c91 100644 --- a/buildscripts/resmokelib/testing/docker_cluster_image_builder.py +++ b/buildscripts/resmokelib/testing/docker_cluster_image_builder.py @@ -9,7 +9,7 @@ import yaml from buildscripts.resmokelib import config from buildscripts.resmokelib.errors import RequiresForceRemove -from buildscripts.util.read_config import read_config_file +from buildscripts.util.expansions import get_expansion def build_images(suite_name, fixture_instance): @@ -82,8 +82,7 @@ class DockerComposeImageBuilder: def _get_san_options(self): if self.in_evergreen: - expansions = read_config_file("../expansions.yml") - san_options = expansions.get("san_options", "") + san_options = get_expansion("san_options", "") else: san_options = os.environ.get("san_options", "") lines = [line for line in san_options.split() if line] @@ -454,29 +453,6 @@ class DockerComposeImageBuilder: git.Repo.clone_from("./", mongo_repo_destination, branch=active_branch) print("Done cloning MongoDB repo to build context.") - print("Cloning current MongoDB Enterprise Modules repo to build context...") - print(clone_repo_warning_message) - - # Create the modules directory in the mongo repo at the build context - modules_directory_at_build_context = os.path.join( - mongo_repo_destination, self.MODULES_RELATIVE_PATH - ) - os.mkdir(modules_directory_at_build_context) - - mongo_enterprise_modules_destination = os.path.join( - mongo_repo_destination, self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH - ) - - # Copy the mongo enterprise modules repo to the build context. - # If this fails to clone, the `git` library will raise an exception. - active_branch = git.Repo(self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH).active_branch.name - git.Repo.clone_from( - self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH, - mongo_enterprise_modules_destination, - branch=active_branch, - ) - print("Done cloning MongoDB Enterprise Modules repo to build context.") - def _clone_qa_repo_to_build_context(self, dir_path): """ Clone the QA repo to the build context. @@ -490,7 +466,7 @@ class DockerComposeImageBuilder: print(f"\n\tFound existing QA repo at: {qa_repo_destination}\n") else: print("Cloning QA repo to build context...") - git.Repo.clone_from("git@github.com:10gen/QA.git", qa_repo_destination) + self._clone_repo("10gen", "QA", qa_repo_destination, get_expansion("github_token_qa")) print("Done cloning QA repo to build context.") def _clone_jstestfuzz_to_build_context(self, dir_path): @@ -506,7 +482,12 @@ class DockerComposeImageBuilder: print(f"\n\tFound existing jstestfuzz repo at: {jstestfuzz_repo_destination}\n") else: print("Cloning jstestfuzz repo to build context...") - git.Repo.clone_from("git@github.com:10gen/jstestfuzz.git", jstestfuzz_repo_destination) + self._clone_repo( + "10gen", + "jstestfuzz", + jstestfuzz_repo_destination, + get_expansion("github_token_jstestfuzz"), + ) print("Done cloning jstestfuzz repo to build context.") def _copy_config_files_to_build_context(self, dir_path): @@ -587,3 +568,18 @@ class DockerComposeImageBuilder: print( "Done writing stub `libvoidstar.so` to build context -- This is for development only." ) + + def _clone_repo(self, owner, repo, destination, token): + """ + Conditionally clone using either https or ssh depending on if running in evergreen. + """ + + if token: + print(f"Found token for {owner}/{repo} git repo, using http clone") + url = f"https://x-access-token:{token}@github.com/{owner}/{repo}.git" + else: + print(f"No token found for {owner}/{repo} git repo, using ssh clone") + assert not self.in_evergreen, "SSH cloning should only be done when not in evergreen" + url = f"git@github.com:{owner}/{repo}.git" + + git.Repo.clone_from(url, destination) diff --git a/buildscripts/util/BUILD.bazel b/buildscripts/util/BUILD.bazel index e0d491a4a22..0dc6f6b231c 100644 --- a/buildscripts/util/BUILD.bazel +++ b/buildscripts/util/BUILD.bazel @@ -7,6 +7,7 @@ py_library( "cedar_report.py", "cmdutils.py", "codeowners_utils.py", + "expansions.py", "fileops.py", "generate_co_jira_map.py", "oauth.py", diff --git a/buildscripts/util/expansions.py b/buildscripts/util/expansions.py new file mode 100644 index 00000000000..c98ddcac4ee --- /dev/null +++ b/buildscripts/util/expansions.py @@ -0,0 +1,27 @@ +"""Python utilities around evergreen expansions.""" + +import os +from functools import cache +from pathlib import Path +from typing import Any + +import yaml + + +@cache +def get_expansions() -> dict: + current_path = Path(__file__).resolve() + evergreen_workdir = current_path.parents[3] + expansions_file = os.path.join(evergreen_workdir, "expansions.yml") + if not os.path.exists(expansions_file): + return None + + with open(expansions_file, "r") as file: + return yaml.safe_load(file) + + +def get_expansion(key: str, default: Any = None) -> Any: + expansions = get_expansions() + if expansions is None: + return default + return expansions.get(key, default) diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 5058d59f395..2960c3c25d3 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1637,7 +1637,15 @@ functions: params: owner: 10gen repo: jepsen-io-mongodb - expansion_name: github_token + expansion_name: jepsen_io_github_token + permissions: + metadata: read + contents: read + - command: github.generate_token + params: + owner: 10gen + repo: jepsen + expansion_name: jepsen_github_token permissions: metadata: read contents: read @@ -1648,7 +1656,8 @@ functions: args: - "./src/evergreen/jepsen_docker/setup.sh" include_expansions_in_env: - - github_token + - jepsen_io_github_token + - jepsen_github_token "setup jepsen config fuzzer": - *f_expansions_write - command: subprocess.exec @@ -2817,6 +2826,33 @@ functions: display_name: multiversion_exclude_tags.yml from resmoke invocation "antithesis image build and push": + - command: github.generate_token + params: + owner: 10gen + repo: QA + expansion_name: github_token_qa_temp + permissions: + metadata: read + contents: read + - command: github.generate_token + params: + owner: 10gen + repo: jstestfuzz + expansion_name: github_token_jstestfuzz_temp + permissions: + metadata: read + contents: read + - command: subprocess.exec + params: + binary: "bash" + args: + - "-c" + - | + echo "github_token_qa: ${github_token_qa_temp}" >> github_expansions.yml + echo "github_token_jstestfuzz: ${github_token_jstestfuzz_temp}" >> github_expansions.yml + - command: expansions.update + params: + file: github_expansions.yml - *f_expansions_write - command: subprocess.exec params: diff --git a/evergreen/jepsen_docker/setup.sh b/evergreen/jepsen_docker/setup.sh index ef74f788e6e..5036a6a449b 100644 --- a/evergreen/jepsen_docker/setup.sh +++ b/evergreen/jepsen_docker/setup.sh @@ -2,14 +2,14 @@ set -euo pipefail # Clone our internal fork of jepsen-io/jepsen to get the core # functionality with a few tweaks meant for evergreen integration. -git clone --branch=v0.2.0-evergreen-master git@github.com:10gen/jepsen.git jepsen +git clone --branch=v0.2.0-evergreen-master https://x-access-token:${jepsen_github_token}@github.com/10gen/jepsen.git jepsen # Copy our mongodb source for jepsen to run into the docker area to be # copied into the image during the build process. cp -rf src/dist-test jepsen/docker/node # Clone our internal tests to run -git clone --branch=v0.2.2 https://x-access-token:${github_token}@github.com/10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb +git clone --branch=v0.2.2 https://x-access-token:${jepsen_io_github_token}@github.com/10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb # Kill any running containers sudo docker container kill $(docker ps -q) || true