SERVER-98555 Switch Jepsen and Antithesis to https cloning (#30435)

GitOrigin-RevId: d7c45fdd00f35bfc02ad12816bef806ba2558a30

buildscripts/resmokelib/testing/docker_cluster_image_builder.py

@@ -9,7 +9,7 @@ import yaml
 
 from buildscripts.resmokelib import config
 from buildscripts.resmokelib.errors import RequiresForceRemove
-from buildscripts.util.read_config import read_config_file
+from buildscripts.util.expansions import get_expansion
 
 
 def build_images(suite_name, fixture_instance):
@@ -82,8 +82,7 @@ class DockerComposeImageBuilder:
 
     def _get_san_options(self):
         if self.in_evergreen:
-            expansions = read_config_file("../expansions.yml")
-            san_options = expansions.get("san_options", "")
+            san_options = get_expansion("san_options", "")
         else:
             san_options = os.environ.get("san_options", "")
         lines = [line for line in san_options.split() if line]
@@ -454,29 +453,6 @@ class DockerComposeImageBuilder:
         git.Repo.clone_from("./", mongo_repo_destination, branch=active_branch)
         print("Done cloning MongoDB repo to build context.")
 
-        print("Cloning current MongoDB Enterprise Modules repo to build context...")
-        print(clone_repo_warning_message)
-
-        # Create the modules directory in the mongo repo at the build context
-        modules_directory_at_build_context = os.path.join(
-            mongo_repo_destination, self.MODULES_RELATIVE_PATH
-        )
-        os.mkdir(modules_directory_at_build_context)
-
-        mongo_enterprise_modules_destination = os.path.join(
-            mongo_repo_destination, self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH
-        )
-
-        # Copy the mongo enterprise modules repo to the build context.
-        # If this fails to clone, the `git` library will raise an exception.
-        active_branch = git.Repo(self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH).active_branch.name
-        git.Repo.clone_from(
-            self.MONGO_ENTERPRISE_MODULES_RELATIVE_PATH,
-            mongo_enterprise_modules_destination,
-            branch=active_branch,
-        )
-        print("Done cloning MongoDB Enterprise Modules repo to build context.")
-
     def _clone_qa_repo_to_build_context(self, dir_path):
         """
         Clone the QA repo to the build context.
@@ -490,7 +466,7 @@ class DockerComposeImageBuilder:
             print(f"\n\tFound existing QA repo at: {qa_repo_destination}\n")
         else:
             print("Cloning QA repo to build context...")
-            git.Repo.clone_from("git@github.com:10gen/QA.git", qa_repo_destination)
+            self._clone_repo("10gen", "QA", qa_repo_destination, get_expansion("github_token_qa"))
             print("Done cloning QA repo to build context.")
 
     def _clone_jstestfuzz_to_build_context(self, dir_path):
@@ -506,7 +482,12 @@ class DockerComposeImageBuilder:
             print(f"\n\tFound existing jstestfuzz repo at: {jstestfuzz_repo_destination}\n")
         else:
             print("Cloning jstestfuzz repo to build context...")
-            git.Repo.clone_from("git@github.com:10gen/jstestfuzz.git", jstestfuzz_repo_destination)
+            self._clone_repo(
+                "10gen",
+                "jstestfuzz",
+                jstestfuzz_repo_destination,
+                get_expansion("github_token_jstestfuzz"),
+            )
             print("Done cloning jstestfuzz repo to build context.")
 
     def _copy_config_files_to_build_context(self, dir_path):
@@ -587,3 +568,18 @@ class DockerComposeImageBuilder:
             print(
                 "Done writing stub `libvoidstar.so` to build context -- This is for development only."
             )
+
+    def _clone_repo(self, owner, repo, destination, token):
+        """
+        Conditionally clone using either https or ssh depending on if running in evergreen.
+        """
+
+        if token:
+            print(f"Found token for {owner}/{repo} git repo, using http clone")
+            url = f"https://x-access-token:{token}@github.com/{owner}/{repo}.git"
+        else:
+            print(f"No token found for {owner}/{repo} git repo, using ssh clone")
+            assert not self.in_evergreen, "SSH cloning should only be done when not in evergreen"
+            url = f"git@github.com:{owner}/{repo}.git"
+
+        git.Repo.clone_from(url, destination)

buildscripts/util/BUILD.bazel

@@ -7,6 +7,7 @@ py_library(
         "cedar_report.py",
         "cmdutils.py",
         "codeowners_utils.py",
+        "expansions.py",
         "fileops.py",
         "generate_co_jira_map.py",
         "oauth.py",

buildscripts/util/expansions.py

@@ -0,0 +1,27 @@
+"""Python utilities around evergreen expansions."""
+
+import os
+from functools import cache
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+@cache
+def get_expansions() -> dict:
+    current_path = Path(__file__).resolve()
+    evergreen_workdir = current_path.parents[3]
+    expansions_file = os.path.join(evergreen_workdir, "expansions.yml")
+    if not os.path.exists(expansions_file):
+        return None
+
+    with open(expansions_file, "r") as file:
+        return yaml.safe_load(file)
+
+
+def get_expansion(key: str, default: Any = None) -> Any:
+    expansions = get_expansions()
+    if expansions is None:
+        return default
+    return expansions.get(key, default)

etc/evergreen_yml_components/definitions.yml

@@ -1637,7 +1637,15 @@ functions:
       params:
         owner: 10gen
         repo: jepsen-io-mongodb
-        expansion_name: github_token
+        expansion_name: jepsen_io_github_token
+        permissions:
+          metadata: read
+          contents: read
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: jepsen
+        expansion_name: jepsen_github_token
         permissions:
           metadata: read
           contents: read
@@ -1648,7 +1656,8 @@ functions:
         args:
           - "./src/evergreen/jepsen_docker/setup.sh"
         include_expansions_in_env:
-          - github_token
+          - jepsen_io_github_token
+          - jepsen_github_token
   "setup jepsen config fuzzer":
     - *f_expansions_write
     - command: subprocess.exec
@@ -2817,6 +2826,33 @@ functions:
       display_name: multiversion_exclude_tags.yml from resmoke invocation
 
   "antithesis image build and push":
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: QA
+        expansion_name: github_token_qa_temp
+        permissions:
+          metadata: read
+          contents: read
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: jstestfuzz
+        expansion_name: github_token_jstestfuzz_temp
+        permissions:
+          metadata: read
+          contents: read
+    - command: subprocess.exec
+      params:
+        binary: "bash"
+        args:
+          - "-c"
+          - |
+            echo "github_token_qa: ${github_token_qa_temp}" >> github_expansions.yml
+            echo "github_token_jstestfuzz: ${github_token_jstestfuzz_temp}" >> github_expansions.yml
+    - command: expansions.update
+      params:
+        file: github_expansions.yml
     - *f_expansions_write
     - command: subprocess.exec
       params:

evergreen/jepsen_docker/setup.sh

@@ -2,14 +2,14 @@ set -euo pipefail
 
 # Clone our internal fork of jepsen-io/jepsen to get the core
 # functionality with a few tweaks meant for evergreen integration.
-git clone --branch=v0.2.0-evergreen-master git@github.com:10gen/jepsen.git jepsen
+git clone --branch=v0.2.0-evergreen-master https://x-access-token:${jepsen_github_token}@github.com/10gen/jepsen.git jepsen
 
 # Copy our mongodb source for jepsen to run into the docker area to be
 # copied into the image during the build process.
 cp -rf src/dist-test jepsen/docker/node
 
 # Clone our internal tests to run
-git clone --branch=v0.2.2 https://x-access-token:${github_token}@github.com/10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb
+git clone --branch=v0.2.2 https://x-access-token:${jepsen_io_github_token}@github.com/10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb
 
 # Kill any running containers
 sudo docker container kill $(docker ps -q) || true