diff --git a/buildfarm/toolchain.dockerfile b/buildfarm/toolchain.dockerfile index 2cec037638b..da50e29a5b6 100644 --- a/buildfarm/toolchain.dockerfile +++ b/buildfarm/toolchain.dockerfile @@ -14,7 +14,6 @@ RUN git clone -b 2.3.1 https://github.com/bazelbuild/bazel-buildfarm.git # Switch into the cloned Buildfarm repository WORKDIR /bazel-buildfarm -# TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted. # Obtain Bazelisk and make it executable RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 -O bazelisk && chmod +x bazelisk diff --git a/buildscripts/libdeps/libdeps/analyzer.py b/buildscripts/libdeps/libdeps/analyzer.py index b0b766d8abc..25dff541613 100644 --- a/buildscripts/libdeps/libdeps/analyzer.py +++ b/buildscripts/libdeps/libdeps/analyzer.py @@ -540,7 +540,6 @@ class BazelConversionCandidates(Analyzer): # Note: //... is the bazel catch-all for referencing all targets in that directory. For # example, //src/... will expand to include all targets under //src/. - # TODO(SERVER-81038): remove /tmp/ prefix once bazel/bazelisk is self-hosted. proc = subprocess.run(["/tmp/bazelisk", "query", "//..."], capture_output=True, text=True, check=True) diff --git a/docs/bazel.md b/docs/bazel.md index 7735be1fef1..05b42618111 100644 --- a/docs/bazel.md +++ b/docs/bazel.md @@ -16,7 +16,7 @@ To generate and install the engflow credentials: To perform a Bazel build and _bypass_ SCons: -- Install Bazelisk: `curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 --output /tmp/bazelisk && chmod +x /tmp/bazelisk` +- Install Bazelisk: `curl -L https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-arm64 --output /tmp/bazelisk && chmod +x /tmp/bazelisk` - Build the Bazel-compatible target: `/tmp/bazelisk build --verbose_failures src/mongo/db/commands:fsync_locked` To perform a Bazel build using a local Buildfarm (to test remote execution capability): diff --git a/etc/evergreen_yml_components/tasks/compile_tasks.yml b/etc/evergreen_yml_components/tasks/compile_tasks.yml index 582b745d463..03df6a43a4d 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml @@ -216,8 +216,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - - func: "fetch bazel" - func: "scons compile" vars: targets: >- @@ -241,8 +239,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - - func: "fetch bazel" - func: "scons compile" vars: targets: >- @@ -269,7 +265,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - func: "fetch bazel" - func: "bazel compile (gcc)" vars: @@ -285,8 +280,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - - func: "fetch bazel" - func: "scons compile" vars: targets: >- @@ -310,8 +303,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - - func: "fetch bazel" - func: "scons compile" vars: targets: >- @@ -336,8 +327,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - - func: "fetch bazel" - func: "scons compile" vars: targets: >- @@ -361,8 +350,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - - func: "fetch bazel" - func: "scons compile" vars: targets: >- @@ -388,7 +375,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - func: "fetch bazel" - func: "bazel run" vars: @@ -403,7 +389,6 @@ tasks: - name: version_expansions_gen variant: generate-tasks-for-version commands: - # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. - func: "fetch bazel" - func: "bazel run" vars: diff --git a/evergreen/get_bazelisk.sh b/evergreen/get_bazelisk.sh index 41f8c081932..c010bc0633c 100755 --- a/evergreen/get_bazelisk.sh +++ b/evergreen/get_bazelisk.sh @@ -33,8 +33,7 @@ if [[ $ARCH == "ppc64le" ]]; then REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-${ARCH} LOCAL_PATH=$TMPDIR/bazel else - # TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted. - REMOTE_PATH=https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-${OS}-${ARCH}${EXT} + REMOTE_PATH=https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-${OS}-${ARCH}${EXT} LOCAL_PATH=$TMPDIR/bazelisk fi diff --git a/site_scons/site_tools/integrate_bazel.py b/site_scons/site_tools/integrate_bazel.py index cbc276fa9ac..f8e4ab05fc6 100644 --- a/site_scons/site_tools/integrate_bazel.py +++ b/site_scons/site_tools/integrate_bazel.py @@ -15,6 +15,7 @@ import time from typing import List, Dict, Set, Tuple, Any import urllib.request import requests +from retry import retry import sys import socket @@ -47,6 +48,16 @@ _SANITIZER_MAP = { _S3_HASH_MAPPING = { "https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-ppc64le": "dd21c75817533ff601bf797e64f0eb2f7f6b813af26c829f0bda30e328caef46", + "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-amd64": + "f2ba5f721a995b54bab68c6b76a340719888aa740310e634771086b6d1528ecd", + "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-darwin-arm64": + "69fa21cd2ccffc2f0970c21aa3615484ba89e3553ecce1233a9d8ad9570d170e", + "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-amd64": + "d28b588ac0916abd6bf02defb5433f6eddf7cba35ffa808eabb65a44aab226f7", + "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-linux-arm64": + "861a16ba9979613e70bd3d2f9d9ab5e3b59fe79471c5753acdc9c431ab6c9d94", + "https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-windows-amd64.exe": + "d04555245a99dfb628e33da24e2b9198beb8f46d7e7661c313eb045f6a59f5e4", } @@ -486,6 +497,11 @@ def generate_bazel_info_for_ninja(env: SCons.Environment.Environment) -> None: env["NINJA_BAZEL_INPUTS"] = ninja_bazel_ins +@retry(tries=5, delay=3) +def download_path_with_retry(*args, **kwargs): + urllib.request.urlretrieve(*args, **kwargs) + + def sha256_file(filename: str) -> str: sha256_hash = hashlib.sha256() with open(filename, "rb") as f: @@ -544,20 +560,19 @@ def generate(env: SCons.Environment.Environment) -> None: bazel_bin_dir, "bazel") if normalized_arch in ["ppc64le"] else os.path.join( bazel_bin_dir, "bazelisk") - # TODO(SERVER-81038): remove once bazel/bazelisk is self-hosted. if not os.path.exists(bazel_executable): print(f"Downloading {bazel_executable}...") # TODO(SERVER-86050): remove the branch once bazelisk is built on s390x & ppc64le if normalized_arch in ["ppc64le"]: s3_path = f"https://mdb-build-public.s3.amazonaws.com/bazel-binaries/bazel-6.4.0-{normalized_arch}" - urllib.request.urlretrieve(s3_path, bazel_executable) - verify_s3_hash(s3_path, bazel_executable) else: ext = ".exe" if normalized_os == "windows" else "" os_str = normalized_os.replace("macos", "darwin") - urllib.request.urlretrieve( - f"https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-{os_str}-{normalized_arch}{ext}", - bazel_executable) + s3_path = f"https://mdb-build-public.s3.amazonaws.com/bazelisk-binaries/v1.19.0/bazelisk-{os_str}-{normalized_arch}{ext}" + + download_path_with_retry(s3_path, bazel_executable) + verify_s3_hash(s3_path, bazel_executable) + print(f"Downloaded {bazel_executable}") os.chmod(bazel_executable, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) else: