From 5edfd7020c6ba105ff6cb5e3f76bc17cc68da8a2 Mon Sep 17 00:00:00 2001 From: Steve Gross Date: Mon, 21 Aug 2023 18:54:12 +0000 Subject: [PATCH] SERVER-79754 Add basic ability for remote execution --- buildfarm/README.md | 1 + buildfarm/config.yml | 15 +++++++++++++++ buildfarm/docker-compose.yml | 29 +++++++++++++++++++++++++++++ buildfarm/logging.properties | 6 ++++++ buildfarm/toolchain.dockerfile | 25 +++++++++++++++++++++++++ docs/bazel.md | 10 ++++++++++ toolchains/mongo_toolchain.BUILD | 13 +++++++++++++ 7 files changed, 99 insertions(+) create mode 100644 buildfarm/README.md create mode 100644 buildfarm/config.yml create mode 100644 buildfarm/docker-compose.yml create mode 100644 buildfarm/logging.properties create mode 100644 buildfarm/toolchain.dockerfile diff --git a/buildfarm/README.md b/buildfarm/README.md new file mode 100644 index 00000000000..da9a7e0f417 --- /dev/null +++ b/buildfarm/README.md @@ -0,0 +1 @@ +This directory exists to manage a Buildfarm; see docs/bazel.md for more details. \ No newline at end of file diff --git a/buildfarm/config.yml b/buildfarm/config.yml new file mode 100644 index 00000000000..555944d6605 --- /dev/null +++ b/buildfarm/config.yml @@ -0,0 +1,15 @@ +# Note: This file is read by Buildfarm server at runtime to know how to talk to redis and the worker + +backplane: + # Note: The port must correspond to the value specified in `docker-compose.yml` for services.redis.ports + redisUri: "redis://localhost:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + # Note: The port must correspond to the value specified in `docker-compose.yml` for services.worker.ports + publicName: "localhost:8981" diff --git a/buildfarm/docker-compose.yml b/buildfarm/docker-compose.yml new file mode 100644 index 00000000000..969033e6bca --- /dev/null +++ b/buildfarm/docker-compose.yml @@ -0,0 +1,29 @@ +version: "3" +services: + worker: + build: + context: . + dockerfile: toolchain.dockerfile + ports: + # Note: This value must correspond to the value specified in `config.yml` for worker.publicName + - "8981:8981" + depends_on: + - redis + network_mode: host + command: ./bazelisk run //src/main/java/build/buildfarm:buildfarm-shard-worker -- --jvm_flag=-Dlogging.config=file:/bazel-buildfarm/logging.properties /bazel-buildfarm/config.yml + server: + build: + context: . + dockerfile: toolchain.dockerfile + ports: + - "8980:8980" + network_mode: host + depends_on: + - redis + command: ./bazelisk run //src/main/java/build/buildfarm:buildfarm-server -- --jvm_flag=-Dlogging.config=file:/bazel-buildfarm/logging.properties /bazel-buildfarm/config.yml + redis: + image: "redis:5.0.9" + network_mode: host + ports: + # Note: This value must correspond to the value specified in `config.yml` for backplane.redisUri + - "6379:6379" diff --git a/buildfarm/logging.properties b/buildfarm/logging.properties new file mode 100644 index 00000000000..626de7675e5 --- /dev/null +++ b/buildfarm/logging.properties @@ -0,0 +1,6 @@ +# Note: These logging properties will be read by Buildfarm executables at runtime + +handlers=java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.level=FINE +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%4$-7s] %2$s - %5$s %6$s %n diff --git a/buildfarm/toolchain.dockerfile b/buildfarm/toolchain.dockerfile new file mode 100644 index 00000000000..da50e29a5b6 --- /dev/null +++ b/buildfarm/toolchain.dockerfile @@ -0,0 +1,25 @@ +# This file tells Docker how to build an image for both Buildfarm server and Builder shard-worker. +# Note that this file is referenced in docker-compose.yml for the aforementioned processes. + +FROM ubuntu:22.04 AS toolchain +ENV USER="root" + +# Install necessary tools/libraries +RUN apt-get update && apt-get install -y curl perl libxml2-dev libssl-dev git wget openjdk-19-jdk g++ gcc + +# Pull in the Buildfarm repository +# Note: We are not verifying the commit hash, because the buildfarm solution is temporary and thus not worth it. +RUN git clone -b 2.3.1 https://github.com/bazelbuild/bazel-buildfarm.git + +# Switch into the cloned Buildfarm repository +WORKDIR /bazel-buildfarm + +# Obtain Bazelisk and make it executable +RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 -O bazelisk && chmod +x bazelisk + +# Build the Buildfarm server and shard-worker in advance. (Note that this is not strictly necessary, since Bazel run will perform a build if necessary) +RUN ./bazelisk build //src/main/java/build/buildfarm:buildfarm-server //src/main/java/build/buildfarm:buildfarm-shard-worker + +# Ensure that Buildform's configuration files are availabel at runtime: +COPY config.yml config.yml +COPY logging.properties logging.properties diff --git a/docs/bazel.md b/docs/bazel.md index f344f3c328a..057e34fd7da 100644 --- a/docs/bazel.md +++ b/docs/bazel.md @@ -7,3 +7,13 @@ To perform a Bazel build via SCons: To perform a Bazel build and *bypass* SCons: * Install Bazelisk: `curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-arm64 --output /tmp/bazelisk && chmod +x /tmp/bazelisk` * Build the Bazel-compatible target: `/tmp/bazelisk build --verbose_failures src/mongo/db/commands:fsync_locked` + +To perform a Bazel build using a local Buildfarm (to test remote execution capability): +* For more details on Buildfarm, see https://bazelbuild.github.io/bazel-buildfarm +* (One time only) Build and start the Buildfarm: +** Change into the `buildfarm` directory: `cd buildfarm` +** Build the image: `docker-compose build` +** Start the container: `docker-compose up --detach` +** Poll until the containers report status `running`: `docker ps --filter status=running --filter name=buildfarm` +* (Whenever you build): +** Build the Bazel-compatible target with remote execution enabled: `/tmp/bazelisk build --verbose_failures --remote_executor=grpc://localhost:8980 src/mongo/db/commands:fsync_locked` diff --git a/toolchains/mongo_toolchain.BUILD b/toolchains/mongo_toolchain.BUILD index af316654d4b..81a9508a175 100644 --- a/toolchains/mongo_toolchain.BUILD +++ b/toolchains/mongo_toolchain.BUILD @@ -36,12 +36,17 @@ cc_toolchain_config( "--verbose", "-std=c++20", "-nostdinc++", + # These flags are necessary to get system includes properly available for compilation: "-isystem", "external/mongo_toolchain/stow/gcc-v4/lib/gcc/aarch64-mongodb-linux/11.3.0/include", "-isystem", "external/mongo_toolchain/stow/gcc-v4/include/c++/11.3.0", "-isystem", "external/mongo_toolchain/stow/gcc-v4/include/c++/11.3.0/aarch64-mongodb-linux", + # These flags are necessary for the link step to work remotely: + "-Bexternal/mongo_toolchain/v4/bin", + "-Bexternal/mongo_toolchain/v4/lib", + "-Bexternal/mongo_toolchain/stow/gcc-v4/libexec/gcc/aarch64-mongodb-linux/11.3.0", ], compiler = "gcc", cpu = "arm64", @@ -49,6 +54,14 @@ cc_toolchain_config( "/usr/include", ], host_system_name = "local", + link_flags = [ + # These flags are necessary for the link step to work remotely: + "-nostdinc++", + "-Lexternal/mongo_toolchain/v4/lib", + "-Lexternal/mongo_toolchain/stow/gcc-v4/lib/gcc/aarch64-mongodb-linux/11.3.0", + "-Bexternal/mongo_toolchain/stow/gcc-v4/libexec/gcc/aarch64-mongodb-linux/11.3.0", + "-Bexternal/mongo_toolchain/stow/gcc-v4/lib/gcc/aarch64-mongodb-linux/11.3.0", + ], target_libc = "unknown", target_system_name = "local", tool_paths = {