107 lines
4.9 KiB
Python
Executable File
107 lines
4.9 KiB
Python
Executable File
# Use to update remote_execution_containers.bzl with a new set of remote execution containers generated from
|
|
# the dockerfiles listed in bazel/remote_execution_container/.
|
|
|
|
import argparse
|
|
from datetime import datetime
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import tempfile
|
|
import urllib.request
|
|
|
|
|
|
def log_subprocess_run(*args, **kwargs):
|
|
arg_list_or_string = kwargs["args"] if "args" in kwargs else args[0]
|
|
print(" ".join(arg_list_or_string) if type(arg_list_or_string) == list else arg_list_or_string)
|
|
return subprocess.run(*args, **kwargs)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--distro", type=str,
|
|
help="Restrict to only update a single distro.")
|
|
parser.add_argument("--skip-cleanup", action="store_true",
|
|
help="Disable cleanup between container builds. This requires a large amount of disk space.")
|
|
args = parser.parse_args()
|
|
|
|
if not args.skip_cleanup:
|
|
user_input = input("""Warning: to build remote execution containers the docker data on the host system must be
|
|
purged after each step to preserve disk space.
|
|
|
|
Pass --skip-cleanup to disable this step, but be aware that this will require a large amount of disk space.
|
|
|
|
Your docker images, volumes and containers will be purged if you continue. Enter 'y' to continue or 'n' to exit:""")
|
|
if user_input.lower() != 'y':
|
|
return 0
|
|
|
|
remote_execution_containers = {}
|
|
container_file_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "remote_execution_containers.bzl")
|
|
with open(container_file_path, "r") as f:
|
|
code = compile(f.read(), container_file_path, "exec")
|
|
exec(code, {}, remote_execution_containers)
|
|
|
|
for distro, re_container in remote_execution_containers["REMOTE_EXECUTION_CONTAINERS"].items():
|
|
if args.distro is not None:
|
|
if distro != args.distro:
|
|
continue
|
|
|
|
if not args.skip_cleanup:
|
|
# Clean host system between container builds to avoid running into disk space issues.
|
|
print("Cleaning host system's docker images, containers, volumes, and networks...")
|
|
for command in [
|
|
"docker stop $(docker ps -a -q)", # Stop all running containers
|
|
"docker rm $(docker ps -a -q)", # Remove all containers
|
|
"docker rmi $(docker images -q)", # Remove all images
|
|
"docker volume rm $(docker volume ls -q)", # Remove all volumes
|
|
]:
|
|
log_subprocess_run(command, shell=True)
|
|
|
|
dockerfile = re_container["dockerfile"]
|
|
tag = f"quay.io/mongodb/bazel-remote-execution:{distro}-{datetime.now().strftime('%Y_%m_%d-%H_%M_%S')}"
|
|
|
|
print(f"Updating {distro} container...")
|
|
print(f"Using dockerfile: {dockerfile}")
|
|
print(f"Using tag: {tag}\n")
|
|
|
|
log_subprocess_run(["docker", "buildx", "create", "--use", "default"], check=True)
|
|
log_subprocess_run([
|
|
"docker", "buildx", "build", "--push",
|
|
"--platform", "linux/arm64/v8,linux/amd64",
|
|
"--tag", tag,
|
|
str(pathlib.Path(re_container["dockerfile"]).parent.resolve()),
|
|
], check=True)
|
|
|
|
log_subprocess_run(["docker", "pull", tag], check=True)
|
|
result = log_subprocess_run(["docker", "inspect", "--format='{{.RepoDigests}}'", tag], capture_output=True, text=True, check=True)
|
|
|
|
# The output of this command is a list of strings, ex. ['URL'] so we need to strip off the brackets and quotes.
|
|
re_container["container-url"] = "docker://" + result.stdout.strip()[2:-2]
|
|
re_container["web-url"] = "https://" + result.stdout.strip()[2:-2].replace("quay.io/", "quay.io/repository/").replace("@sha256", "/manifest/sha256")
|
|
|
|
print(f"Finished updating {distro}")
|
|
print("************************************\n")
|
|
|
|
with open(container_file_path, "w") as f:
|
|
print(f"Writing remote execution container map to {container_file_path}...")
|
|
print("# Use bazel/platforms/remote_execution_containers_generator.py to generate this mapping for a given patch build.\n", file=f)
|
|
|
|
# Manually print out the dict to maintain the trailing comma in each last element to satisfy the buildifier lint rules and
|
|
# avoid reformating.
|
|
print("REMOTE_EXECUTION_CONTAINERS = {", file=f)
|
|
for key, value in sorted(remote_execution_containers["REMOTE_EXECUTION_CONTAINERS"].items(), key=lambda x: x[0]):
|
|
print(f" \"{key}\": {{", file=f)
|
|
for subkey, subvalue in sorted(value.items(), key=lambda x: x[0]):
|
|
print(f" \"{subkey}\": \"{subvalue}\",", file=f)
|
|
print(" },", file=f)
|
|
print("}", file=f)
|
|
|
|
with open(container_file_path, "r") as f:
|
|
print(f"Finished writing to {container_file_path}:")
|
|
print(f.read())
|
|
|
|
return 0
|
|
|
|
if __name__ == '__main__':
|
|
exit(main())
|