312 lines
11 KiB
Python
312 lines
11 KiB
Python
import subprocess
|
|
import sys
|
|
import json
|
|
import platform
|
|
import os
|
|
import shlex
|
|
import concurrent.futures
|
|
import glob
|
|
import traceback
|
|
import shutil
|
|
from typing import Annotated
|
|
|
|
import typer
|
|
|
|
|
|
def main(
|
|
target_library: Annotated[str, typer.Option()],
|
|
silent: Annotated[bool, typer.Option()] = False,
|
|
skip_scons: Annotated[bool, typer.Option()] = False,
|
|
):
|
|
extra_args = []
|
|
if os.name == "nt":
|
|
extra_args += [
|
|
"CPPPATH=C:\sasl\include",
|
|
"LIBPATH=C:\sasl\lib",
|
|
]
|
|
target_library = os.path.join(
|
|
os.path.dirname(target_library), os.path.basename(target_library)[3:-2] + "lib"
|
|
)
|
|
|
|
if platform.system() == "Darwin":
|
|
target_library = target_library[:-2] + "a"
|
|
|
|
path = shutil.which("icecc")
|
|
if path is None:
|
|
extra_args += ["ICECC="]
|
|
|
|
cmd = [
|
|
sys.executable,
|
|
"buildscripts/scons.py",
|
|
"--build-profile=opt",
|
|
f"--bazel-includes-info={target_library}",
|
|
"--libdeps-linting=off",
|
|
"--ninja=disabled",
|
|
"compiledb",
|
|
] + extra_args
|
|
|
|
if not skip_scons:
|
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|
|
|
while True:
|
|
line = p.stdout.readline()
|
|
if not line:
|
|
break
|
|
print(line.strip(), file=sys.stderr)
|
|
|
|
_, _ = p.communicate()
|
|
|
|
if p.returncode != 0:
|
|
print(f"SCons build failed, exit code {p.returncode}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
with open("compile_commands.json") as f:
|
|
cc = json.load(f)
|
|
|
|
headers = set()
|
|
original_headers = set()
|
|
|
|
def get_headers(line):
|
|
try:
|
|
with open(target_library + ".bazel_headers") as f:
|
|
bazel_headers = [line.strip() for line in f.readlines()]
|
|
bazel_headers += [
|
|
"src/mongo/platform/basic.h",
|
|
"src/mongo/platform/windows_basic.h",
|
|
]
|
|
|
|
with open(target_library + ".env_vars") as f:
|
|
tmp_env_vars = json.load(f)
|
|
env_vars = {}
|
|
# subprocess requies only strings
|
|
for k, v in tmp_env_vars.items():
|
|
env_vars[str(k)] = str(v)
|
|
|
|
for command in cc:
|
|
cmd_output = command["output"].replace("\\", "/").strip("'").strip('"')
|
|
line_output = line.replace("\\", "/")
|
|
|
|
if cmd_output == line_output:
|
|
os.makedirs(os.path.dirname(line), exist_ok=True)
|
|
if os.name == "nt":
|
|
header_arg = " /showIncludes"
|
|
else:
|
|
header_arg = " -H"
|
|
|
|
if not silent:
|
|
print(f"compiling {line}")
|
|
|
|
p = subprocess.run(
|
|
shlex.split((command["command"].replace("\\", "/") + header_arg)),
|
|
env=env_vars,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if p.returncode != 0:
|
|
print(f"Error compiling, exitcode: {p.returncode}", file=sys.stderr)
|
|
print(f"STDOUT: {p.stdout}", file=sys.stderr)
|
|
print(f"STDERR: {p.stderr}", file=sys.stderr)
|
|
sys.exit(1)
|
|
if os.name == "nt":
|
|
for line in p.stdout.split("\n"):
|
|
line = (
|
|
line.replace("Note: including file:", "")
|
|
.strip(" ")
|
|
.replace("\\", "/")
|
|
)
|
|
|
|
if not line.startswith(os.getcwd().replace("\\", "/")):
|
|
continue
|
|
|
|
line = os.path.relpath(
|
|
line, start=os.getcwd().replace("\\", "/")
|
|
).replace("\\", "/")
|
|
if line not in bazel_headers:
|
|
if line.startswith("src/") or line.startswith("bazel-out/"):
|
|
original_headers.add(line)
|
|
line = "//" + line
|
|
line = ":".join(line.rsplit("/", 1))
|
|
|
|
headers.add(line)
|
|
else:
|
|
for line in p.stderr.split("\n"):
|
|
if ". src/" in line or ". bazel-out/" in line:
|
|
while line.startswith("."):
|
|
line = line[1:]
|
|
line = line.replace("\\", "/")
|
|
|
|
if line[1:] not in bazel_headers:
|
|
original_headers.add(line[1:])
|
|
line = "//" + line[1:]
|
|
line = ":".join(line.rsplit("/", 1))
|
|
|
|
headers.add(line)
|
|
except Exception as exc:
|
|
print(traceback.format_exc(), file=sys.stderr)
|
|
raise exc
|
|
|
|
sources = []
|
|
with open(target_library + ".obj_files") as f:
|
|
lines = f.readlines()
|
|
for line in lines:
|
|
line = line.strip()
|
|
line = line.replace("build/opt", "//src")
|
|
line = line[: line.find(".")] + ".cpp"
|
|
line = ":".join(line.rsplit("/", 1))
|
|
if line.endswith("_gen.cpp"):
|
|
line = line[:-4]
|
|
sources.append(line)
|
|
if platform.system() == "Linux":
|
|
cpu_count = len(os.sched_getaffinity(0)) + 4
|
|
else:
|
|
cpu_count = os.cpu_count() + 4
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor:
|
|
jobs = {executor.submit(get_headers, line.strip()): line.strip() for line in lines}
|
|
for completed_job in concurrent.futures.as_completed(jobs):
|
|
if not silent:
|
|
print(f"finished {jobs[completed_job]}")
|
|
|
|
with open(".bazel_include_info.json") as f:
|
|
bazel_include_info = json.load(f)
|
|
|
|
header_map = bazel_include_info["header_map"]
|
|
gen_header_map = bazel_include_info["gen_header_map"]
|
|
bazel_exec = bazel_include_info["bazel_exec"]
|
|
bazel_config = bazel_include_info["config"]
|
|
|
|
global_headers = (
|
|
"src/mongo:config.h",
|
|
"src/mongo/config.h",
|
|
"src/mongo/platform/basic.h",
|
|
"src/mongo/platform/windows_basic.h",
|
|
)
|
|
|
|
reverse_header_map = {}
|
|
for k, v in header_map.items():
|
|
for hdr in v:
|
|
if not hdr or hdr.endswith(global_headers):
|
|
continue
|
|
bazel_header = "//" + hdr.replace("\\", "/")
|
|
bazel_header = ":".join(bazel_header.rsplit("/", 1))
|
|
if bazel_header.startswith("//src/third_party/SafeInt"):
|
|
reverse_header_map[bazel_header] = ["//src/third_party/SafeInt:headers"]
|
|
elif bazel_header.startswith("//src/third_party/immer"):
|
|
reverse_header_map[bazel_header] = ["//src/third_party/immer:headers"]
|
|
elif bazel_header in reverse_header_map:
|
|
if bazel_header.startswith("//src/third_party/"):
|
|
continue
|
|
reverse_header_map[bazel_header].append(k)
|
|
else:
|
|
reverse_header_map[bazel_header] = [k]
|
|
|
|
for k, v in gen_header_map.items():
|
|
for hdr in v:
|
|
if not hdr or hdr.endswith(global_headers):
|
|
continue
|
|
bazel_header = "//" + hdr.replace("\\", "/")
|
|
bazel_header = ":".join(bazel_header.rsplit("/", 1))
|
|
if bazel_header not in reverse_header_map:
|
|
reverse_header_map[bazel_header] = [k]
|
|
else:
|
|
reverse_header_map[bazel_header].append(k)
|
|
|
|
recommended_deps = set()
|
|
minimal_headers = set()
|
|
for header in headers:
|
|
if header in reverse_header_map:
|
|
found = False
|
|
for lib in reverse_header_map[header]:
|
|
if lib in gen_header_map:
|
|
minimal_headers.add(lib)
|
|
found = True
|
|
break
|
|
if not found:
|
|
for lib in reverse_header_map[header]:
|
|
recommended_deps.add(lib)
|
|
else:
|
|
if not header.endswith(global_headers):
|
|
minimal_headers.add(header)
|
|
|
|
deps_order_by_height = []
|
|
deps_queries = {}
|
|
for dep in recommended_deps:
|
|
p = subprocess.run(
|
|
[bazel_exec, "cquery"] + bazel_config + [f'kind("extract_debuginfo", deps("@{dep}"))'],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
deps_queries[dep] = [
|
|
line.split(" ")[0] for line in p.stdout.splitlines() if line.startswith("//")
|
|
]
|
|
deps_order_by_height.append((dep, len(deps_queries[dep])))
|
|
|
|
deps_order_by_height.sort(key=lambda x: x[1])
|
|
|
|
deps_order_by_height = [dep[0] for dep in deps_order_by_height]
|
|
optimal_header_deps = set()
|
|
for header in headers:
|
|
if header in minimal_headers:
|
|
continue
|
|
|
|
path_header = "/".join(header.rsplit(":", 1))
|
|
path_header = path_header[2:]
|
|
for dep in deps_order_by_height:
|
|
if path_header in header_map[dep]:
|
|
optimal_header_deps.add(dep)
|
|
break
|
|
optimal_header_deps = list(optimal_header_deps)
|
|
|
|
working_deps = optimal_header_deps.copy()
|
|
for dep in optimal_header_deps:
|
|
if dep in working_deps:
|
|
for test_dep in optimal_header_deps:
|
|
if test_dep == dep:
|
|
continue
|
|
if test_dep in working_deps and test_dep in deps_queries[dep]:
|
|
working_deps.remove(test_dep)
|
|
|
|
with open(target_library + ".bazel_deps") as f:
|
|
original_deps = f.readlines()
|
|
|
|
link_deps = []
|
|
header_deps = []
|
|
for dep in sorted(list(working_deps) + list(set(original_deps))):
|
|
if dep in original_deps:
|
|
link_deps.append(dep)
|
|
else:
|
|
header_deps.append(dep)
|
|
|
|
target_name = os.path.splitext(os.path.basename(target_library))[0]
|
|
if target_name.startswith("lib"):
|
|
target_name = target_name[3:]
|
|
|
|
local_bazel_path = os.path.dirname(target_library.replace("build/opt", "//src")) + ":"
|
|
print("mongo_cc_library(")
|
|
print(f' name = "{target_name}",')
|
|
if sources:
|
|
print(f" srcs = [")
|
|
for src in sources:
|
|
print(f' "{src.replace(local_bazel_path, "")}",')
|
|
print(" ],")
|
|
if minimal_headers:
|
|
print(" hdrs = [")
|
|
for header in sorted(minimal_headers):
|
|
print(f' "{header.replace(local_bazel_path, "")}",')
|
|
print(" ],")
|
|
if header_deps:
|
|
print(" header_deps = [")
|
|
for dep in sorted(header_deps):
|
|
print(f' "{dep.strip().replace(local_bazel_path, "")}",')
|
|
print(" ],")
|
|
if link_deps:
|
|
print(" deps = [")
|
|
for dep in sorted(link_deps):
|
|
print(f' "{dep.strip().replace(local_bazel_path, "")}",')
|
|
print(" ],")
|
|
print(")")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
typer.run(main)
|