diff --git a/buildscripts/resmokelib/setup_multiversion/config.py b/buildscripts/resmokelib/setup_multiversion/config.py new file mode 100644 index 00000000000..72cb82a4fb3 --- /dev/null +++ b/buildscripts/resmokelib/setup_multiversion/config.py @@ -0,0 +1,37 @@ +"""Setup multiversion config.""" +from typing import List + +SETUP_MULTIVERSION_CONFIG = "etc/setup_multiversion_config.yml" + + +class Buildvariant: + """Class represents buildvariant in setup multiversion config.""" + + name: str + edition: str + platform: str + architecture: str + versions: List[str] + + def __init__(self, buildvariant_yaml: dict): + """Initialize.""" + self.name = buildvariant_yaml.get("name", "") + self.edition = buildvariant_yaml.get("edition", "") + self.platform = buildvariant_yaml.get("platform", "") + self.architecture = buildvariant_yaml.get("architecture", "") + self.versions = buildvariant_yaml.get("versions", []) + + +class SetupMultiversionConfig: + """Class represents setup multiversion config.""" + + evergreen_projects: List[str] + evergreen_buildvariants: List[Buildvariant] + + def __init__(self, raw_yaml: dict): + """Initialize.""" + self.evergreen_projects = raw_yaml.get("evergreen_projects", []) + self.evergreen_buildvariants = [] + buildvariants_raw_yaml = raw_yaml.get("evergreen_buildvariants", "") + for buildvariant_yaml in buildvariants_raw_yaml: + self.evergreen_buildvariants.append(Buildvariant(buildvariant_yaml)) diff --git a/buildscripts/resmokelib/setup_multiversion/download.py b/buildscripts/resmokelib/setup_multiversion/download.py new file mode 100644 index 00000000000..fc9368dec0b --- /dev/null +++ b/buildscripts/resmokelib/setup_multiversion/download.py @@ -0,0 +1,141 @@ +"""Helper functions to download.""" +import contextlib +import errno +import os +import shutil +import tarfile +import tempfile +import zipfile + +import boto3 +import structlog +from botocore import UNSIGNED +from botocore.config import Config +from botocore.exceptions import ClientError + +S3_BUCKET = "mciuploads" + +LOGGER = structlog.getLogger(__name__) + + +class DownloadError(Exception): + """Errors in download.py.""" + + pass + + +def download_mongodb(url): + """Download file from S3 bucket by a given URL.""" + + if not url: + raise DownloadError("Download URL not found.") + + LOGGER.info("Downloading mongodb.", url=url) + s3_key = url.split('/', 3)[-1].replace(f"{S3_BUCKET}/", "") + filename = os.path.join(tempfile.gettempdir(), url.split('/')[-1]) + + LOGGER.debug("Downloading mongodb from S3.", s3_bucket=S3_BUCKET, s3_key=s3_key, + filename=filename) + s3_client = boto3.client("s3", config=Config(signature_version=UNSIGNED)) + try: + s3_client.download_file(S3_BUCKET, s3_key, filename) + except ClientError as s3_client_error: + LOGGER.error("Download failed due to S3 client error.") + raise s3_client_error + except Exception as ex: # pylint: disable=broad-except + LOGGER.error("Download failed.") + raise ex + else: + LOGGER.info("Download completed.", filename=filename) + + return filename + + +def extract_archive(archive_file, install_dir): + """Uncompress file and return root of extracted directory.""" + + LOGGER.info("Extracting archive data.", archive=archive_file, install_dir=install_dir) + temp_dir = tempfile.mkdtemp() + archive_name = os.path.basename(archive_file) + install_subdir, file_suffix = os.path.splitext(archive_name) + + if file_suffix == ".zip": + # Support .zip downloads, used for Windows binaries. + with zipfile.ZipFile(archive_file) as zip_handle: + first_file = zip_handle.namelist()[0] + zip_handle.extractall(temp_dir) + elif file_suffix == ".tgz": + # Support .tgz downloads, used for Linux binaries. + with contextlib.closing(tarfile.open(archive_file, "r:gz")) as tar_handle: + first_file = tar_handle.getnames()[0] + tar_handle.extractall(path=temp_dir) + else: + raise DownloadError(f"Unsupported file extension {file_suffix}") + + extracted_root_dir = os.path.join(temp_dir, os.path.dirname(first_file)) + temp_install_dir = tempfile.mkdtemp() + shutil.move(extracted_root_dir, os.path.join(temp_install_dir, install_subdir)) + + try: + os.makedirs(install_dir) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(install_dir): + pass + else: + raise + + already_downloaded = os.path.isdir(os.path.join(install_dir, install_subdir)) + if not already_downloaded: + shutil.move(os.path.join(temp_install_dir, install_subdir), install_dir) + + shutil.rmtree(temp_dir) + shutil.rmtree(temp_install_dir) + + installed_dir = os.path.join(install_dir, install_subdir) + LOGGER.info("Extract archive completed.", installed_dir=installed_dir) + + return installed_dir + + +def symlink_version(version, installed_dir, link_dir): + """Symlink the binaries in the 'installed_dir' to the 'link_dir'.""" + try: + os.makedirs(link_dir) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(link_dir): + pass + else: + raise + + for executable in os.listdir(os.path.join(installed_dir, "bin")): + + executable_name, executable_extension = os.path.splitext(executable) + link_name = f"{executable_name}-{version}{executable_extension}" + + try: + executable = os.path.join(installed_dir, "bin", executable) + executable_link = os.path.join(link_dir, link_name) + + if os.name == "nt": + # os.symlink is not supported on Windows, use a direct method instead. + def symlink_ms(source, symlink_name): + """Provide symlink for Windows.""" + import ctypes + csl = ctypes.windll.kernel32.CreateSymbolicLinkW + csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) + csl.restype = ctypes.c_ubyte + flags = 1 if os.path.isdir(source) else 0 + if csl(symlink_name, source.replace("/", "\\"), flags) == 0: + raise ctypes.WinError() + + os.symlink = symlink_ms + os.symlink(executable, executable_link) + LOGGER.debug("Symlink created.", executable=executable, executable_link=executable_link) + + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + + LOGGER.info("Symlinks for all executables are created in the directory.", link_dir=link_dir) diff --git a/buildscripts/resmokelib/setup_multiversion/evergreen_conn.py b/buildscripts/resmokelib/setup_multiversion/evergreen_conn.py new file mode 100644 index 00000000000..26511a71bbc --- /dev/null +++ b/buildscripts/resmokelib/setup_multiversion/evergreen_conn.py @@ -0,0 +1,143 @@ +"""Helper functions to interact with evergreen.""" +import os + +import structlog +from evergreen import RetryingEvergreenApi +from requests import HTTPError + +EVERGREEN_HOST = "https://evergreen.mongodb.com" +EVERGREEN_CONFIG_LOCATIONS = ( + # Common for machines in Evergreen + os.path.join(os.getcwd(), ".evergreen.yml"), + # Common for local machines + os.path.expanduser(os.path.join("~", ".evergreen.yml")), +) + +GENERIC_EDITION = "base" +GENERIC_PLATFORM = "linux_x86_64" +GENERIC_ARCHITECTURE = "x86_64" + +LOGGER = structlog.getLogger(__name__) + + +class EvergreenConnError(Exception): + """Errors in evergreen_conn.py.""" + + pass + + +def get_evergreen_api(evergreen_config): + """Return evergreen API.""" + config_to_pass = evergreen_config + if not config_to_pass: + # Pickup the first config file found in common locations. + for file in EVERGREEN_CONFIG_LOCATIONS: + if os.path.isfile(file): + config_to_pass = file + break + try: + evg_api = RetryingEvergreenApi.get_api(config_file=config_to_pass) + except Exception as ex: + LOGGER.error("Most likely something is wrong with evergreen config file.", + config_file=config_to_pass) + raise ex + else: + return evg_api + + +def get_buildvariant_name(config, edition, platform, architecture, major_minor_version): + """Return Evergreen buildvariant name.""" + + buildvariant_name = "" + evergreen_buildvariants = config.evergreen_buildvariants + + for buildvariant in evergreen_buildvariants: + if (buildvariant.edition == edition and buildvariant.platform == platform + and buildvariant.architecture == architecture): + versions = buildvariant.versions + if major_minor_version in versions: + buildvariant_name = buildvariant.name + break + elif not versions: + buildvariant_name = buildvariant.name + + return buildvariant_name + + +def get_generic_buildvariant_name(config, major_minor_version): + """Return Evergreen buildvariant name for generic platform.""" + + LOGGER.info("Falling back to generic architecture.", edition=GENERIC_EDITION, + platform=GENERIC_PLATFORM, architecture=GENERIC_ARCHITECTURE) + + generic_buildvariant_name = get_buildvariant_name( + config=config, edition=GENERIC_EDITION, platform=GENERIC_PLATFORM, + architecture=GENERIC_ARCHITECTURE, major_minor_version=major_minor_version) + + if not generic_buildvariant_name: + raise EvergreenConnError("Generic architecture buildvariant not found.") + + return generic_buildvariant_name + + +def get_evergreen_project_and_version(config, evg_api, commit_hash): + """Return evergreen project and version by commit hash.""" + + for evg_project in config.evergreen_projects: + try: + version_id = evg_project.replace("-", "_") + "_" + commit_hash + evg_version = evg_api.version_by_id(version_id) + except HTTPError: + continue + else: + LOGGER.debug("Found evergreen version.", + evergreen_version=f"{EVERGREEN_HOST}/version/{evg_version.version_id}") + return evg_project, evg_version + + raise EvergreenConnError(f"Evergreen version for commit hash {commit_hash} not found.") + + +def get_evergreen_versions(evg_api, evg_project): + """Return the list of evergreen versions by evergreen project name.""" + return evg_api.versions_by_project(evg_project) + + +def get_compile_artifact_urls(evg_api, evg_version, buildvariant_name): + """Return compile urls from buildvariant in Evergreen version.""" + compile_artifact_urls = {} + + try: + build_id = evg_version.build_variants_map[buildvariant_name] + except KeyError: + raise EvergreenConnError(f"Buildvariant {buildvariant_name} not found.") + else: + evg_build = evg_api.build_by_id(build_id) + LOGGER.debug("Found evergreen build.", evergreen_build=f"{EVERGREEN_HOST}/build/{build_id}") + evg_tasks = evg_build.get_tasks() + compile_task = None + push_task = None + + for evg_task in evg_tasks: + if evg_task.display_name == "compile": + compile_task = evg_task + if evg_task.display_name == "push": + push_task = evg_task + if compile_task and push_task: + break + + if compile_task and push_task and compile_task.status == push_task.status == "success": + LOGGER.info("Found successful evergreen tasks.", + compile_task=f"{EVERGREEN_HOST}/task/{compile_task.task_id}", + push_task=f"{EVERGREEN_HOST}/task/{push_task.task_id}") + evg_artifacts = compile_task.artifacts + for artifact in evg_artifacts: + compile_artifact_urls[artifact.name] = artifact.url + elif compile_task and push_task: + LOGGER.warning("Found evergreen tasks, but they are not both successful.", + compile_task=f"{EVERGREEN_HOST}/task/{compile_task.task_id}", + push_task=f"{EVERGREEN_HOST}/task/{push_task.task_id}") + else: + LOGGER.error("There are no `compile` and/or 'push' tasks in the evergreen build.", + evergreen_build=f"{EVERGREEN_HOST}/build/{build_id}") + + return compile_artifact_urls diff --git a/buildscripts/resmokelib/setup_multiversion/github_conn.py b/buildscripts/resmokelib/setup_multiversion/github_conn.py new file mode 100644 index 00000000000..83f31817a23 --- /dev/null +++ b/buildscripts/resmokelib/setup_multiversion/github_conn.py @@ -0,0 +1,28 @@ +"""Helper functions to interact with github.""" +from github import Github, GithubException + + +class GithubConnError(Exception): + """Errors in github_conn.py.""" + + pass + + +def get_git_tag_and_commit(github_oauth_token, version): + """Return git tag and commit hash by associating the version with git tag.""" + + github = Github(github_oauth_token) + repo = github.get_repo("mongodb/mongo") + + try: + git_ref_list = list(repo.get_git_matching_refs(f"tags/r{version}")) + # If git tag fully matches the version, it will be only one git_ref in the list, + # otherwise picking up the latest git_ref + git_ref = git_ref_list[-1] + git_tag = repo.get_git_tag(git_ref.object.sha) + git_commit = repo.get_commit(git_tag.object.sha) + + except (GithubException, IndexError): + raise GithubConnError(f"Commit hash for a version {version} not found.") + + return git_tag.tag, git_commit.sha diff --git a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py index a230476cba0..b8b2e46affb 100644 --- a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py +++ b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py @@ -7,102 +7,170 @@ enterprise builds. """ import logging import os +import re import sys -import tempfile -import boto3 import structlog -from botocore import UNSIGNED -from botocore.config import Config -from botocore.exceptions import ClientError -from evergreen import RetryingEvergreenApi +import yaml from buildscripts.resmokelib.plugin import PluginInterface, Subcommand +from buildscripts.resmokelib.setup_multiversion import config, download, evergreen_conn, github_conn SUBCOMMAND = "setup-multiversion" -EVERGREEN_CONFIG_LOCATIONS = ( - # Common for machines in Evergreen - os.path.join(os.getcwd(), ".evergreen.yml"), - # Common for local machines - os.path.expanduser(os.path.join("~", ".evergreen.yml")), -) -S3_BUCKET = "mciuploads" LOGGER = structlog.getLogger(__name__) -def setup_logging(): - """Enable INFO level logging.""" +def setup_logging(debug=False): + """Enable logging.""" + log_level = logging.DEBUG if debug else logging.INFO logging.basicConfig( format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", - level=logging.INFO, + level=log_level, stream=sys.stdout, ) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("s3transfer").setLevel(logging.WARNING) + logging.getLogger("botocore").setLevel(logging.WARNING) + logging.getLogger("boto3").setLevel(logging.WARNING) + logging.getLogger("evergreen").setLevel(logging.WARNING) + logging.getLogger("github").setLevel(logging.WARNING) structlog.configure(logger_factory=structlog.stdlib.LoggerFactory()) -def get_evergreen_api(evergreen_config): - """Return evergreen API.""" - config_to_pass = evergreen_config - if not config_to_pass: - # Pickup the first config file found in common locations. - for file in EVERGREEN_CONFIG_LOCATIONS: - if os.path.isfile(file): - config_to_pass = file - break - try: - evg_api = RetryingEvergreenApi.get_api(config_file=config_to_pass) - except Exception as ex: - LOGGER.error("Most likely something is wrong with evergreen config file.", - config_file=config_to_pass) - raise ex - else: - return evg_api - - -def download_mongodb(url): - """Download file from S3 bucket by a given URL.""" - - LOGGER.info("Downloading mongodb.", url=url) - s3_key = url.split('/', 3)[-1].replace(f"{S3_BUCKET}/", "") - filename = os.path.join(tempfile.gettempdir(), url.split('/')[-1]) - - LOGGER.info("Downloading mongodb from S3.", s3_bucket=S3_BUCKET, s3_key=s3_key, - filename=filename) - s3_client = boto3.client("s3", config=Config(signature_version=UNSIGNED)) - try: - s3_client.download_file(S3_BUCKET, s3_key, filename) - except ClientError as s3_client_error: - LOGGER.error("Download failed due to S3 client error.") - raise s3_client_error - except Exception as ex: # pylint: disable=broad-except - LOGGER.error("Download failed.") - raise ex - else: - LOGGER.info("Download completed.", filename=filename) - - return filename - - class SetupMultiversion(Subcommand): - """Main class for the hang analyzer subcommand.""" + """Main class for the setup multiversion subcommand.""" # pylint: disable=too-many-instance-attributes def __init__(self, options): """Initialize.""" - self.install_dir = options.install_dir - self.link_dir = options.link_dir - self.edition = options.edition.lower() - self.platform = options.platform.lower() - self.architecture = options.architecture.lower() + setup_logging(options.debug) + cwd = os.getcwd() + self.install_dir = os.path.join(cwd, options.install_dir) + self.link_dir = os.path.join(cwd, options.link_dir) + + self.edition = options.edition.lower() if options.edition else None + self.platform = options.platform.lower() if options.platform else None + self.architecture = options.architecture.lower() if options.architecture else None self.use_latest = options.use_latest self.versions = options.versions - self.evg_api = get_evergreen_api(options.evergreen_config) - self.git_token = options.git_token + + self.evg_api = evergreen_conn.get_evergreen_api(options.evergreen_config) + # In evergreen github oauth token is stored as `token ******`, so we remove the leading part + self.github_oauth_token = options.github_oauth_token.replace( + "token ", "") if options.github_oauth_token else None + with open(config.SETUP_MULTIVERSION_CONFIG) as file_handle: + raw_yaml = yaml.safe_load(file_handle) + self.config = config.SetupMultiversionConfig(raw_yaml) def execute(self): """Execute setup multiversion mongodb.""" - pass # Not implemented yet. + + for version in self.versions: + LOGGER.info("Setting up version.", version=version) + LOGGER.info("Fetching download URL from Evergreen.") + + try: + re.match(r"\d+\.\d+", version).group(0) + except AttributeError: + LOGGER.error( + "Input version is not recognized. Some correct examples: 4.0, 4.0.1, 4.0.0-rc0") + exit(1) + + try: + urls = {} + if self.use_latest: + urls = self.get_latest_urls(version) + if not urls: + LOGGER.warning("Latest URL is not available or not requested, " + "we fallback to getting the URL for the version.") + urls = self.get_urls(version) + + binaries_url = urls.get("Binaries", "") + mongodb_archive = download.download_mongodb(binaries_url) + installed_dir = download.extract_archive(mongodb_archive, self.install_dir) + os.remove(mongodb_archive) + download.symlink_version(version, installed_dir, self.link_dir) + + except (github_conn.GithubConnError, evergreen_conn.EvergreenConnError, + download.DownloadError) as ex: + LOGGER.error(ex) + exit(1) + + else: + LOGGER.info("Setup version completed.", version=version) + LOGGER.info("-" * 50) + + def get_latest_urls(self, version): + """Return latest urls.""" + urls = {} + + evg_project = f"mongodb-mongo-v{version}" + if evg_project not in self.config.evergreen_projects: + return urls + + LOGGER.debug("Found evergreen project.", evergreen_project=evg_project) + # Assuming that project names contain . version + major_minor_version = version + + buildvariant_name = self.get_buildvariant_name(major_minor_version) + LOGGER.debug("Found buildvariant.", buildvariant_name=buildvariant_name) + + evg_versions = evergreen_conn.get_evergreen_versions(self.evg_api, evg_project) + + for evg_version in evg_versions: + if buildvariant_name not in evg_version.build_variants_map: + buildvariant_name = self.fallback_to_generic_buildvariant(major_minor_version) + + curr_urls = evergreen_conn.get_compile_artifact_urls(self.evg_api, evg_version, + buildvariant_name) + if "Binaries" in curr_urls: + urls = curr_urls + break + + return urls + + def get_urls(self, version): + """Return urls.""" + git_tag, commit_hash = github_conn.get_git_tag_and_commit(self.github_oauth_token, version) + LOGGER.info("Found git attributes.", git_tag=git_tag, commit_hash=commit_hash) + + evg_project, evg_version = evergreen_conn.get_evergreen_project_and_version( + self.config, self.evg_api, commit_hash) + LOGGER.debug("Found evergreen project.", evergreen_project=evg_project) + try: + major_minor_version = re.findall(r"\d+\.\d+", evg_project)[-1] + except IndexError: + major_minor_version = "master" + + buildvariant_name = self.get_buildvariant_name(major_minor_version) + LOGGER.debug("Found buildvariant.", buildvariant_name=buildvariant_name) + if buildvariant_name not in evg_version.build_variants_map: + buildvariant_name = self.fallback_to_generic_buildvariant(major_minor_version) + + urls = evergreen_conn.get_compile_artifact_urls(self.evg_api, evg_version, + buildvariant_name) + + return urls + + def get_buildvariant_name(self, major_minor_version): + """Return buildvariant name. + + Wrapper around evergreen_conn.get_buildvariant_name(). + """ + + return evergreen_conn.get_buildvariant_name( + config=self.config, edition=self.edition, platform=self.platform, + architecture=self.architecture, major_minor_version=major_minor_version) + + def fallback_to_generic_buildvariant(self, major_minor_version): + """Return generic buildvariant name. + + Wrapper around evergreen_conn.get_generic_buildvariant_name(). + """ + + return evergreen_conn.get_generic_buildvariant_name(config=self.config, + major_minor_version=major_minor_version) class SetupMultiversionPlugin(PluginInterface): @@ -110,7 +178,6 @@ class SetupMultiversionPlugin(PluginInterface): def parse(self, subcommand, parser, parsed_args, **kwargs): """Parse command-line options.""" - setup_logging() if subcommand == SUBCOMMAND: return SetupMultiversion(parsed_args) @@ -128,33 +195,36 @@ class SetupMultiversionPlugin(PluginInterface): help="Directory to contain links to all binaries for each version " "in the install directory. [REQUIRED]") editions = ("base", "enterprise", "targeted") - parser.add_argument("-e", "--edition", dest="edition", choices=editions, default="base", + parser.add_argument("-e", "--edition", dest="edition", choices=editions, + default="enterprise", help="Edition of the build to download, [default: %(default)s].") parser.add_argument( "-p", "--platform", dest="platform", required=True, - help="Platform to download [REQUIRED]. Examples include: 'linux', " - "'ubuntu1804', 'osx', 'rhel62', 'windows'.") + help="Platform to download [REQUIRED]. " + f"Available platforms can be found in {config.SETUP_MULTIVERSION_CONFIG}.") parser.add_argument( "-a", "--architecture", dest="architecture", default="x86_64", help="Architecture to download, [default: %(default)s]. Examples include: " "'arm64', 'ppc64le', 's390x' and 'x86_64'.") parser.add_argument( "-u", "--useLatest", dest="use_latest", action="store_true", - help="If specified, the latest (nightly) version will be downloaded, " - "if it exists, for the version specified. For example, if specifying " - "version 3.2 for download, the nightly version for 3.2 will be " - "downloaded if it exists, otherwise the 'highest' version will be " - "downloaded, i.e., '3.2.17'") + help="If specified, the latest version from Evergreen will be downloaded, if it exists, " + "for the version specified. For example, if specifying version 4.4 for download, the latest " + "version from `mongodb-mongo-v4.4` Evergreen project will be downloaded. Otherwise the latest " + "by git tag version will be downloaded.") parser.add_argument( "versions", nargs="*", help= - "Examples: 4.2 4.2.1 4.4. If 'rc' is included in the version name, we'll use the exact rc, " + "Examples: 4.0, 4.0.1, 4.0.0-rc0. If 'rc' is included in the version name, we'll use the exact rc, " "otherwise we'll pull the highest non-rc version compatible with the version specified." ) parser.add_argument( - "--evergreen-config", dest="evergreen_config", + "-ec", "--evergreenConfig", dest="evergreen_config", help="Location of evergreen configuration file. If not specified it will look " - f"for it in the following locations: {EVERGREEN_CONFIG_LOCATIONS}") + f"for it in the following locations: {evergreen_conn.EVERGREEN_CONFIG_LOCATIONS}") parser.add_argument( - "--git-token", dest="git_token", help= - "In most cases it works without git auth. Otherwise you can pass OAth token to increase " - "the github API rate limit. See https://developer.github.com/v3/#rate-limiting") + "-gt", "--githubOauthToken", dest="github_oauth_token", + help="Set the token to increase your rate limit. In most cases it works without auth. " + "Otherwise you can pass OAuth token to increase the github API rate limit. See " + "https://developer.github.com/v3/#rate-limiting") + parser.add_argument("-d", "--debug", dest="debug", action="store_true", default=False, + help="Set DEBUG logging level.") diff --git a/buildscripts/setup_multiversion_mongodb.py b/buildscripts/setup_multiversion_mongodb.py index 51514ec2c1f..94e5d056816 100755 --- a/buildscripts/setup_multiversion_mongodb.py +++ b/buildscripts/setup_multiversion_mongodb.py @@ -19,9 +19,6 @@ import zipfile import requests import requests.exceptions -import yaml - -SETUP_MULTIVERSION_FILE = "etc/setup_multiversion_mapping.yml" def dump_stacks(_signal_num, _frame): # pylint: disable=unused-argument @@ -114,27 +111,6 @@ def download_file(url, file_name, download_retries=5): raise Exception("Unknown download problem for {} to file {}".format(url, file_name)) -def get_buildvariant_name(edition, platform, architecture, version): - """Return Evergreen buildvariant name.""" - - with open(SETUP_MULTIVERSION_FILE) as file_handle: - config_yaml = yaml.safe_load(file_handle) - - buildvariant_name = "" - for buildvariant in config_yaml.get("evergreen_buildvariants", []): - if (buildvariant.get("edition", "") == edition - and buildvariant.get("platform", "") == platform - and buildvariant.get("architecture", "") == architecture): - versions = buildvariant.get("versions", []) - if version in versions: - buildvariant_name = buildvariant.get("name", "") - break - elif not versions: - buildvariant_name = buildvariant.get("name", "") - - return buildvariant_name - - class MultiVersionDownloader(object): # pylint: disable=too-many-instance-attributes """Class to support multiversion downloads.""" diff --git a/buildscripts/tests/resmokelib/setup_multiversion/test_download.py b/buildscripts/tests/resmokelib/setup_multiversion/test_download.py new file mode 100644 index 00000000000..80dd877db79 --- /dev/null +++ b/buildscripts/tests/resmokelib/setup_multiversion/test_download.py @@ -0,0 +1,48 @@ +"""Unit tests for buildscripts/resmokelib/setup_multiversion/download.py.""" +# pylint: disable=missing-docstring,no-self-use +import os +import tempfile +import unittest + +from botocore.exceptions import ClientError +from mock import patch + +from buildscripts.resmokelib.setup_multiversion.download import download_mongodb, DownloadError + + +class TestDownloadMongodb(unittest.TestCase): + @patch("boto3.s3.transfer.S3Transfer.download_file") + def test_correct_url_1(self, mock_download_file): + url = "https://mciuploads.s3.amazonaws.com/mongodb-mongo-v4.4/ubuntu1804/f3699db3cf64566a0f87a84cdf9d2f26a6ebff73/binaries/mongo-mongodb_mongo_v4.4_ubuntu1804_f3699db3cf64566a0f87a84cdf9d2f26a6ebff73_20_11_02_22_53_54.tgz" + bucket = "mciuploads" + key = "mongodb-mongo-v4.4/ubuntu1804/f3699db3cf64566a0f87a84cdf9d2f26a6ebff73/binaries/mongo-mongodb_mongo_v4.4_ubuntu1804_f3699db3cf64566a0f87a84cdf9d2f26a6ebff73_20_11_02_22_53_54.tgz" + filename = os.path.join( + tempfile.gettempdir(), + "mongo-mongodb_mongo_v4.4_ubuntu1804_f3699db3cf64566a0f87a84cdf9d2f26a6ebff73_20_11_02_22_53_54.tgz" + ) + + download_mongodb(url) + mock_download_file.assert_called_with(bucket=bucket, key=key, filename=filename, + extra_args=None, callback=None) + + @patch("boto3.s3.transfer.S3Transfer.download_file") + def test_correct_url_2(self, mock_download_file): + url = "https://s3.amazonaws.com/mciuploads/mongodb-mongo-v3.4/ubuntu1604/68fadc9173d8565ff687b4b769700b48d35ca5d5/binaries/mongo-mongodb_mongo_v3.4_ubuntu1604_68fadc9173d8565ff687b4b769700b48d35ca5d5_20_03_31_15_44_18.tgz" + bucket = "mciuploads" + key = "mongodb-mongo-v3.4/ubuntu1604/68fadc9173d8565ff687b4b769700b48d35ca5d5/binaries/mongo-mongodb_mongo_v3.4_ubuntu1604_68fadc9173d8565ff687b4b769700b48d35ca5d5_20_03_31_15_44_18.tgz" + filename = os.path.join( + tempfile.gettempdir(), + "mongo-mongodb_mongo_v3.4_ubuntu1604_68fadc9173d8565ff687b4b769700b48d35ca5d5_20_03_31_15_44_18.tgz" + ) + + download_mongodb(url) + mock_download_file.assert_called_with(bucket=bucket, key=key, filename=filename, + extra_args=None, callback=None) + + def test_incorrect_url(self): + url = "https://mciuploads.s3.amazonaws.com/no-file-here" + self.assertRaises(ClientError, download_mongodb, url) + + def test_no_url(self): + url = "" + self.assertRaises(DownloadError, download_mongodb, url) diff --git a/buildscripts/tests/resmokelib/setup_multiversion/test_evergreen_conn.py b/buildscripts/tests/resmokelib/setup_multiversion/test_evergreen_conn.py new file mode 100644 index 00000000000..fcaa6c71299 --- /dev/null +++ b/buildscripts/tests/resmokelib/setup_multiversion/test_evergreen_conn.py @@ -0,0 +1,207 @@ +"""Unit tests for buildscripts/resmokelib/setup_multiversion/evergreen_conn.py.""" +# pylint: disable=missing-docstring,too-many-arguments +import unittest + +from evergreen import RetryingEvergreenApi +from mock import patch +from requests import HTTPError + +from buildscripts.resmokelib.setup_multiversion import evergreen_conn +from buildscripts.resmokelib.setup_multiversion.config import SetupMultiversionConfig + + +class TestGetEvgApi(unittest.TestCase): + def test_incorrect_evergreen_config(self): + evergreen_config = "some-random-file-i-hope-does-not-exist" + self.assertRaises(Exception, evergreen_conn.get_evergreen_api, evergreen_config) + + def test_not_passing_evergreen_config(self): + evergreen_config = None + evg_api = evergreen_conn.get_evergreen_api(evergreen_config) + self.assertIsInstance(evg_api, RetryingEvergreenApi) + + +class TestGetBuildvariantName(unittest.TestCase): + def setUp(self): + raw_yaml = { + "evergreen_buildvariants": [ + { + "name": "macos-any", + "edition": "base", + "platform": "osx", + "architecture": "x86_64", + }, + { + "name": "macos-4.0", + "edition": "base", + "platform": "osx", + "architecture": "x86_64", + "versions": ["4.0"], + }, + ] + } + self.config = SetupMultiversionConfig(raw_yaml) + + def test_version_4_0(self): + edition = "base" + platform = "osx" + architecture = "x86_64" + major_minor_version = "4.0" + + buildvariant_name = evergreen_conn.get_buildvariant_name( + config=self.config, edition=edition, platform=platform, architecture=architecture, + major_minor_version=major_minor_version) + self.assertEqual(buildvariant_name, "macos-4.0") + + def test_any_version(self): + edition = "base" + platform = "osx" + architecture = "x86_64" + major_minor_version = "any" + + buildvariant_name = evergreen_conn.get_buildvariant_name( + config=self.config, edition=edition, platform=platform, architecture=architecture, + major_minor_version=major_minor_version) + self.assertEqual(buildvariant_name, "macos-any") + + def test_buildvariant_not_found(self): + edition = "test" + platform = "test" + architecture = "test" + major_minor_version = "any" + + buildvariant_name = evergreen_conn.get_buildvariant_name( + config=self.config, edition=edition, platform=platform, architecture=architecture, + major_minor_version=major_minor_version) + self.assertEqual(buildvariant_name, "") + + +class TestGetGenericBuildvariantName(unittest.TestCase): + def setUp(self): + raw_yaml = { + "evergreen_buildvariants": [{ + "name": "generic-buildvariant-name", + "edition": evergreen_conn.GENERIC_EDITION, + "platform": evergreen_conn.GENERIC_PLATFORM, + "architecture": evergreen_conn.GENERIC_ARCHITECTURE, + "versions": ["3.4", "3.6", "4.0"], + }, ] + } + self.config = SetupMultiversionConfig(raw_yaml) + + def test_buildvariant_found(self): + major_minor_version = "4.0" + generic_buildvariant_name = evergreen_conn.get_generic_buildvariant_name( + config=self.config, major_minor_version=major_minor_version) + self.assertEqual(generic_buildvariant_name, "generic-buildvariant-name") + + def test_buildvarinat_not_found(self): + major_minor_version = "4.2" + self.assertRaises(evergreen_conn.EvergreenConnError, + evergreen_conn.get_generic_buildvariant_name, self.config, + major_minor_version) + + +class TestGetEvergreenProjectAndVersion(unittest.TestCase): + def setUp(self): + raw_yaml = {"evergreen_projects": [ + "mongodb-mongo-master", + "mongodb-mongo-v4.4", + ]} + self.config = SetupMultiversionConfig(raw_yaml) + + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi.version_by_id") + @patch("evergreen.api.EvergreenApi") + def test_version_found(self, mock_evg_api, mock_version_by_id, mock_version): + expected_evg_project = "mongodb-mongo-master" + mock_version_by_id.return_value = mock_version + + evg_project, evg_version = evergreen_conn.get_evergreen_project_and_version( + self.config, mock_evg_api, "commit_hash") + self.assertEqual(expected_evg_project, evg_project) + self.assertEqual(mock_version, evg_version) + + @patch("evergreen.api.EvergreenApi.version_by_id") + @patch("evergreen.api.EvergreenApi") + def test_version_not_found(self, mock_evg_api, mock_version_by_id): + mock_version_by_id.side_effect = HTTPError + self.assertRaises(evergreen_conn.EvergreenConnError, + evergreen_conn.get_evergreen_project_and_version, self.config, + mock_evg_api, "commit_hash") + + +class TestGetCompileArtifactUrls(unittest.TestCase): + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi") + def test_buildvariant_not_found(self, mock_evg_api, mock_version): + buildvariant_name = "test" + mock_version.build_variants_map = {"not-test": "build_id"} + self.assertRaises(evergreen_conn.EvergreenConnError, + evergreen_conn.get_compile_artifact_urls, mock_evg_api, mock_version, + buildvariant_name) + + @patch("evergreen.task.Artifact") + @patch("evergreen.task.Task") + @patch("evergreen.task.Task") + @patch("evergreen.build.Build") + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi") + def test_urls_found(self, mock_evg_api, mock_version, mock_build, mock_compile_task, + mock_push_task, mock_artifact): + expected_urls = { + "Binaries": + "https://mciuploads.s3.amazonaws.com/mongodb-mongo-master/ubuntu1804/90f767adbb1901d007ee4dd8714f53402d893669/binaries/mongo-mongodb_mongo_master_ubuntu1804_90f767adbb1901d007ee4dd8714f53402d893669_20_11_30_03_14_30.tgz" + } + mock_evg_api.build_by_id.return_value = mock_build + mock_artifact.name = "Binaries" + mock_artifact.url = expected_urls["Binaries"] + mock_compile_task.display_name = "compile" + mock_compile_task.artifacts = [mock_artifact] + mock_compile_task.status = "success" + mock_push_task.display_name = "push" + mock_push_task.status = "success" + mock_build.get_tasks.return_value = [mock_compile_task, mock_push_task] + + urls = evergreen_conn.get_compile_artifact_urls(mock_evg_api, mock_version, "test") + self.assertEqual(urls, expected_urls) + + @patch("evergreen.task.Task") + @patch("evergreen.task.Task") + @patch("evergreen.build.Build") + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi") + def test_push_task_failed(self, mock_evg_api, mock_version, mock_build, mock_compile_task, + mock_push_task): + mock_evg_api.build_by_id.return_value = mock_build + mock_compile_task.display_name = "compile" + mock_compile_task.status = "success" + mock_push_task.display_name = "push" + mock_push_task.status = "failed" + mock_build.get_tasks.return_value = [mock_compile_task, mock_push_task] + + urls = evergreen_conn.get_compile_artifact_urls(mock_evg_api, mock_version, "test") + self.assertEqual(urls, {}) + + @patch("evergreen.task.Task") + @patch("evergreen.build.Build") + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi") + def test_no_push_task(self, mock_evg_api, mock_version, mock_build, mock_compile_task): + mock_evg_api.build_by_id.return_value = mock_build + mock_compile_task.display_name = "compile" + mock_compile_task.status = "success" + mock_build.get_tasks.return_value = [mock_compile_task] + + urls = evergreen_conn.get_compile_artifact_urls(mock_evg_api, mock_version, "test") + self.assertEqual(urls, {}) + + @patch("evergreen.build.Build") + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi") + def test_no_tasks(self, mock_evg_api, mock_version, mock_build): + mock_evg_api.build_by_id.return_value = mock_build + mock_build.get_tasks.return_value = [] + + urls = evergreen_conn.get_compile_artifact_urls(mock_evg_api, mock_version, "test") + self.assertEqual(urls, {}) diff --git a/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py b/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py index c30c042373e..f63f878231b 100644 --- a/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py +++ b/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py @@ -1,34 +1,195 @@ """Unit tests for buildscripts/resmokelib/setup_multiversion/setup_multiversion.py.""" # pylint: disable=missing-docstring -import os import unittest +from argparse import Namespace -from evergreen import RetryingEvergreenApi +from mock import patch -from buildscripts.resmokelib.setup_multiversion.setup_multiversion import download_mongodb, get_evergreen_api +from buildscripts.resmokelib.setup_multiversion import evergreen_conn +from buildscripts.resmokelib.setup_multiversion.config import SetupMultiversionConfig +from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion -class TestGetEvgApi(unittest.TestCase): - """Unit tests for the get_evergreen_api() function.""" - - def test_incorrect_evergreen_config(self): - evergreen_config = "some-random-file-i-hope-does-not-exist" - self.assertRaises(Exception, get_evergreen_api, evergreen_config) - - def test_not_passing_evergreen_config(self): - evergreen_config = None - evg_api = get_evergreen_api(evergreen_config) - self.assertIsInstance(evg_api, RetryingEvergreenApi) +class TestSetupMultiversionBase(unittest.TestCase): + def setUp(self): + self.buildvariant_name = "buildvariant-name" + self.generic_buildvariant_name = "generic-buildvariant-name" + edition = "edition" + platform = "platform" + architecture = "architecture" + raw_yaml_config = { + "evergreen_projects": [ + "mongodb-mongo-master", + "mongodb-mongo-v4.4", + ], "evergreen_buildvariants": [ + { + "name": self.buildvariant_name, + "edition": edition, + "platform": platform, + "architecture": architecture, + }, + { + "name": self.generic_buildvariant_name, + "edition": evergreen_conn.GENERIC_EDITION, + "platform": evergreen_conn.GENERIC_PLATFORM, + "architecture": evergreen_conn.GENERIC_ARCHITECTURE, + }, + ] + } + options = Namespace( + install_dir="install", + link_dir="link", + edition=edition, + platform=platform, + architecture=architecture, + use_latest=False, + versions=["4.2.1"], + evergreen_config=None, + github_oauth_token=None, + debug=False, + ) + with patch("buildscripts.resmokelib.setup_multiversion.config.SetupMultiversionConfig" + ) as mock_config: + mock_config.return_value = SetupMultiversionConfig(raw_yaml_config) + self.setup_multiversion = SetupMultiversion(options) -class TestDownloadMongodb(unittest.TestCase): - """Unit tests for the download_mongodb() function.""" +class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): + def test_no_such_project(self): + """Project `mongodb-mongo-v4.2.1` doesn't exist.""" + version = "4.2.1" + urls = self.setup_multiversion.get_latest_urls(version) + self.assertEqual(urls, {}) - def test_download_mongodb(self): - url = "https://mciuploads.s3.amazonaws.com/mongodb-mongo-v4.4/ubuntu1804" \ - "/f3699db3cf64566a0f87a84cdf9d2f26a6ebff73/binaries" \ - "/mongo-mongodb_mongo_v4.4_ubuntu1804_f3699db3cf64566a0f87a84cdf9d2f26a6ebff73_20_11_02_22_53_54.tgz" - file = download_mongodb(url) - self.assertTrue(os.path.exists(file)) - if os.path.exists(file): - os.remove(file) + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi.versions_by_project") + @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + def test_no_compile_artifacts(self, mock_get_compile_artifact_urls, mock_versions_by_project, + mock_version): + mock_version.build_variants_map = {self.buildvariant_name: "build_id"} + mock_versions_by_project.return_value = [mock_version] + mock_get_compile_artifact_urls.return_value = {} + + urls = self.setup_multiversion.get_latest_urls("4.4") + self.assertEqual(urls, {}) + + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi.versions_by_project") + @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + def test_urls_found_on_last_version(self, mock_get_compile_artifact_urls, + mock_versions_by_project, mock_version): + expected_urls = { + "Binaries": + "https://mciuploads.s3.amazonaws.com/mongodb-mongo-master/ubuntu1804/90f767adbb1901d007ee4dd8714f53402d893669/binaries/mongo-mongodb_mongo_master_ubuntu1804_90f767adbb1901d007ee4dd8714f53402d893669_20_11_30_03_14_30.tgz" + } + + mock_version.build_variants_map = {self.buildvariant_name: "build_id"} + mock_versions_by_project.return_value = [mock_version] + mock_get_compile_artifact_urls.return_value = expected_urls + + urls = self.setup_multiversion.get_latest_urls("4.4") + self.assertEqual(urls, expected_urls) + + @patch("evergreen.version.Version") + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi.versions_by_project") + @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + def test_urls_found_on_not_last_version(self, mock_get_compile_artifact_urls, + mock_versions_by_project, mock_version, + mock_expected_version): + expected_urls = { + "Binaries": + "https://mciuploads.s3.amazonaws.com/mongodb-mongo-master/ubuntu1804/90f767adbb1901d007ee4dd8714f53402d893669/binaries/mongo-mongodb_mongo_master_ubuntu1804_90f767adbb1901d007ee4dd8714f53402d893669_20_11_30_03_14_30.tgz" + } + + mock_version.build_variants_map = {self.buildvariant_name: "build_id"} + mock_expected_version.build_variants_map = {self.buildvariant_name: "build_id"} + evg_versions = [mock_version for _ in range(3)] + evg_versions.append(mock_expected_version) + mock_versions_by_project.return_value = evg_versions + mock_get_compile_artifact_urls.side_effect = lambda evg_api, evg_version, buildvariant_name: { + (self.setup_multiversion.evg_api, mock_version, self.buildvariant_name): {}, + (self.setup_multiversion.evg_api, mock_expected_version, self.buildvariant_name): + expected_urls, + }[evg_api, evg_version, buildvariant_name] + + urls = self.setup_multiversion.get_latest_urls("4.4") + self.assertEqual(urls, expected_urls) + + @patch("evergreen.version.Version") + @patch("evergreen.api.EvergreenApi.versions_by_project") + @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + def test_fallback_to_generic_buildvariant(self, mock_get_compile_artifact_urls, + mock_versions_by_project, mock_version): + expected_urls = { + "Binaries": + "https://mciuploads.s3.amazonaws.com/mongodb-mongo-master/ubuntu1804/90f767adbb1901d007ee4dd8714f53402d893669/binaries/mongo-mongodb_mongo_master_ubuntu1804_90f767adbb1901d007ee4dd8714f53402d893669_20_11_30_03_14_30.tgz" + } + + mock_version.build_variants_map = {self.generic_buildvariant_name: "build_id"} + mock_versions_by_project.return_value = [mock_version] + mock_get_compile_artifact_urls.return_value = expected_urls + + urls = self.setup_multiversion.get_latest_urls("4.4") + self.assertEqual(urls, expected_urls) + + +class TestSetupMultiversionGetUrls(TestSetupMultiversionBase): + @patch("evergreen.version.Version") + @patch( + "buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_evergreen_project_and_version" + ) + @patch("buildscripts.resmokelib.setup_multiversion.github_conn.get_git_tag_and_commit") + @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + def test_urls_found(self, mock_get_compile_artifact_urls, mock_get_git_tag_and_commit, + mock_get_evergreen_project_and_version, mock_version): + expected_urls = { + "Binaries": + "https://mciuploads.s3.amazonaws.com/mongodb-mongo-master/ubuntu1804/90f767adbb1901d007ee4dd8714f53402d893669/binaries/mongo-mongodb_mongo_master_ubuntu1804_90f767adbb1901d007ee4dd8714f53402d893669_20_11_30_03_14_30.tgz" + } + + mock_get_git_tag_and_commit.return_value = ("git_tag", "commit_hash") + mock_version.build_variants_map = {self.buildvariant_name: "build_id"} + mock_get_evergreen_project_and_version.return_value = ("mongodb-mongo-v4.4", mock_version) + mock_get_compile_artifact_urls.return_value = expected_urls + + urls = self.setup_multiversion.get_urls("4.4.1") + self.assertEqual(urls, expected_urls) + + @patch("evergreen.version.Version") + @patch( + "buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_evergreen_project_and_version" + ) + @patch("buildscripts.resmokelib.setup_multiversion.github_conn.get_git_tag_and_commit") + @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + def test_urls_not_found(self, mock_get_compile_artifact_urls, mock_get_git_tag_and_commit, + mock_get_evergreen_project_and_version, mock_version): + mock_get_git_tag_and_commit.return_value = ("git_tag", "commit_hash") + mock_version.build_variants_map = {self.buildvariant_name: "build_id"} + mock_get_evergreen_project_and_version.return_value = ("mongodb-mongo-v4.4", mock_version) + mock_get_compile_artifact_urls.return_value = {} + + urls = self.setup_multiversion.get_urls("4.4.1") + self.assertEqual(urls, {}) + + @patch("evergreen.version.Version") + @patch( + "buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_evergreen_project_and_version" + ) + @patch("buildscripts.resmokelib.setup_multiversion.github_conn.get_git_tag_and_commit") + @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + def test_fallback_to_generic_buildvariant(self, mock_get_compile_artifact_urls, + mock_get_git_tag_and_commit, + mock_get_evergreen_project_and_version, mock_version): + expected_urls = { + "Binaries": + "https://mciuploads.s3.amazonaws.com/mongodb-mongo-master/ubuntu1804/90f767adbb1901d007ee4dd8714f53402d893669/binaries/mongo-mongodb_mongo_master_ubuntu1804_90f767adbb1901d007ee4dd8714f53402d893669_20_11_30_03_14_30.tgz" + } + + mock_get_git_tag_and_commit.return_value = ("git_tag", "commit_hash") + mock_version.build_variants_map = {self.generic_buildvariant_name: "build_id"} + mock_get_evergreen_project_and_version.return_value = ("mongodb-mongo-v4.4", mock_version) + mock_get_compile_artifact_urls.return_value = expected_urls + + urls = self.setup_multiversion.get_urls("4.4.1") + self.assertEqual(urls, expected_urls) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 9ba9dcb78fb..1289a090494 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -10041,6 +10041,7 @@ buildvariants: multiversion_platform: windows_x86_64-2008plus-ssl multiversion_platform_42_or_later: windows_x86_64-2012plus multiversion_platform_44_or_later: windows + multiversion_edition: base content_type: application/zip compile_flags: --ssl MONGO_DISTMOD=windows -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) --win-version-min=win10 num_scons_link_jobs_available: 0.5 diff --git a/etc/pip/components/resmoke.req b/etc/pip/components/resmoke.req index 0457d28259d..7a21b21e194 100644 --- a/etc/pip/components/resmoke.req +++ b/etc/pip/components/resmoke.req @@ -10,3 +10,4 @@ grpcio == 1.28.1; platform_machine == "x86_64" grpcio-tools == 1.28.1; platform_machine == "x86_64" googleapis-common-protos == 1.51.0; platform_machine == "x86_64" blackduck == 0.0.51 +PyGithub == 1.53 \ No newline at end of file diff --git a/etc/setup_multiversion_config.yml b/etc/setup_multiversion_config.yml new file mode 100644 index 00000000000..642f58db157 --- /dev/null +++ b/etc/setup_multiversion_config.yml @@ -0,0 +1,308 @@ +evergreen_projects: + - mongodb-mongo-master + - mongodb-mongo-v4.8 + - mongodb-mongo-v4.7 + - mongodb-mongo-v4.4 + - mongodb-mongo-v4.2 + - mongodb-mongo-v4.0 + - mongodb-mongo-v3.6 + - mongodb-mongo-v3.4 + + +evergreen_buildvariants: + + - name: ubuntu1604 + edition: targeted + platform: ubuntu1604 + architecture: x86_64 + + - name: enterprise-ubuntu1604-64 + edition: enterprise + platform: ubuntu1604 + architecture: x86_64 + + - name: enterprise-ubuntu1604-arm64 + edition: enterprise + platform: ubuntu1604 + architecture: aarch64 + + - name: enterprise-ubuntu1604-arm64 + edition: enterprise + platform: ubuntu1604 + architecture: arm64 + versions: ["3.4", "3.6", "4.0"] + + - name: ubuntu1804 + edition: targeted + platform: ubuntu1804 + architecture: x86_64 + + - name: enterprise-ubuntu1804-64 + edition: enterprise + platform: ubuntu1804 + architecture: x86_64 + + - name: ubuntu1804-arm64 + edition: targeted + platform: ubuntu1804 + architecture: aarch64 + + - name: enterprise-ubuntu1804-arm64 + edition: enterprise + platform: ubuntu1804 + architecture: aarch64 + + - name: enterprise-ubuntu1804-ppc64le + edition: enterprise + platform: ubuntu1804 + architecture: ppc64le + + - name: ubuntu1804-s390x + edition: targeted + platform: ubuntu1804 + architecture: s390x + + - name: enterprise-ubuntu1804-s390x + edition: enterprise + platform: ubuntu1804 + architecture: s390x + + - name: ubuntu2004 + edition: targeted + platform: ubuntu2004 + architecture: x86_64 + + - name: enterprise-ubuntu2004-64 + edition: enterprise + platform: ubuntu2004 + architecture: x86_64 + + - name: ubuntu2004-arm64 + edition: targeted + platform: ubuntu2004 + architecture: aarch64 + + - name: enterprise-ubuntu2004-arm64 + edition: enterprise + platform: ubuntu2004 + architecture: aarch64 + + - name: amazon + edition: targeted + platform: amazon + architecture: x86_64 + + - name: enterprise-linux-64-amazon-ami + edition: enterprise + platform: amazon + architecture: x86_64 + + - name: amazon2 + edition: targeted + platform: amazon2 + architecture: x86_64 + + - name: enterprise-amazon2 + edition: enterprise + platform: amazon2 + architecture: x86_64 + + - name: rhel62 + edition: targeted + platform: rhel62 + architecture: x86_64 + + - name: enterprise-rhel-62-64-bit + edition: enterprise + platform: rhel62 + architecture: x86_64 + + - name: rhel70 + edition: targeted + platform: rhel70 + architecture: x86_64 + + - name: enterprise-rhel-70-64-bit + edition: enterprise + platform: rhel70 + architecture: x86_64 + + - name: rhel80 + edition: targeted + platform: rhel80 + architecture: x86_64 + + - name: enterprise-rhel-80-64-bit + edition: enterprise + platform: rhel80 + architecture: x86_64 + + - name: rhel-82-arm64 + edition: targeted + platform: rhel82 + architecture: arm64 + + - name: enterprise-rhel-82-arm64 + edition: enterprise + platform: rhel82 + architecture: arm64 + + - name: enterprise-rhel-71-ppc64le + edition: enterprise + platform: rhel71 + architecture: ppc64le + + - name: enterprise-rhel-81-ppc64le + edition: enterprise + platform: rhel81 + architecture: ppc64le + + - name: rhel-67-s390x + edition: targeted + platform: rhel67 + architecture: s390x + + - name: enterprise-rhel-67-s390x + edition: enterprise + platform: rhel67 + architecture: s390x + + - name: rhel-72-s390x + edition: targeted + platform: rhel72 + architecture: s390x + + - name: enterprise-rhel-72-s390x + edition: enterprise + platform: rhel72 + architecture: s390x + + - name: suse12 + edition: targeted + platform: suse12 + architecture: x86_64 + + - name: enterprise-suse12-64 + edition: enterprise + platform: suse12 + architecture: x86_64 + + - name: suse12-s390x + edition: targeted + platform: suse12 + architecture: s390x + + - name: enterprise-suse12-s390x + edition: enterprise + platform: suse12 + architecture: s390x + + - name: suse15 + edition: targeted + platform: suse15 + architecture: x86_64 + + - name: enterprise-suse15-64 + edition: enterprise + platform: suse15 + architecture: x86_64 + + - name: debian92 + edition: targeted + platform: debian92 + architecture: x86_64 + + - name: enterprise-debian92-64 + edition: enterprise + platform: debian92 + architecture: x86_64 + + - name: debian10 + edition: targeted + platform: debian10 + architecture: x86_64 + + - name: enterprise-debian10-64 + edition: enterprise + platform: debian10 + architecture: x86_64 + + - name: windows + edition: base + platform: windows + architecture: x86_64 + + - name: windows-64-2k8-ssl + edition: base + platform: windows_x86_64-2012plus + architecture: x86_64 + versions: ["4.2"] + + - name: windows-64-2k8-ssl + edition: base + platform: windows_x86_64-2008plus-ssl + architecture: x86_64 + versions: ["3.4", "3.6", "4.0"] + + - name: enterprise-windows + edition: enterprise + platform: windows + architecture: x86_64 + + - name: enterprise-windows-64-2k8 + edition: enterprise + platform: windows + architecture: x86_64 + versions: ["3.6", "4.0", "4.2"] + + - name: enterprise-windows-64 + edition: enterprise + platform: windows + architecture: x86_64 + versions: ["3.4"] + + - name: macos + edition: base + platform: osx + architecture: x86_64 + + - name: osx-1010 + edition: base + platform: osx-ssl + architecture: x86_64 + versions: ["4.0"] + + - name: osx-1010-ssl + edition: base + platform: osx-ssl + architecture: x86_64 + versions: ["3.6"] + + - name: osx-108-ssl + edition: base + platform: osx-ssl + architecture: x86_64 + versions: ["3.4"] + + - name: enterprise-macos + edition: enterprise + platform: osx + architecture: x86_64 + + - name: enterprise-osx-1010 + edition: enterprise + platform: osx + architecture: x86_64 + versions: ["3.6", "4.0"] + + - name: enterprise-osx-108 + edition: enterprise + platform: osx + architecture: x86_64 + versions: ["3.4"] + + - name: linux-64 + edition: base + platform: linux_x86_64 + architecture: x86_64 + versions: ["3.4", "3.6", "4.0"] diff --git a/etc/setup_multiversion_mapping.yml b/etc/setup_multiversion_mapping.yml deleted file mode 100644 index 6e69f7dbeeb..00000000000 --- a/etc/setup_multiversion_mapping.yml +++ /dev/null @@ -1,291 +0,0 @@ -evergreen_buildvariants: - -- name: ubuntu1604 - edition: targeted - platform: ubuntu1604 - architecture: x86_64 - -- name: enterprise-ubuntu1604-64 - edition: enterprise - platform: ubuntu1604 - architecture: x86_64 - -- name: enterprise-ubuntu1604-arm64 - edition: enterprise - platform: ubuntu1604 - architecture: aarch64 - -- name: enterprise-ubuntu1604-arm64 - edition: enterprise - platform: ubuntu1604 - architecture: arm64 - versions: ["3.4", "3.6", "4.0"] - -- name: ubuntu1804 - edition: targeted - platform: ubuntu1804 - architecture: x86_64 - -- name: enterprise-ubuntu1804-64 - edition: enterprise - platform: ubuntu1804 - architecture: x86_64 - -- name: ubuntu1804-arm64 - edition: targeted - platform: ubuntu1804 - architecture: aarch64 - -- name: enterprise-ubuntu1804-arm64 - edition: enterprise - platform: ubuntu1804 - architecture: aarch64 - -- name: enterprise-ubuntu1804-ppc64le - edition: enterprise - platform: ubuntu1804 - architecture: ppc64le - -- name: ubuntu1804-s390x - edition: targeted - platform: ubuntu1804 - architecture: s390x - -- name: enterprise-ubuntu1804-s390x - edition: enterprise - platform: ubuntu1804 - architecture: s390x - -- name: ubuntu2004 - edition: targeted - platform: ubuntu2004 - architecture: x86_64 - -- name: enterprise-ubuntu2004-64 - edition: enterprise - platform: ubuntu2004 - architecture: x86_64 - -- name: ubuntu2004-arm64 - edition: targeted - platform: ubuntu2004 - architecture: aarch64 - -- name: enterprise-ubuntu2004-arm64 - edition: enterprise - platform: ubuntu2004 - architecture: aarch64 - -- name: amazon - edition: targeted - platform: amazon - architecture: x86_64 - -- name: enterprise-linux-64-amazon-ami - edition: enterprise - platform: amazon - architecture: x86_64 - -- name: amazon2 - edition: targeted - platform: amazon2 - architecture: x86_64 - -- name: enterprise-amazon2 - edition: enterprise - platform: amazon2 - architecture: x86_64 - -- name: rhel62 - edition: targeted - platform: rhel62 - architecture: x86_64 - -- name: enterprise-rhel-62-64-bit - edition: enterprise - platform: rhel62 - architecture: x86_64 - -- name: rhel70 - edition: targeted - platform: rhel70 - architecture: x86_64 - -- name: enterprise-rhel-70-64-bit - edition: enterprise - platform: rhel70 - architecture: x86_64 - -- name: rhel80 - edition: targeted - platform: rhel80 - architecture: x86_64 - -- name: enterprise-rhel-80-64-bit - edition: enterprise - platform: rhel80 - architecture: x86_64 - -- name: rhel-82-arm64 - edition: targeted - platform: rhel82 - architecture: arm64 - -- name: enterprise-rhel-82-arm64 - edition: enterprise - platform: rhel82 - architecture: arm64 - -- name: enterprise-rhel-71-ppc64le - edition: enterprise - platform: rhel71 - architecture: ppc64le - -- name: enterprise-rhel-81-ppc64le - edition: enterprise - platform: rhel81 - architecture: ppc64le - -- name: rhel-67-s390x - edition: targeted - platform: rhel67 - architecture: s390x - -- name: enterprise-rhel-67-s390x - edition: enterprise - platform: rhel67 - architecture: s390x - -- name: rhel-72-s390x - edition: targeted - platform: rhel72 - architecture: s390x - -- name: enterprise-rhel-72-s390x - edition: enterprise - platform: rhel72 - architecture: s390x - -- name: suse12 - edition: targeted - platform: suse12 - architecture: x86_64 - -- name: enterprise-suse12-64 - edition: enterprise - platform: suse12 - architecture: x86_64 - -- name: suse12-s390x - edition: targeted - platform: suse12 - architecture: s390x - -- name: enterprise-suse12-s390x - edition: enterprise - platform: suse12 - architecture: s390x - -- name: suse15 - edition: targeted - platform: suse15 - architecture: x86_64 - -- name: enterprise-suse15-64 - edition: enterprise - platform: suse15 - architecture: x86_64 - -- name: debian92 - edition: targeted - platform: debian92 - architecture: x86_64 - -- name: enterprise-debian92-64 - edition: enterprise - platform: debian92 - architecture: x86_64 - -- name: debian10 - edition: targeted - platform: debian10 - architecture: x86_64 - -- name: enterprise-debian10-64 - edition: enterprise - platform: debian10 - architecture: x86_64 - -- name: windows - edition: targeted - platform: windows - architecture: x86_64 - -- name: windows-64-2k8-ssl - edition: targeted - platform: windows_x86_64-2012plus - architecture: x86_64 - versions: ["4.2"] - -- name: windows-64-2k8-ssl - edition: targeted - platform: windows_x86_64-2008plus-ssl - architecture: x86_64 - versions: ["3.4", "3.6", "4.0"] - -- name: enterprise-windows - edition: enterprise - platform: windows - architecture: x86_64 - -- name: enterprise-windows-64-2k8 - edition: enterprise - platform: windows - architecture: x86_64 - versions: ["3.6", "4.0", "4.2"] - -- name: enterprise-windows-64 - edition: enterprise - platform: windows - architecture: x86_64 - versions: ["3.4"] - -- name: macos - edition: targeted - platform: osx - architecture: x86_64 - -- name: osx-1010 - edition: targeted - platform: osx - architecture: x86_64 - versions: ["4.0"] - -- name: osx-1010-ssl - edition: targeted - platform: osx - architecture: x86_64 - versions: ["3.6"] - -- name: osx-108-ssl - edition: targeted - platform: osx - architecture: x86_64 - versions: ["3.4"] - -- name: enterprise-macos - edition: enterprise - platform: osx - architecture: x86_64 - -- name: enterprise-osx-1010 - edition: enterprise - platform: osx - architecture: x86_64 - versions: ["3.6", "4.0"] - -- name: enterprise-osx-108 - edition: enterprise - platform: osx - architecture: x86_64 - versions: ["3.4"]