265 lines
8.8 KiB
Python
265 lines
8.8 KiB
Python
"""Methods for working with Evergreen API."""
|
|
import logging
|
|
import os
|
|
import time
|
|
|
|
try:
|
|
from urllib.parse import urlparse
|
|
except ImportError:
|
|
from urllib.parse import urlparse # type: ignore
|
|
|
|
import requests
|
|
import yaml
|
|
|
|
from buildscripts.resmokelib import utils
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
DEFAULT_API_SERVER = "https://evergreen.mongodb.com"
|
|
|
|
EVERGREEN_FILES = ["./.evergreen.yml", "~/.evergreen.yml", "~/cli_bin/.evergreen.yml"]
|
|
|
|
|
|
def read_evg_config():
|
|
"""
|
|
Search known locations for the Evergreen config file.
|
|
|
|
Read the first config file that is found and return the results.
|
|
|
|
:return: Evergreen config dict or None.
|
|
"""
|
|
known_locations = [os.path.expanduser(evg_file) for evg_file in EVERGREEN_FILES]
|
|
for filename in known_locations:
|
|
if os.path.isfile(filename):
|
|
with open(filename, "r") as fstream:
|
|
return yaml.safe_load(fstream)
|
|
|
|
return None
|
|
|
|
|
|
def get_evergreen_headers():
|
|
"""Return the Evergreen API headers from the config file.
|
|
|
|
:return API headers dict.
|
|
"""
|
|
evg_config = read_evg_config()
|
|
evg_config = evg_config if evg_config is not None else {}
|
|
api_headers = {}
|
|
if evg_config.get("api_key"):
|
|
api_headers["api-key"] = evg_config["api_key"]
|
|
if evg_config.get("user"):
|
|
api_headers["api-user"] = evg_config["user"]
|
|
return api_headers
|
|
|
|
|
|
def get_evergreen_server():
|
|
"""
|
|
Determine the Evergreen server based on config files.
|
|
|
|
If it cannot be determined from config files, fallback to the default.
|
|
|
|
:return Evergreen API server string.
|
|
"""
|
|
evg_config = read_evg_config()
|
|
evg_config = evg_config if evg_config is not None else {}
|
|
api_server = "{url.scheme}://{url.netloc}".format(
|
|
url=urlparse(evg_config.get("api_server_host", DEFAULT_API_SERVER)))
|
|
|
|
return api_server
|
|
|
|
|
|
def get_evergreen_api():
|
|
"""Get an instance of the EvergreenApi object.
|
|
|
|
:return Evergreen API instance.
|
|
"""
|
|
return EvergreenApi(get_evergreen_server())
|
|
|
|
|
|
def get_evergreen_apiv2(**kwargs):
|
|
"""Get an instance of the EvergreenApiV2 object.
|
|
|
|
:param **kwargs: Named arguments passed to the EvergreenApiV2 class init.
|
|
:return Evergreen API V2 instance.
|
|
"""
|
|
return EvergreenApiV2(get_evergreen_server(), get_evergreen_headers(), **kwargs)
|
|
|
|
|
|
class EvergreenApi(object):
|
|
"""Module for interacting with the Evergreen API."""
|
|
|
|
def __init__(self, api_server=DEFAULT_API_SERVER, api_headers=None):
|
|
"""Initialize the object.
|
|
|
|
:param api_server: Evergreen API server string.
|
|
:param api_headers: Evergreen API headers dict.
|
|
"""
|
|
self.api_server = api_server
|
|
self.api_headers = api_headers
|
|
|
|
def get_history(self, project, params):
|
|
"""Get the test history from Evergreen.
|
|
|
|
:param project: Project string for query.
|
|
:param params: Params dict for API call.
|
|
:return: JSON dict response from API call.
|
|
"""
|
|
url = "{}/rest/v1/projects/{}/test_history".format(self.api_server, project)
|
|
|
|
start = time.time()
|
|
response = requests.get(url=url, params=params)
|
|
LOGGER.debug("Request took %fs:", round(time.time() - start, 2))
|
|
response.raise_for_status()
|
|
|
|
return response.json()
|
|
|
|
|
|
def _check_type(obj, instance_type):
|
|
"""Raise error if type mismatch.
|
|
|
|
:param obj: Object to check.
|
|
:param instance_type: Python type instance.
|
|
"""
|
|
if not isinstance(obj, instance_type):
|
|
raise TypeError("Type error mismatch, expected {}".format(instance_type))
|
|
|
|
|
|
def _add_list_param(params, param_name, param_list):
|
|
"""Add the params param_name to comma separated string from param_list.
|
|
|
|
:param params: Dict of param_name and param_value.
|
|
:param param_name: String name of param. Raise an exception if it already has been added.
|
|
:param param_list: List of param values to be converted to commad separated string.
|
|
"""
|
|
if param_list:
|
|
_check_type(param_list, list)
|
|
if param_name in params:
|
|
raise RuntimeError("Cannot add {} as it already exists".format(param_name))
|
|
params[param_name] = ",".join(param_list)
|
|
|
|
|
|
class EvergreenApiV2(EvergreenApi):
|
|
"""Module for interacting with the Evergreen V2 API."""
|
|
|
|
DEFAULT_GROUP_NUM_DAYS = 1
|
|
DEFAULT_LIMIT = 1000
|
|
DEFAULT_REQUESTERS = ["mainline"]
|
|
DEFAULT_RETRIES = 3
|
|
DEFAULT_SORT = "earliest"
|
|
|
|
def __init__(self, api_server=DEFAULT_API_SERVER, api_headers=None,
|
|
num_retries=DEFAULT_RETRIES):
|
|
"""Initialize the object.
|
|
|
|
:param api_server: API server string.
|
|
:param api_headers: API headers dict.
|
|
:param num_retries: Number of retries.
|
|
"""
|
|
super(EvergreenApiV2, self).__init__(api_server, api_headers)
|
|
self.session = requests.Session()
|
|
retry = requests.packages.urllib3.util.retry.Retry(
|
|
total=num_retries,
|
|
read=num_retries,
|
|
connect=num_retries,
|
|
backoff_factor=0.1, # Enable backoff starting at 0.1s.
|
|
status_forcelist=[
|
|
500, 502, 504
|
|
]) # We are not retrying 503 errors as they are used to indicate degraded service
|
|
adapter = requests.adapters.HTTPAdapter(max_retries=retry)
|
|
self.session.mount("{url.scheme}://".format(url=urlparse(api_server)), adapter)
|
|
if api_headers:
|
|
self.session.headers.update(api_headers)
|
|
|
|
def tests_by_task(self, task_id, execution, limit=100, status=None):
|
|
"""Return list of tests from the given task_id.
|
|
|
|
:param task_id: API Task ID string name.
|
|
:param execution: Execution number query parameter.
|
|
:param limit: Limit query parameter.
|
|
:param status: Status string query parameter to filter on.
|
|
:return: List of tests.
|
|
"""
|
|
params = {"execution": execution, "limit": limit}
|
|
if status:
|
|
params["status"] = status
|
|
url = "{}/rest/v2/tasks/{}/tests".format(self.api_server, task_id)
|
|
return self._paginate(url, params=params)
|
|
|
|
def project_patches_gen(self, project, limit=100):
|
|
"""Return generator for the project patches from Evergreen.
|
|
|
|
:param project: API project string name.
|
|
:param limit: Limit query parameter.
|
|
:return: Generator of project patches.
|
|
"""
|
|
params = {"limit": limit}
|
|
url = "{}/rest/v2/projects/{}/patches".format(self.api_server, project)
|
|
return self._paginate_gen(url, params)
|
|
|
|
def version_builds(self, version):
|
|
"""Return list of builds for the specified version from Evergreen.
|
|
|
|
:param version: API version string name.
|
|
:return: List of version builds.
|
|
"""
|
|
url = "{}/rest/v2/versions/{}/builds".format(self.api_server, version)
|
|
return self._paginate(url)
|
|
|
|
def _call_api(self, url, params=None):
|
|
"""Return requests.response object or None.
|
|
|
|
:param url: URL to retrieve.
|
|
:param params: Dict for query parameters.
|
|
:return: requests.response object.
|
|
"""
|
|
start_time = time.time()
|
|
response = self.session.get(url=url, params=params)
|
|
duration = round(time.time() - start_time, 2)
|
|
if duration > 10:
|
|
# If the request took over 10 seconds, increase the log level.
|
|
LOGGER.info("Request %s took %fs:", response.request.url, duration)
|
|
else:
|
|
LOGGER.debug("Request %s took %fs:", response.request.url, duration)
|
|
try:
|
|
response.raise_for_status()
|
|
except requests.exceptions.HTTPError as err:
|
|
LOGGER.error("Response text (%s): %s", response.request.url, response.text)
|
|
raise err
|
|
return response
|
|
|
|
def _paginate(self, url, params=None):
|
|
"""Paginate until all pages are requested and return a list of all JSON results.
|
|
|
|
:param url: URL to retrieve.
|
|
:param params: Dict for query parameters.
|
|
:return: Dict of all JSON data.
|
|
"""
|
|
return list(self._paginate_gen(url, params))
|
|
|
|
def _paginate_gen(self, url, params=None):
|
|
"""Return generator of items for each paginated page.
|
|
|
|
:param url: URL to retrieve.
|
|
:param params: Dict for query parameters.
|
|
:return: Generator for JSON data.
|
|
"""
|
|
while True:
|
|
response = self._call_api(url, params)
|
|
if not response:
|
|
break
|
|
json_response = response.json()
|
|
if json_response:
|
|
if isinstance(json_response, list):
|
|
for result in json_response:
|
|
yield result
|
|
else:
|
|
yield json_response
|
|
url = self._get_next_url(response)
|
|
if not url:
|
|
break
|
|
params = None
|
|
|
|
@staticmethod
|
|
def _get_next_url(response):
|
|
return response.links.get("next", {}).get("url")
|