Files
mongo/etc/system_perf.yml

742 lines
22 KiB
YAML

stepback: false
command_type: system
pre:
post:
- command: shell.exec
params:
working_dir: work
script: |
source ./dsienv.sh
make_artifact.sh
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: work/reports.tgz
remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-${execution}.${ext|tgz}
bucket: mciuploads
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: test-log - Execution ${execution}
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: src/workloads/workloads/jsdoc/jsdocs-redirect.html
remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/workloads-${task_name}-${build_id}.html
bucket: mciuploads
permissions: public-read
content_type: text/html
display_name: workloads documentation
- command: attach.results
params:
file_location: work/report.json
- command: "json.send"
params:
name: "perf"
file: "work/perf.json"
- command: shell.exec
params:
working_dir: work
script: |
source ./dsienv.sh
if [ -e /data/infrastructure_provisioning/terraform/provisioned.${cluster} ]; then
mark_idle.sh
fi
- command: shell.exec
params:
working_dir: src
script: |
# removes files from the (local) scons cache when it's over a
# threshold, to the $prune_ratio percentage. Ideally override
# these default values in the distro config in evergreen.
if [ -d "${scons_cache_path}" ]; then
${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
fi
functions:
"prepare environment":
- command: shell.exec
params:
script: |
rm -rf ./*
mkdir src
mkdir work
- command: manifest.load
- command: git.get_project
params:
directory: src
revisions: # for each module include revision as <module_name> : ${<module_name>_rev}
dsi: ${dsi_rev}
workloads: ${workloads_rev}
- command: shell.exec
params:
working_dir: work
script: |
cat > bootstrap.yml <<EOF
cluster_type: ${cluster}
platform: ${platform}
setup: ${setup}
storageEngine: ${storageEngine}
test: ${test}
production: true
mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz"
EOF
cat > runtime.yml <<EOF
# evergreen default expansions
is_patch: ${is_patch}
task_id: ${task_id}
task_name: ${task_name}
execution: ${execution}
build_id: ${build_id}
build_variant: ${build_variant}
version_id: ${version_id}
workdir: ${workdir}
revision: ${revision}
project: ${project}
branch_name: ${branch_name}
# sys-perf expansions
# Shouldn't be needed: testList: ${testList}
ext: ${ext}
script_flags : ${script_flags}
dsi_rev: ${dsi_rev}
workloads_rev: ${workloads_rev}
EOF
- command: shell.exec
params:
silent: true
working_dir: work
script: |
# AWS ssh secret key
echo "${ec2_pem}" > aws_ssh_key.pem
chmod 400 aws_ssh_key.pem
cat > runtime_secret.yml <<EOF
# Note that inside system_perf.yml we have ${aws_key} & ${aws_secret}, which are used for
# Evergreen resources. The below are used for dsi resources, and are NOT the same!
aws_access_key: "${terraform_key}"
aws_secret_key: "${terraform_secret}"
perf_jira_user: "${perf_jira_user}"
perf_jira_pw: "${perf_jira_pw}"
EOF
chmod 400 runtime_secret.yml
- command: shell.exec
params:
working_dir: work
# setup execution environment
# configure environment, has private information, no logging
script: |
set -e
virtualenv ./venv
source ./venv/bin/activate
pip install -r ../src/dsi/dsi/requirements.txt
python ../src/dsi/dsi/bin/bootstrap.py
- command: shell.exec
params:
script: |
set -v
set -e
source work/dsienv.sh
setup-dsi-env.sh
ls -a work
"deploy cluster":
- command: shell.exec
params:
working_dir: work
script: |
set -e
set -v
source ./dsienv.sh
source ./venv/bin/activate
infrastructure_provisioning.py
mongodb_setup.py
"run test":
- command: shell.exec
type: test
params:
working_dir: work
script: |
set -e
set -v
source ./dsienv.sh
source ./venv/bin/activate
test_control.py
- command: "json.send"
params:
name: "perf"
file: "work/perf.json"
"make test log artifact":
- command: shell.exec
params:
working_dir: work
script: |
source ./dsienv.sh
make_artifact.sh
"analyze":
- command: json.get_history
params:
task: ${task_name}
file: "work/history.json"
name: "perf"
- command: json.get_history
params:
tags: true
task: ${task_name}
file: "work/tags.json"
name: "perf"
- command: shell.exec
# generate dashboard data
type : test
params:
working_dir: work
silent: true
script: |
set -o errexit
TAGS="3.4.9-Baseline 3.2.17-Baseline"
OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json"
python -u ../src/dsi/dsi/analysis/dashboard_gen.py --rev ${revision} -f history.json -t tags.json --refTag $TAGS --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} --jira-user ${perf_jira_user} --jira-password ${perf_jira_pw} || true
- command: "json.send"
params:
name: "dashboard"
file: "work/dashboard.json"
- command: shell.exec
# post_run_check.py and override.json for DSI tests are part of dsi repo
type : test
params:
working_dir: work
script: |
set -o errexit
set -o verbose
TAG="3.4.9-Baseline"
OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json"
python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file reports/perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name}
#######################################
# Tasks #
#######################################
tasks:
- name: compile
commands:
- command: manifest.load
- command: git.get_project
params:
directory: src
- command: shell.exec
params:
working_dir: src
script: |
set -o errexit
set -o verbose
# We get the raw version string (r1.2.3-45-gabcdef) from git
MONGO_VERSION=$(git describe)
# If this is a patch build, we add the patch version id to the version string so we know
# this build was a patch, and which evergreen task it came from
if [ "${is_patch|}" = "true" ]; then
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
fi
# This script converts the generated version string into a sanitized version string for
# use by scons and uploading artifacts as well as information about for the scons cache.
MONGO_VERSION=$MONGO_VERSION USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml
# Then we load the generated version data into the agent so we can use it in task definitions
- command: expansions.update
params:
file: src/compile_expansions.yml
- command: shell.exec
params:
working_dir: src/src/mongo/gotools
script: |
set -o verbose
set -o errexit
# make sure newlines in the scripts are handled correctly by windows
if [ "Windows_NT" = "$OS" ]; then
set -o igncr
fi;
sed -i.bak "s/built-without-version-string/$(git describe)/" common/options/options.go
sed -i.bak "s/built-without-git-spec/$(git rev-parse HEAD)/" common/options/options.go
. ./${set_tools_gopath|set_gopath.sh}
build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
if [ "${build_mongoreplay}" = "true" ]; then
build_tools="$build_tools mongoreplay"
fi
for i in $build_tools; do
${gorootvars} go build ${tooltags|} -o "../../mongo-tools/$i${exe|}" $i/main/$i.go
"../../mongo-tools/$i${exe|}" --version
done
- command: shell.exec
params:
working_dir: src
script: |
set -o errexit
set -o verbose
${python|/opt/mongodbtoolchain/v2/bin/python2} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} mongo${extension} --use-new-tools mongod${extension} mongos${extension} MONGO_VERSION=${version}
mkdir -p mongodb/bin
mkdir -p mongodb/jstests/hooks
mv mongo${extension|} mongodb/bin
mv mongod${extension|} mongodb/bin
mv mongos${extension|} mongodb/bin
mv src/mongo-tools/* mongodb/bin
if [ -d jstests/hooks ]
then
echo "Fetching JS test DB correctness checks from directory jstests"
cp -a jstests/* mongodb/jstests
fi
tar cvf mongodb.tar mongodb
gzip mongodb.tar
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: src/mongodb.tar.gz
remote_file: ${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz
bucket: mciuploads
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: mongodb.tar.gz
- name: industry_benchmarks_WT
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "ycsb"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
vars:
# Disabling --ycsb-throughput-analysis until BF-7153 fixed
# script_flags: --ycsb-throughput-analysis reports
- name: industry_benchmarks_MMAPv1
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "ycsb"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
vars:
script_flags: --ycsb-throughput-analysis reports
- name: industry_benchmarks_wmajority_WT
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "ycsb-wmajority"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
vars:
script_flags: --ycsb-throughput-analysis reports
- name: industry_benchmarks_wmajority_MMAPv1
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "ycsb-wmajority"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
vars:
script_flags: --ycsb-throughput-analysis reports
- name: core_workloads_WT
depends_on:
- name: compile
variant: linux-standalone
exec_timeout_secs: 36000 # 10 hours
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "core"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: core_workloads_MMAPv1
depends_on:
- name: compile
variant: linux-standalone
exec_timeout_secs: 36000 # 10 hours
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "core"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: agg_query_comparison_bestbuy_WT
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "agg_query_comparison_bestbuy"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: agg_query_comparison_bestbuy_MMAPv1
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "agg_query_comparison_bestbuy"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: non_sharded_workloads_WT
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "non_sharded"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: non_sharded_workloads_MMAPv1
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "non_sharded"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: mongos_workloads_WT
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "mongos"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: mongos_workloads_MMAPv1
depends_on:
- name: compile
variant: linux-standalone
exec_timeout_secs: 32400 # 9 hours
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "mongos"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: move_chunk_workloads_WT
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "move_chunk"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: move_chunk_workloads_MMAPv1
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "move_chunk"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: secondary_performance_WT
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "secondary_performance"
setup: "replica-2node"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: secondary_performance_MMAPv1
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "secondary_performance"
setup: "replica-2node"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: initialsync_WT
depends_on:
- name: compile
variant: linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "initialsync"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: initialsync_MMAPv1
depends_on:
- name: compile
variant : linux-standalone
commands:
- func: "prepare environment"
vars:
storageEngine: "mmapv1"
test: "initialsync"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
- name: initialsync-logkeeper_WT
depends_on:
- name: compile
variant: linux-standalone
exec_timeout_secs: 216000 # 2.5 days
commands:
- func: "prepare environment"
vars:
storageEngine: "wiredTiger"
test: "initialsync-logkeeper"
- func: "deploy cluster"
- func: "run test"
- func: "make test log artifact"
- func: "analyze"
#######################################
# Modules #
#######################################
# if a module is added and to be added to the manifest
# be sure to add the module to git.get_project revisions parameter
modules:
- name: dsi
repo: git@github.com:10gen/dsi.git
prefix: dsi
branch: master
- name: workloads
repo: git@github.com:10gen/workloads.git
prefix: workloads
branch: master
#######################################
# Linux Buildvariants #
#######################################
buildvariants:
- name: linux-1-node-replSet
display_name: Linux 1-Node ReplSet
batchtime: 1440 # 24 hours
modules: &modules
- dsi
- workloads
expansions:
setup: single-replica
cluster: single
platform: linux
project: &project dsi
run_on:
- "rhel70-perf-single"
tasks:
- name: industry_benchmarks_WT
- name: core_workloads_WT
- name: industry_benchmarks_MMAPv1
- name: core_workloads_MMAPv1
- name: non_sharded_workloads_WT
- name: non_sharded_workloads_MMAPv1
- name: agg_query_comparison_bestbuy_WT
- name: agg_query_comparison_bestbuy_MMAPv1
- name: linux-standalone
display_name: Linux Standalone
batchtime: 1440 # 24 hours
modules: *modules
expansions:
compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy
setup: standalone
cluster: single
platform: linux
use_scons_cache: true
gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
tooltags: ""
project: *project
run_on:
- "rhel70-perf-single"
tasks:
- name: compile
distros:
- rhel70
- name: industry_benchmarks_WT
- name: core_workloads_WT
- name: industry_benchmarks_MMAPv1
- name: core_workloads_MMAPv1
- name: non_sharded_workloads_WT
- name: non_sharded_workloads_MMAPv1
- name: agg_query_comparison_bestbuy_WT
- name: agg_query_comparison_bestbuy_MMAPv1
- name: linux-3-shard
display_name: Linux 3-Shard Cluster
batchtime: 10080 # 7 days
modules: *modules
expansions:
setup: shard
cluster: shard
platform: linux
project: *project
run_on:
- "rhel70-perf-shard"
tasks:
- name: industry_benchmarks_WT
- name: core_workloads_WT
- name: industry_benchmarks_MMAPv1
- name: industry_benchmarks_wmajority_WT
- name: industry_benchmarks_wmajority_MMAPv1
- name: core_workloads_MMAPv1
- name: mongos_workloads_WT
- name: mongos_workloads_MMAPv1
- name: move_chunk_workloads_WT
- name: move_chunk_workloads_MMAPv1
- name: agg_query_comparison_bestbuy_WT
- name: agg_query_comparison_bestbuy_MMAPv1
- name: linux-3-node-replSet
display_name: Linux 3-Node ReplSet
batchtime: 1440 # 24 hours
modules: *modules
expansions:
setup: replica
cluster: replica
platform: linux
project: *project
run_on:
- "rhel70-perf-replset"
tasks:
- name: industry_benchmarks_WT
- name: core_workloads_WT
- name: industry_benchmarks_MMAPv1
- name: industry_benchmarks_wmajority_WT
- name: industry_benchmarks_wmajority_MMAPv1
- name: core_workloads_MMAPv1
- name: secondary_performance_WT
- name: secondary_performance_MMAPv1
- name: non_sharded_workloads_WT
- name: non_sharded_workloads_MMAPv1
- name: agg_query_comparison_bestbuy_WT
- name: agg_query_comparison_bestbuy_MMAPv1
- name: linux-3-node-replSet-initialsync
display_name: Linux 3-Node ReplSet Initial Sync
batchtime: 1440 # 24 hours
modules: *modules
expansions:
setup: replica-2node
cluster: replica
platform: linux
project: *project
run_on:
- "rhel70-perf-replset"
tasks:
- name: initialsync_WT
- name: initialsync_MMAPv1
- name: linux-replSet-initialsync-logkeeper
display_name: Linux ReplSet Initial Sync LogKeeper
batchtime: 10080 # 7 days
modules: *modules
expansions:
setup: initialsync-logkeeper
cluster: initialsync-logkeeper
platform: linux
project: *project
run_on:
- "rhel70-perf-initialsync-logkeeper"
tasks:
- name: initialsync-logkeeper_WT