mongo/etc/system_perf.yml

command_type: system
stepback: false
exec_timeout_secs: &exec_timeout_secs 21600
timeout_secs: &timeout_secs 7200

include:
  - filename: etc/evergreen_yml_components/definitions.yml
  - filename: etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml
  - filename: etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml
  - filename: etc/evergreen_yml_components/tasks/compile_tasks.yml
  - filename: etc/evergreen_yml_components/tasks/misc_tasks.yml
  - filename: etc/evergreen_yml_components/variants/misc/task_generation.yml
  - filename: etc/system_perf_yml_components/tasks.yml
  - filename: etc/system_perf_yml_components/variants.yml
  - filename: etc/system_perf_yml_components/variants_branching.yml

## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds)
parameters:
  - key: patch_compile_flags
    description: "Additional SCons flags to be applied during scons compile invocations in this patch"
  # see https://github.com/10gen/performance-tooling-docs/blob/main/patch_testing.md#skipping-compilation-on-sys-perf-projects
  - key: reuse_compile_from
    description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_ef8504eefef4c9fe2062b8424e85053b3e1ad8d2"
  - key: exec_timeout_secs_override
    value: *exec_timeout_secs
    description: "Overrides the default exec_timeout_secs defined at the top of this file for the f_dsi_run_workload task"
  - key: timeout_secs_override
    value: *timeout_secs
    description: "Overrides the default timeout_secs defined at the top of this file for the f_dsi_run_workload task"

variables:
  _src_dir: &src_dir src/mongo
  _modules: &modules
    - dsi
    - genny
    - workloads
    - linkbench
    - linkbench2
    - tsbs
    - mongo-perf
    - YCSB
    - py-tpcc
    - PrivateWorkloads
    - flamegraph

  _schedule_variant_auto_tasks_task: &schedule_variant_auto_tasks_task
    name: schedule_variant_auto_tasks
    activate: false
    depends_on:
      - name: schedule_global_auto_tasks
        variant: task_generation

  _schedule_patch_auto_tasks_task: &schedule_patch_auto_tasks_task
    name: schedule_patch_auto_tasks
    activate: false
    depends_on:
      - name: schedule_global_auto_tasks
        variant: task_generation

  _shared_compile_expansions: &shared_compile_expansions
    platform: linux
    project_dir: dsi
    scons_cache_scope: shared
    scons_cache_mode: all
    has_packages: false

###
# Same in every DSI project. Ensure that this block is synchronized with
# evergreen-dsitest.yml, and src/dsi/onboarding.py (search update-repos-here)
# in the dsi repo, and etc/system_perf.yml in mongodb/mongo.
modules:
  - name: dsi
    owner: 10gen
    repo: dsi
    prefix: ${workdir}/src
    branch: master
  - name: genny
    owner: mongodb
    repo: genny
    prefix: ${workdir}/src
    branch: master
  - name: workloads
    owner: 10gen
    repo: workloads
    prefix: ${workdir}/src
    branch: master
  - name: linkbench
    owner: 10gen
    repo: linkbench
    prefix: ${workdir}/src
    branch: master
  - name: linkbench2
    owner: 10gen
    repo: linkbench2
    prefix: ${workdir}/src
    branch: master
  - name: tsbs
    owner: mongodb-forks
    repo: tsbs
    prefix: ${workdir}/src
    branch: main
  - name: mongo-perf
    owner: mongodb
    repo: mongo-perf
    prefix: ${workdir}/src
    branch: master
  - name: YCSB
    owner: mongodb-labs
    repo: YCSB
    prefix: ${workdir}/src
    branch: production
  - name: py-tpcc
    owner: mongodb-labs
    repo: py-tpcc
    prefix: ${workdir}/src
    branch: production
  - name: flamegraph
    owner: mongodb-forks
    repo: flamegraph
    prefix: ${workdir}/src
    branch: master
  - name: PrivateWorkloads
    owner: 10gen
    repo: PrivateWorkloads
    prefix: ${workdir}/src
    branch: production
###

###
# Same in every DSI project
pre:
  - func: "f_other_pre_ops"
  - func: "f_dsi_pre_run"
  - func: "get version expansions"
  - func: "apply version expansions"
post:
  - func: "f_dsi_post_run"
  - func: "f_other_post_ops"
timeout:
  - func: "f_dsi_timeout"
  - func: "f_other_timeout"
###

functions:
  ###
  # Same in every DSI project
  f_dsi_pre_run:
    - command: manifest.load
  f_dsi_post_run:
    - command: shell.exec
      params:
        script: ./src/dsi/run-dsi post_run
    - command: perf.send
      params:
        file: ./build/CedarReports/cedar_report.json
        aws_key: ${terraform_key}
        aws_secret: ${terraform_secret}
        bucket: genny-metrics
        region: us-east-1
        prefix: ${task_id}_${execution}
    - command: attach.results
      params:
        file_location: ./build/EvergreenResultsJson/results.json
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: ./build/Artifacts/DSIArtifacts.tgz
        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/x-gzip
        display_name: DSI Artifacts - Execution ${execution}
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: ./build/Documentation/index.html
        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-index.html
        bucket: mciuploads
        permissions: public-read
        content_type: text/html
        display_name: Documentation
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: bootstrap.yml
        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/bootstrap-${task_name}-${build_id}-${execution}.yml
        bucket: mciuploads
        permissions: public-read
        content_type: text/plain
        display_name: Task Bootstrap Config
  f_dsi_timeout:
    - command: shell.exec
      params:
        script: ./src/dsi/run-dsi on_timeout
  ###

  f_other_post_ops:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          # removes files from the (local) scons cache when it's over a
          # threshold, to the $prune_ratio percentage. Ideally override
          # these default values in the distro config in evergreen.

          if [ -d "${scons_cache_path}" ]; then
              /opt/mongodbtoolchain/v4/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
          fi
  f_other_pre_ops:
    - &f_other_pre_ops
      command: shell.exec
      params:
        silent: true
        script: |
          for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do
              pkill -9 "$PS"
          done
  f_other_timeout:
    # Can't be empty so just `echo`.
    - command: shell.exec
      params: {script: "echo"}

  f_generate_all_variant_auto_tasks:
    - command: git.get_project
      params:
        directory: *src_dir
        revisions:
          dsi: ${dsi_rev}
          genny: ${genny_rev}
          linkbench: ${linkbench_rev}
          linkbench2: ${linkbench2_rev}
          tsbs: ${tsbs_rev}
          workloads: ${workloads_rev}
          mongo-perf: ${mongo-perf_rev}
          YCSB: ${YCSB_rev}
          py-tpcc: ${py-tpcc_rev}
          PrivateWorkloads: ${PrivateWorkloads_rev}
    - command: expansions.write
      params:
        file: ./expansions.yml
    - command: shell.exec
      params:
        # The script below makes sure that if we generate all tasks in a patch, we will only generate
        # them and not run all of them, since activating all tasks is the default behavior for task
        # generation in a patch.
        script: |
          pushd ./src/mongo
          ~/evergreen evaluate ./etc/system_perf.yml > evaluated_project_configuration.yml
          popd
          if [ "${is_patch|false}" = "true" ]; then
            ./src/genny/run-genny auto-tasks-all --project-file ./src/mongo/evaluated_project_configuration.yml --no-activate
          else
            ./src/genny/run-genny auto-tasks-all --project-file ./src/mongo/evaluated_project_configuration.yml
          fi
    - command: generate.tasks
      params:
        files:
          - build/TaskJSON/Tasks.json

  ###
  # Package the supplementary DSI data (mongo tools, feature flag list,
  # server params list, jstests, and buildscripts)
  download mongo tools:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o verbose
          set -o errexit

          arch=$(uname -m)
          case $arch in
            "x86_64")
              if [ -f /etc/os-release ]; then
                . /etc/os-release
                if [ "$ID" == "amzn" ]; then
                  case $VERSION_ID in
                    "2")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2-x86_64-100.9.4.tgz"
                      ;;
                    "2023")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2023-x86_64-100.9.4.tgz"
                      ;;
                    *)
                      echo "Unsupported Amazon Linux version: $VERSION_ID"
                      exit 1
                      ;;
                  esac
                else
                  echo "Unsupported Linux distribution: $ID"
                  exit 1
                fi
              else
                echo "Unable to determine Linux distribution"
                exit 1
              fi
              ;;
            "aarch64")
              if [ -f /etc/os-release ]; then
                . /etc/os-release
                if [ "$ID" == "amzn" ]; then
                  case $VERSION_ID in
                    "2")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2-aarch64-100.9.4.tgz"
                      ;;
                    "2023")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2023-aarch64-100.9.4.tgz"
                      ;;
                    *)
                      echo "Unsupported Amazon Linux version: $VERSION_ID"
                      exit 1
                      ;;
                  esac
                else
                  echo "Unsupported Linux distribution: $ID"
                  exit 1
                fi
              else
                echo "Unable to determine Linux distribution"
                exit 1
              fi
              ;;
            *)
              echo "Unsupported architecture: $arch"
              exit 1
              ;;
          esac

          echo "Downloading and installing MongoDB tools..."
          mkdir -p mongodb/
          wget "$binary_url" -O mongo-tools.tar.gz
          tar -xzvf mongo-tools.tar.gz -C mongodb/ --strip-components=1 "mong*/bin"
          echo "MongoDB tools installed successfully!"
  generate feature flag list:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          pushd ..
          . src/evergreen/prelude.sh
          activate_venv
          popd

          python ./buildscripts/idl/gen_all_feature_flag_list.py
          mkdir -p mongodb/feature_flags
          cp ./all_feature_flags.txt mongodb/feature_flags
  generate server params list:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          pushd ..
          . src/evergreen/prelude.sh
          activate_venv
          popd

          python ./buildscripts/idl/gen_all_server_params_list.py
          mkdir -p mongodb/server_params
          cp ./all_server_params.txt mongodb/server_params
  copy jstests:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          mkdir -p mongodb/jstests/hooks
          if [ -d jstests/hooks ]
          then
            echo "Fetching JS test DB correctness checks from directory jstests"
            cp -a jstests/* mongodb/jstests

            echo "Now adding our own special run_validate_collections.js wrapper"
            mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js

            cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
            print("NOTE: run_validate_collections.js will skip the oplog!");
            TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
            await import("jstests/hooks/run_validate_collections.actual.js");
          EOF
          fi
  copy buildscripts:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          mkdir -p mongodb/buildscripts
          cp -a buildscripts/* mongodb/buildscripts
  package supplementary data:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${build_patch_id}" ] || [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            tar czf supplementary-data.tgz mongodb
          else
            # Evergreen does not handle nested escaped expansions well
            reuse_compile_from="${reuse_compile_from}"
            build_patch_id="${build_patch_id}"
            version_to_reuse_from=$(if [ -n "${build_patch_id}" ]; then echo "${build_patch_id}"; else echo "${reuse_compile_from}"; fi)

            curl -o supplementary-data.tgz https://s3.amazonaws.com/mciuploads/${project}/${compile_variant}/$${version_to_reuse_from}/dsi/supplementary-data.tgz
          fi
  upload supplementary data:
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/supplementary-data.tgz
        remote_file: ${project}/${compile_variant}/${version_id}/dsi/supplementary-data.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/x-gzip}
        display_name: supplementary-data.tgz

tasks:
  - name: package_supplementary_data
    commands:
      - command: manifest.load
      - command: git.get_project
        params:
          directory: src
      - func: "set up venv"
      - func: "download mongo tools"
      - func: "generate feature flag list"
      - func: "generate server params list"
      - func: "copy jstests"
      - func: "copy buildscripts"
      - func: "package supplementary data"
      - func: "upload supplementary data"

buildvariants:
  - name: task_generation
    display_name: " * Task Generation"
    cron: "0 0 1 1 *" # Every year starting 1/1 at 00:00
    modules: *modules
    run_on:
      - amazon2-build
    tasks:
      - name: generate_all_variant_auto_tasks
        activate: true
      - name: schedule_global_auto_tasks

  - name: amazon2-x86-compile
    display_name: "* Compile for Amazon Linux 2 x86"
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-x86-compile
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
    run_on:
      - amazon2-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_supplementary_data

  - name: amazon2-arm64-compile
    display_name: "* Compile for Amazon Linux 2 arm64"
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-arm64-compile
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_supplementary_data

  - name: amazon2-arm64-mongocrypt-shlib-compile
    display_name: "* Compile mongo_crypt_v1.so for Amazon Linux 2 arm64"
    modules: *modules
    cron: "0 0 * * 4" # 00:00 on Thursday
    tags: ["bazel_check"]
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-arm64-compile
      use_scons_cache: true
      compile_flags: >-
        --separate-debug
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
        --use-diagnostic-latches=off
        --allocator=system
        --enterprise-features=fle
        --js-engine=none
        --link-model=dynamic-sdk
        --enable-http-client=off
        --ssl=off
        SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique"
        CCFLAGS="-fno-gnu-unique"
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: .crypt

  - name: amazon2-x86-streams-compile
    display_name: "* Compile Streams for Amazon Linux 2 x86"
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-x86-streams-compile
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
        --streams-release-build
    run_on:
      - amazon2-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_supplementary_data

  # This variant tests the reuse_compile_from feature
  # by reusing compile artifacts from another compile
  # in the same patch.
  - name: test-reuse-compile-from
    display_name: "~ Test reuse_compile_from"
    cron: "0 0 * * *" # Every day at 00:00
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-arm64-compile
      reuse_compile_from: ${version_id}
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
        depends_on:
          - name: archive_dist_test
            variant: amazon2-arm64-compile
          - name: archive_dist_test_debug
            variant: amazon2-arm64-compile