SERVER-106402: push mongostream image to streams ecr (#39895)

GitOrigin-RevId: 5cbd7bb9ee8bb351b31d8a75f924ecf088de7f7e

.github/CODEOWNERS

@@ -56,6 +56,7 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
 /buildscripts/resmoke_tests_runtime_validate.py @10gen/devprod-correctness @svc-auto-approve-bot
 /buildscripts/powercycle* @10gen/devprod-correctness @svc-auto-approve-bot
 /buildscripts/golden_test.py @10gen/query-optimization @svc-auto-approve-bot
+/buildscripts/evergreen_gen_streams* @10gen/streams-engine @svc-auto-approve-bot
 
 # The following patterns are parsed from ./buildscripts/antithesis/OWNERS.yml
 /buildscripts/antithesis/ @10gen/devprod-correctness @svc-auto-approve-bot
@@ -329,6 +330,7 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
 /evergreen/validate_compile_commands.py @10gen/devprod-build @svc-auto-approve-bot
 /evergreen/packager_crypt_py_run.sh @10gen/devprod-build @svc-auto-approve-bot
 /evergreen/perf-submission.sh @10gen/devprod-performance-analysis @svc-auto-approve-bot
+/evergreen/streams* @10gen/streams-engine @svc-auto-approve-bot
 
 # The following patterns are parsed from ./jstests/OWNERS.yml
 /jstests/README.md @10gen/devprod-correctness @svc-auto-approve-bot

.gitignore

@@ -96,6 +96,7 @@ scratch
 /mongoshim*
 /mongosniff*
 /mongotrafficreader*
+/bin
 
 # artifacts from db-contrib-tool
 /ksdecode*

buildscripts/OWNERS.yml

@@ -41,3 +41,6 @@ filters:
   - "golden_test.py":
     approvers:
       - 10gen/query-optimization
+  - "evergreen_gen_streams*":
+    approvers:
+      - 10gen/streams-engine

buildscripts/evergreen_gen_streams_build_and_publish_task.py

@@ -0,0 +1,97 @@
+import os.path
+import sys
+
+import typer
+from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, Task, TaskDependency
+from shrub.v2.command import BuiltInCommand
+from typing_extensions import Annotated
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from buildscripts.resmokelib.utils import evergreen_conn
+from buildscripts.util.fileops import write_file
+from buildscripts.util.read_config import read_config_file
+
+# This file is for generating the task that builds and publishes the streams docker image.
+# depends_on is only evaluated on task creation/validation, so all dependencies must exist prior to streams_build_and_publish.
+# Streams currently depends on multiple generated test suite tasks, which is why this task must also be generated.
+
+def make_task(compile_variant: str, additional_dependencies: set[str]) -> Task:
+    commands = [
+        BuiltInCommand("manifest.load", {}),
+        FunctionCall("git get project and add git tag"),
+        FunctionCall("f_expansions_write"),
+        FunctionCall("set up venv"),
+        FunctionCall("fetch binaries"),
+        FunctionCall("extract binaries"),
+        FunctionCall("set up remote credentials", {
+            "aws_key_remote": "${repo_aws_key}",
+            "aws_secret_remote": "${repo_aws_secret}"
+        }),
+        BuiltInCommand("ec2.assume_role", {"role_arn": "arn:aws:iam::664315256653:role/mongo-tf-project"}),
+        BuiltInCommand("subprocess.exec", {
+            "add_expansions_to_env": True,
+            "binary": "bash",
+            "args": ["./src/evergreen/streams_image_push.sh"]
+        }),
+    ]
+    dependencies = {
+        TaskDependency("archive_dist_test", compile_variant),
+        TaskDependency("aggregation", compile_variant),
+        TaskDependency(".streams_release_test"),
+    }
+    for dep in additional_dependencies:
+        dependencies.add(TaskDependency(dep))
+    return Task(f"streams_build_and_publish_{compile_variant}", commands, dependencies)
+
+def main(
+    expansions_file: Annotated[str, typer.Argument()] = "expansions.yml",
+    output_file: Annotated[str, typer.Option("--output-file")] = "streams_build_and_publish.json",
+):
+    evg_api = evergreen_conn.get_evergreen_api()
+    expansions = read_config_file(expansions_file)
+    version_id = expansions.get("version_id")
+    build_variant_name = expansions.get("build_variant")
+    required_tasks = {"streams", "streams_kafka"}
+    evg_version = evg_api.version_by_id(version_id)
+    variant = evg_version.build_by_variant(build_variant_name)
+    task_deps = []
+    for task in variant.get_tasks():
+        if task.display_name not in required_tasks:
+            continue
+        if task.execution_tasks:
+            # is a display task
+            for child_task_id in task.execution_tasks:
+                child_task = evg_api.task_by_id(child_task_id)
+                task_deps.append(child_task.display_name)
+        else:
+            # is not a display task
+            task_deps.append(task.display_name)
+        
+        required_tasks.remove(task.display_name)
+    
+    print(task_deps)
+    if required_tasks:
+        print("The following required tasks were not found", required_tasks)
+        raise RuntimeError("Could not find all required tasks")
+    
+    distro = expansions.get("distro_id")
+    compile_variant_name = expansions.get("compile_variant")
+    current_task_name = expansions.get("task_name", "streams_build_and_publish_gen")
+
+    build_variant = BuildVariant(name=build_variant_name)
+    build_variant.display_task(
+        current_task_name.replace("_gen", ""),
+        [make_task(compile_variant_name, additional_dependencies=task_deps)],
+        distros=[distro],
+    )
+    shrub_project = ShrubProject.empty()
+    shrub_project.add_build_variant(build_variant)
+
+    write_file(output_file, shrub_project.json())
+
+
+if __name__ == "__main__":
+    typer.run(main)

buildscripts/evergreen_gen_streams_publish_manifest_task.py

@@ -0,0 +1,69 @@
+import os.path
+import sys
+
+import typer
+from shrub.v2 import BuildVariant, FunctionCall, ShrubProject, Task, TaskDependency
+from shrub.v2.command import BuiltInCommand
+from typing_extensions import Annotated
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from buildscripts.util.fileops import write_file
+from buildscripts.util.read_config import read_config_file
+
+# This file is for generating the task creates a docker manifest for the distro images produced via streams_build_and_publish.
+# The docker manifest is used in order for the different architecture images to be pulled correctly without needing the particular architecture tag.
+
+def make_task(compile_variant: str) -> Task:
+    commands = [
+        BuiltInCommand("manifest.load", {}),
+        FunctionCall("git get project and add git tag"),
+        FunctionCall("f_expansions_write"),
+        FunctionCall("set up venv"),
+        FunctionCall("set up remote credentials", {
+            "aws_key_remote": "${repo_aws_key}",
+            "aws_secret_remote": "${repo_aws_secret}"
+        }),
+        BuiltInCommand("ec2.assume_role", {"role_arn": "arn:aws:iam::664315256653:role/mongo-tf-project"}),
+        BuiltInCommand("subprocess.exec", {
+            "add_expansions_to_env": True,
+            "binary": "bash",
+            "args": ["./src/evergreen/streams_docker_manifest.sh"]
+        }),
+    ]
+    dependencies = {
+        TaskDependency(f"streams_build_and_publish_{compile_variant.replace('-arm64', '')}"),
+        TaskDependency(f"streams_build_and_publish_{compile_variant}"),
+    }
+    return Task(f"streams_publish_manifest_{compile_variant}", commands, dependencies)
+
+
+def main(
+    expansions_file: Annotated[str, typer.Argument()] = "expansions.yml",
+    output_file: Annotated[str, typer.Option("--output-file")] = "streams_publish_manifest.json",
+):
+    expansions = read_config_file(expansions_file)
+    distro = expansions.get("distro_id")
+    build_variant_name = expansions.get("build_variant")
+    current_task_name = expansions.get("task_name", "streams_publish_manifest_gen")
+
+    compile_variant_name = expansions.get("compile_variant")
+    if (not compile_variant_name.endswith("-arm64")):
+        raise RuntimeError("This task should only run on the arm64 compile variant")
+
+    build_variant = BuildVariant(name=build_variant_name)
+    build_variant.display_task(
+        current_task_name.replace("_gen", ""),
+        [make_task(compile_variant_name)],
+        distros=[distro],
+    )
+    shrub_project = ShrubProject.empty()
+    shrub_project.add_build_variant(build_variant)
+
+    write_file(output_file, shrub_project.json())
+
+
+if __name__ == "__main__":
+    typer.run(main)

etc/evergreen_yml_components/definitions.yml

@@ -1036,6 +1036,78 @@ functions:
         files:
           - powercycle_tasks.json
 
+  "streams build and publish":
+    - command: manifest.load
+    - *git_get_shallow_project
+    - *f_expansions_write
+    - *restore_git_history_and_tags
+    - *add_git_tag
+    - *kill_processes
+    - *cleanup_environment
+    - *set_up_venv
+    - *upload_pip_requirements
+    - *configure_evergreen_api_credentials
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "src/evergreen/run_python_script.sh"
+          - "buildscripts/evergreen_gen_streams_build_and_publish_task.py"
+          - "../expansions.yml"
+          - "--output-file=../streams_build_and_publish.json"
+    - command: s3.put
+      display_name: "upload streams_build_and_publish.json"
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: streams_build_and_publish.json
+        remote_file: ${project}/${build_variant}/${revision}/streams_build_and_publish/${task_name}-${build_id}.json
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/gzip
+        display_name: Generated Task Config - Execution ${execution}
+    - command: generate.tasks
+      display_name: "generate.tasks streams_build_and_publish.json"
+      params:
+        files:
+          - streams_build_and_publish.json
+
+  "streams publish manifest":
+    - command: manifest.load
+    - *git_get_shallow_project
+    - *f_expansions_write
+    - *restore_git_history_and_tags
+    - *add_git_tag
+    - *kill_processes
+    - *cleanup_environment
+    - *set_up_venv
+    - *upload_pip_requirements
+    - *configure_evergreen_api_credentials
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "src/evergreen/run_python_script.sh"
+          - "buildscripts/evergreen_gen_streams_publish_manifest_task.py"
+          - "../expansions.yml"
+          - "--output-file=../streams_publish_manifest.json"
+    - command: s3.put
+      display_name: "upload streams_publish_manifest.json"
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: streams_publish_manifest.json
+        remote_file: ${project}/${build_variant}/${revision}/streams_publish_manifest/${task_name}-${build_id}.json
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/gzip
+        display_name: Generated Task Config - Execution ${execution}
+    - command: generate.tasks
+      display_name: "generate.tasks streams_publish_manifest.json"
+      params:
+        files:
+          - streams_publish_manifest.json
+
   "generate smoke test tasks":
     - command: manifest.load
     - *git_get_shallow_project

etc/evergreen_yml_components/tasks/compile_tasks_nightly.yml

@@ -739,3 +739,33 @@ tasks:
           filenames:
             - "src/mongodb-${push_name}-${push_arch}*"
             - "src/mongodb-cryptd-${push_name}-${push_arch}*"
+
+  - name: streams_publish_manifest_gen
+    tags: ["assigned_to_jira_team_streams", "auxiliary"]
+    depends_on:
+      - name: version_gen
+        variant: generate-tasks-for-version
+        # This is added because of EVG-18211.
+        # Without this we are adding extra dependencies on evergreen and it is causing strain
+        omit_generated_tasks: true
+      - name: streams_build_and_publish_gen
+        variant: enterprise-amazon2-streams
+      - name: streams_build_and_publish_gen
+        variant: enterprise-amazon2-streams-arm64
+    commands:
+      - func: "streams publish manifest"
+
+  - name: streams_publish_manifest_al2023_gen
+    tags: ["assigned_to_jira_team_streams", "auxiliary"]
+    depends_on:
+      - name: version_gen
+        variant: generate-tasks-for-version
+        # This is added because of EVG-18211.
+        # Without this we are adding extra dependencies on evergreen and it is causing strain
+        omit_generated_tasks: true
+      - name: streams_build_and_publish_gen
+        variant: enterprise-amazon2023-streams
+      - name: streams_build_and_publish_gen
+        variant: enterprise-amazon2023-streams-arm64
+    commands:
+      - func: "streams publish manifest"

etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml

@@ -288,7 +288,7 @@ tasks:
 
   - <<: *gen_task_template
     name: streams_gen
-    tags: ["assigned_to_jira_team_streams", "default"]
+    tags: ["assigned_to_jira_team_streams", "default", "streams_release_test"]
     commands:
       - func: "generate resmoke tasks"
         vars:
@@ -296,7 +296,8 @@ tasks:
 
   - <<: *gen_task_template
     name: streams_kafka_gen
-    tags: ["assigned_to_jira_team_streams", "experimental"]
+    tags:
+      ["assigned_to_jira_team_streams", "experimental", "streams_release_test"]
     commands:
       - func: "generate resmoke tasks"
         vars:
@@ -305,7 +306,8 @@ tasks:
 
   - <<: *task_template
     name: streams_kafka_gwproxy
-    tags: ["assigned_to_jira_team_streams", "experimental"]
+    tags:
+      ["assigned_to_jira_team_streams", "experimental", "streams_release_test"]
     commands:
       - command: ec2.assume_role
         params:
@@ -315,32 +317,49 @@ tasks:
 
   - <<: *task_template
     name: streams_kafka_benchmark
-    tags: ["assigned_to_jira_team_streams", "experimental"]
+    tags:
+      ["assigned_to_jira_team_streams", "experimental", "streams_release_test"]
     commands:
       - func: "do setup"
       - func: "run tests"
 
   - <<: *task_template
     name: streams_https
-    tags: ["assigned_to_jira_team_streams", "default"]
+    tags: ["assigned_to_jira_team_streams", "default", "streams_release_test"]
     commands:
       - func: "do setup"
       - func: "run tests"
 
   - <<: *task_template
     name: streams_lambda
-    tags: ["assigned_to_jira_team_streams", "experimental"]
+    tags:
+      ["assigned_to_jira_team_streams", "experimental", "streams_release_test"]
     commands:
       - func: "do setup"
       - func: "run tests"
 
   - <<: *task_template
     name: streams_s3
-    tags: ["assigned_to_jira_team_streams", "experimental"]
+    tags:
+      ["assigned_to_jira_team_streams", "experimental", "streams_release_test"]
     commands:
       - func: "do setup"
       - func: "run tests"
 
+  - name: streams_build_and_publish_gen
+    tags: ["assigned_to_jira_team_streams", "auxiliary"]
+    depends_on:
+      - name: version_gen
+        variant: generate-tasks-for-version
+        # This is added because of EVG-18211.
+        # Without this we are adding extra dependencies on evergreen and it is causing strain
+        omit_generated_tasks: true
+      # Update with all gen tasks to depend on here and the display name in evergreen_gen_streams_build_and_publish_task.py
+      - name: streams_gen
+      - name: streams_kafka_gen
+    commands:
+      - func: "streams build and publish"
+
   # Experimental task running the jscore suite bazel target. To be removed with SERVER-103537.
   - name: bazel_jscore
     tags: ["assigned_to_jira_team_devprod_correctness", "experimental"]

etc/evergreen_yml_components/variants/amazon/test_release.yml

@@ -521,6 +521,61 @@ buildvariants:
       - name: streams_https
       - name: streams_lambda
       - name: streams_s3
+      - name: streams_build_and_publish_gen
+
+  - name: enterprise-amazon2023-streams-arm64
+    display_name: "Amazon Linux 2023 enterprise build with streams arm64"
+    cron: "0 3 * * *" # From the ${project_nightly_cron} parameter.
+    run_on:
+      - amazon2023-arm64-latest-small
+    expansions:
+      test_flags: >-
+        --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,requires_external_data_source,requires_ldap_pool
+        --additionalFeatureFlags=featureFlagStreams
+      push_path: linux
+      push_bucket: downloads.10gen.com
+      push_bucket_new: cdn-origin-mongodb-server-enterprise
+      push_role_arn: arn:aws:iam::119629040606:role/s3-access.cdn-origin-mongodb-server-enterprise
+      push_name: linux
+      push_arch: aarch64-enterprise-amazon2023-streams
+      bazel_compile_flags: >-
+        --define=MONGO_DISTMOD=amazon2023
+        --streams_release_build=True
+      multiversion_platform: amazon2023
+      multiversion_edition: enterprise-streams
+      has_packages: true
+      packager_script: packager_enterprise.py
+      packager_arch: aarch64
+      packager_distro: amazon2023
+      repo_edition: enterprise
+      compile_variant: enterprise-amazon2023-streams-arm64
+      large_distro_name: amazon2023-arm64-latest-large
+    tasks:
+      - name: compile_test_and_package_serial_TG
+        distros:
+          - amazon2023-arm64-latest-large
+      - name: run_unit_tests_TG
+        distros:
+          - amazon2023-arm64-latest-large
+      - name: .development_critical !.requires_large_host
+      - name: .development_critical .requires_large_host
+        distros:
+          - amazon2023-arm64-latest-large
+      - name: .release_critical !.requires_large_host !crypt_push
+      - name: .release_critical .requires_large_host !crypt_push
+        distros:
+          - amazon2023-arm64-latest-large
+      - name: streams_gen
+      - name: streams_kafka_gen
+      # TODO(SERVER-103985) - Enabling this is blocked on SRE support arm64 in the sre/gwproxy image (SRE-1481)
+      # - name: streams_kafka_gwproxy
+      - name: streams_kafka_benchmark
+      - name: streams_https
+      - name: streams_lambda
+      - name: streams_s3
+      - name: streams_build_and_publish_gen
+      # Only needed once to generate the streams manifest for both x86 and arm
+      - name: streams_publish_manifest_al2023_gen
 
   - name: enterprise-amazon2-streams
     display_name: "Amazon Linux 2 enterprise build with streams"
@@ -571,6 +626,7 @@ buildvariants:
       - name: streams_https
       - name: streams_lambda
       - name: streams_s3
+      - name: streams_build_and_publish_gen
 
   - name: enterprise-amazon2-streams-arm64
     display_name: "Amazon Linux 2 enterprise build with streams arm64"
@@ -622,6 +678,9 @@ buildvariants:
       - name: streams_https
       - name: streams_lambda
       - name: streams_s3
+      - name: streams_build_and_publish_gen
+      # Only needed once to generate the streams manifest for both x86 and arm
+      - name: streams_publish_manifest_gen
 
   - &enterprise-amazon2023-arm64-fuzzers-template
     <<: *enterprise-amazon2023-arm64-template

evergreen/OWNERS.yml

@@ -39,3 +39,6 @@ filters:
   - "perf-submission.sh":
     approvers:
       - 10gen/devprod-performance-analysis
+  - "streams*":
+    approvers:
+      - 10gen/streams-engine

evergreen/package.sh

@@ -17,6 +17,7 @@ valid_mongocryptd_variants=(
     "enterprise-amazon2023-arm64"
     "enterprise-amazon2023-arm64-grav4"
     "enterprise-amazon2023-streams"
+    "enterprise-amazon2023-streams-arm64"
     "enterprise-debian12-64"
     "enterprise-linux-64-amazon-ami"
     "enterprise-macos"

evergreen/streams_docker_manifest.sh

@@ -0,0 +1,41 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+set -o errexit
+
+REGISTRY="664315256653.dkr.ecr.us-east-1.amazonaws.com"
+REPO="mongo/mongostream"
+IMAGE="$REGISTRY/$REPO"
+GITSHA="$github_commit"
+DISTRO="$packager_distro"
+
+attempts=0
+max_attempts=4
+
+while ! aws ecr get-login-password --region us-east-1 | docker login --password-stdin --username AWS $REGISTRY; do
+    [ "$attempts" -ge "$max_attempts" ] && exit 1
+    ((attempts++))
+    sleep 10
+done
+
+DISTRO_SUFFIX=""
+
+if [ "$DISTRO" == "amazon2023" ]; then
+    DISTRO_SUFFIX="-al2023"
+fi
+
+# Creating the manifest.
+docker manifest create $IMAGE:$GITSHA$DISTRO_SUFFIX \
+    $IMAGE:$GITSHA-arm64$DISTRO_SUFFIX \
+    $IMAGE:$GITSHA-amd64$DISTRO_SUFFIX
+
+# Annotating arm64.
+docker manifest annotate $IMAGE:$GITSHA$DISTRO_SUFFIX \
+    $IMAGE:$GITSHA-arm64$DISTRO_SUFFIX --os linux --arch arm64
+
+# Annotating amd64.
+docker manifest annotate $IMAGE:$GITSHA$DISTRO_SUFFIX \
+    $IMAGE:$GITSHA-amd64$DISTRO_SUFFIX --os linux --arch amd64
+
+# Pushing the manifest.
+docker manifest push $IMAGE:$GITSHA$DISTRO_SUFFIX

evergreen/streams_image_push.sh

@@ -0,0 +1,66 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+set -o errexit
+
+REGISTRY="664315256653.dkr.ecr.us-east-1.amazonaws.com"
+REPO="mongo/mongostream"
+IMAGE="$REGISTRY/$REPO"
+GITSHA="$github_commit"
+ARCH="$packager_arch"
+DISTRO="$packager_distro"
+PATCH="$is_patch"
+
+if [ "$ARCH" == "aarch64" ]; then
+    ARCH="arm64"
+fi
+
+if [ "$ARCH" == "x86_64" ]; then
+    ARCH="amd64"
+fi
+
+TAG_SUFFIX="$ARCH"
+
+# Only these 2 distros are supported today
+if [ "$DISTRO" != "amazon2" ] && [ "$DISTRO" != "amazon2023" ]; then
+    echo "Unsupported distro: $DISTRO" >&2
+    exit 1
+fi
+
+if [ "$DISTRO" == "amazon2023" ]; then
+    TAG_SUFFIX="$ARCH-al2023"
+fi
+
+attempts=0
+max_attempts=4
+
+while ! aws ecr get-login-password --region us-east-1 | docker login --password-stdin --username AWS $REGISTRY; do
+    [ "$attempts" -ge "$max_attempts" ] && exit 1
+    ((attempts++))
+    sleep 10
+done
+
+# Build Image
+MONGOD_PATH="$(find ./src -type f -name 'mongod')"
+MONGO_PATH="$(find ./src -type f -name 'mongo')"
+echo "Current mongod path: $MONGOD_PATH"
+echo "Current mongo path: $MONGO_PATH"
+
+mkdir -p ./src/bin
+mv "$MONGOD_PATH" ./src/bin/mongod
+mv "$MONGO_PATH" ./src/bin/mongo
+
+cd src
+activate_venv
+setup_db_contrib_tool
+
+if [ "$DISTRO" == "amazon2" ]; then
+    docker build --build-arg BUILD_VERSION=$GITSHA-$TAG_SUFFIX -t "$IMAGE" -f ./src/mongo/db/modules/enterprise/src/streams/build/Dockerfile .
+else
+    docker build --build-arg BUILD_VERSION=$GITSHA-$TAG_SUFFIX -t "$IMAGE" -f ./src/mongo/db/modules/enterprise/src/streams/build/Dockerfile.al2023 .
+fi
+docker tag "$IMAGE" "$IMAGE:$GITSHA-$TAG_SUFFIX"
+
+docker images
+
+docker push "$IMAGE:$GITSHA-$TAG_SUFFIX"