SERVER-46983: upload to correct S3 URL for repobuilder

(cherry picked from commit 2261279b51)

buildscripts/packager.py

@@ -242,9 +242,7 @@ class Distro(object):
             else:
                 raise Exception("unsupported build_os: %s" % build_os)
         elif self.n == 'debian':
-            if build_os == 'debian71':
-                return 'wheezy'
-            elif build_os == 'debian81':
+            if build_os == 'debian81':
                 return 'jessie'
             else:
                 raise Exception("unsupported build_os: %s" % build_os)
@@ -261,7 +259,7 @@ class Distro(object):
 
     def build_os(self, arch):
         """Return the build os label in the binary package to download ("rhel55", "rhel62" and "rhel70"
-        for redhat, "ubuntu1204" and "ubuntu1404" for Ubuntu, "debian71" for Debian), and "suse11"
+        for redhat, "ubuntu1204" and "ubuntu1404" for Ubuntu, "debian81" for Debian), and "suse11"
         for SUSE)"""
         # Community builds only support amd64
         if arch not in ['x86_64', 'ppc64le']:
@@ -276,7 +274,7 @@ class Distro(object):
         elif self.n == 'ubuntu':
             return [ "ubuntu1204", "ubuntu1404", "ubuntu1604", ]
         elif self.n == 'debian':
-            return [ "debian71", "debian81" ]
+            return [ "debian81" ]
         else:
             raise Exception("BUG: unsupported platform?")
 
@@ -457,12 +455,8 @@ def make_deb(distro, build_os, arch, spec, srcdir):
     sdir=setupdir(distro, build_os, arch, spec)
     if re.search("debian", distro.name()):
         os.unlink(sdir+"debian/mongod.upstart")
-        if build_os == "debian71":
-            os.link(sdir+"debian/init.d", sdir+"debian/%s%s-server.mongod.init" % (distro.pkgbase(), suffix))
-            os.unlink(sdir+"debian/mongod.service")
-        else:
-            os.link(sdir+"debian/mongod.service", sdir+"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix))
-            os.unlink(sdir+"debian/init.d")
+        os.link(sdir+"debian/mongod.service", sdir+"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix))
+        os.unlink(sdir+"debian/init.d")
     elif re.search("ubuntu", distro.name()):
         os.unlink(sdir+"debian/init.d")
         if build_os in ("ubuntu1204", "ubuntu1404", "ubuntu1410"):

buildscripts/resmokelib/testing/fixtures/standalone.py

@@ -6,7 +6,6 @@ from __future__ import absolute_import
 
 import os
 import os.path
-import shutil
 import socket
 import time
 
@@ -93,8 +92,9 @@ class MongoDFixture(interface.Fixture):
             raise errors.TestFailure("%s did not exit cleanly" % (self))
 
     def setup(self):
-        if not self.preserve_dbpath:
-            shutil.rmtree(self._dbpath, ignore_errors=True)
+        """Set up the mongod."""
+        if not self.preserve_dbpath and os.path.lexists(self._dbpath):
+            utils.rmtree(self._dbpath, ignore_errors=False)
 
         try:
             os.makedirs(self._dbpath)

buildscripts/resmokelib/testing/testcases.py

@@ -6,7 +6,6 @@ from __future__ import absolute_import
 
 import os
 import os.path
-import shutil
 import unittest
 
 from .. import config
@@ -222,7 +221,7 @@ class DBTestCase(TestCase):
         dbpath = os.path.join(dbpath_prefix, "job%d" % (self.fixture.job_num), "unittest")
         self.dbtest_options["dbpath"] = dbpath
 
-        shutil.rmtree(dbpath, ignore_errors=True)
+        utils.rmtree(dbpath, ignore_errors=True)
 
         try:
             os.makedirs(dbpath)
@@ -318,7 +317,7 @@ class JSTestCase(TestCase):
         global_vars["TestData"] = test_data
         self.shell_options["global_vars"] = global_vars
 
-        shutil.rmtree(data_dir, ignore_errors=True)
+        utils.rmtree(data_dir, ignore_errors=True)
 
         try:
             os.makedirs(data_dir)

buildscripts/resmokelib/utils/__init__.py

@@ -5,6 +5,8 @@ Helper functions.
 from __future__ import absolute_import
 
 import os.path
+import shutil
+import sys
 
 import pymongo
 import yaml
@@ -14,6 +16,28 @@ def default_if_none(value, default):
     return value if value is not None else default
 
 
+def rmtree(path, **kwargs):
+    """Wrap shutil.rmtreee.
+
+    Use a UTF-8 unicode path if Windows.
+    See https://bugs.python.org/issue24672, where shutil.rmtree can fail with UTF-8.
+    Use a bytes path to rmtree, otherwise.
+    See https://github.com/pypa/setuptools/issues/706.
+    """
+    if is_windows():
+        if not isinstance(path, unicode):
+            path = unicode(path, "utf-8")
+    else:
+        if isinstance(path, unicode):
+            path = path.encode("utf-8")
+    shutil.rmtree(path, **kwargs)
+
+
+def is_windows():
+    """Return True if Windows."""
+    return sys.platform.startswith("win32") or sys.platform.startswith("cygwin")
+
+
 def is_string_list(lst):
     """
     Returns true if 'lst' is a list of strings, and false otherwise.

buildscripts/tests/resmokelib/utils/__init__.py

@@ -0,0 +1 @@
+"""Empty."""

buildscripts/tests/resmokelib/utils/test_rmtree.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+""" Unit tests for utils.rmtree. """
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+from buildscripts.resmokelib import utils
+
+# pylint: disable=missing-docstring,protected-access
+
+
+def rmtree(dir_root):
+    """Invoke utils.rmtree(dir_root) and return True if removed."""
+    utils.rmtree(dir_root)
+    return not os.path.exists(dir_root)
+
+
+def create_file(path):
+    """Create file named 'path'."""
+    with open(path, "w") as fh:
+        fh.write("")
+
+
+def ascii_filesystemencoding():
+    """Return True if the file system encoding is type ASCII.
+
+    See https://www.iana.org/assignments/character-sets/character-sets.xhtml.
+    """
+    encoding = sys.getfilesystemencoding()
+    return encoding.startswith("ANSI_X3.4") or encoding == "US-ASCII"
+
+
+def string_for_ascii_filesystem_encoding(path):
+    """Return encoded string type for unicode on ASCII file system encoding.
+
+    Some file system encodings are set to ASCII if LANG=C or LC_ALL=C is specified.
+    """
+    if ascii_filesystemencoding() and isinstance(path, unicode):
+        return path.encode("utf-8")
+    return path
+
+
+class RmtreeTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.temp_dir_root = tempfile.mkdtemp()
+        cls.cwd = os.getcwd()
+
+    @classmethod
+    def tearDownClass(cls):
+        os.chdir(cls.cwd)
+        shutil.rmtree(cls.temp_dir_root, ignore_errors=True)
+
+    def do_test(self, name):
+        pass
+
+    def test_ascii(self):
+        # ASCII name
+        self.do_test("ascii")
+
+    def test_unicode(self):
+        # Unicode name
+        self.do_test(u"unicode")
+
+    def test_greek(self):
+        # Name with Greek
+        self.do_test(string_for_ascii_filesystem_encoding(u"ελληνικά"))
+
+    def test_japanese(self):
+        # Name with Japanese
+        self.do_test(string_for_ascii_filesystem_encoding(u"会社案"))
+
+
+class RmtreeFileTests(RmtreeTestCase):
+    def do_test(self, file_name):  # pylint: disable=arguments-differ
+        """Execute file test."""
+        temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root)
+        os.chdir(temp_dir)
+        create_file(file_name)
+        os.chdir(self.temp_dir_root)
+        self.assertTrue(rmtree(temp_dir))
+
+
+class RmtreeDirectoryTests(RmtreeTestCase):
+    def do_test(self, dir_name):  # pylint: disable=arguments-differ
+        """Execute directory test."""
+        os.chdir(self.temp_dir_root)
+        os.mkdir(dir_name)
+        self.assertTrue(rmtree(dir_name))
+
+
+class RmtreeDirectoryWithNonAsciiTests(RmtreeTestCase):
+    def do_test(self, name):
+        """Execute directory with non-ASCII file test."""
+        os.chdir(self.temp_dir_root)
+        os.mkdir(name)
+        os.chdir(name)
+        create_file(name)
+        os.chdir(self.temp_dir_root)
+        self.assertTrue(rmtree(name))
+
+
+class ShutilWindowsRmtreeFileTests(RmtreeFileTests):
+    def do_test(self, file_name):
+        """Execute file test that are known to fail in shutil.rmtree."""
+        if not utils.is_windows():
+            print("Skipping ShutilWindowsRmtreeFileTests on non-Windows platforms")
+            return
+        temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root)
+        os.chdir(temp_dir)
+        create_file(file_name)
+        os.chdir(self.temp_dir_root)
+        with self.assertRaises(WindowsError):  # pylint: disable=undefined-variable
+            shutil.rmtree(temp_dir)
+
+    def test_ascii(self):
+        pass
+
+    def test_unicode(self):
+        pass

etc/evergreen.yml

@@ -73,7 +73,6 @@ variables:
   - enterprise-ubuntu1204-64
   - enterprise-ubuntu1404-64
   - enterprise-ubuntu1604-64
-  - enterprise-debian71-64
   - enterprise-debian81-64
   - linux-32
   - linux-64
@@ -85,7 +84,6 @@ variables:
   - ubuntu1204
   - ubuntu1404
   - ubuntu1604
-  - debian71
   - debian81
   - osx-108-ssl
   - osx-108
@@ -163,6 +161,31 @@ functions:
         build_phase='${task_name}_${execution}'
         END_OF_CREDS
 
+  "call BF Suggestion service":
+    command: shell.exec
+    params:
+      working_dir: src
+      shell: bash
+      silent: true
+      script: |
+        report_file="report.json"
+        # Check if the report file exists and has failures.
+        if [ -f $report_file ] && grep -Eq "\"failures\": [1-9]" $report_file; then
+          # Calling the BF Suggestion server endpoint to start feature extraction.
+          payload="{\"task_id\": \"${task_id}\", \"execution\": ${execution}}"
+          echo "Sending task info to the BF suggestion service"
+          # The --user option is passed through stdin to avoid showing in process list.
+          user_option="--user ${bfsuggestion_user}:${bfsuggestion_password}"
+          curl --header "Content-Type: application/json" \
+               --data "$payload" \
+               --max-time 10 \
+               --silent \
+               --show-error \
+               --config - \
+               https://bfsuggestion.corp.mongodb.com/tasks <<< $user_option
+          echo "Request to BF Suggestion service status: $?"
+        fi
+
   "upload debugsymbols" :
     command: s3.put
     params:
@@ -413,7 +436,7 @@ functions:
         aws_secret_access_key = ${kitchen_aws_secret}
         EOF
 
-        export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz"
+        export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz"
         export KITCHEN_SECURITY_GROUP="${kitchen_security_group}"
         export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}"
         export KITCHEN_SUBNET="${kitchen_subnet}"
@@ -488,6 +511,7 @@ post:
   - command: attach.results
     params:
       file_location: src/report.json
+  - func: "call BF Suggestion service"
   - command: shell.exec
     params:
       silent: true
@@ -677,6 +701,24 @@ tasks:
           - "src/mongo/util/options_parser/test_config_files/**"
         exclude_files:
           - "*_test.pdb"
+    - command: archive.targz_pack
+      params:
+        target: "packages.tgz"
+        source_dir: "src"
+        include:
+          - "repo/**"
+          - "./**.msi"
+    - command: s3.put
+      params:
+        optional: true
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: packages.tgz
+        remote_file: ${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/tar
+        display_name: Packages
     - func: "upload debugsymbols"
     - command: s3.put
       params:
@@ -1991,6 +2033,8 @@ tasks:
 
           cat <<EOF > notary_env.sh
           export NOTARY_TOKEN=${signing_auth_token_32}
+          export BARQUE_USERNAME=${barque_user}
+          export BARQUE_PASSWORD=${barque_password}
           EOF
 
           echo "${signing_auth_token_32}" > signing_auth_token
@@ -2012,10 +2056,10 @@ tasks:
           notary-client.py --key-name "server-3.2" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} mongodb-win32-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${ext|tar.gz}
 
           if [ "${has_packages|}" = "true" ]; then
-             CURATOR_RELEASE="ea8d75dcc1a587111e7418e2428fb67e267af9fe"
+             CURATOR_RELEASE=${curator_release|"latest"}
              curl -L -O http://boxes.10gen.com/build/curator/curator-dist-rhel70-$CURATOR_RELEASE.tar.gz
              tar -zxvf curator-dist-rhel70-$CURATOR_RELEASE.tar.gz
-             ./curator repo --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages repo
+             ./curator repo submit --service ${barque_url} --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz
           fi
 
     # Put the binaries tarball/zipfile
@@ -2759,60 +2803,6 @@ buildvariants:
   - name: tool
   - name: unittests
 
-- name: linux-64-lsm
-  stepback: false
-  display_name: Linux LSM
-  modules:
-  - mongo-tools
-  run_on:
-  - rhel55-test
-  batchtime: 1440 # 1 day
-  expansions:
-    gorootvars: PATH=/opt/mongodbtoolchain/v2/bin:$PATH
-    compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
-    num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
-    test_flags: --wiredTigerCollectionConfig=type=lsm --wiredTigerIndexConfig=type=lsm
-  tasks:
-  - name: compile
-    distros:
-    - rhel55-build
-  - name: aggregation_WT
-  - name: auth_WT
-  - name: dbtest_WT
-  - name: gle_auth_WT
-  - name: gle_auth_write_cmd_WT
-  - name: gle_auth_basics_passthrough_WT
-  - name: gle_auth_basics_passthrough_write_cmd_WT
-  - name: sharding_gle_auth_basics_passthrough_WT
-  - name: sharding_gle_auth_basics_passthrough_write_cmd_WT
-  - name: jsCore_WT
-  - name: jsCore_compatibility_WT
-  - name: jsCore_small_oplog_WT
-  - name: jsCore_small_oplog_rs_WT
-  - name: jstestfuzz_WT
-  - name: jstestfuzz_replication_WT
-  - name: jstestfuzz_sharded_WT
-  - name: noPassthrough_WT
-  - name: noPassthroughWithMongod_WT
-  - name: bulk_gle_passthrough_WT
-  - name: parallel_WT
-  - name: parallel_compatibility_WT
-  - name: concurrency_WT
-  - name: concurrency_replication_WT
-  - name: concurrency_sharded_WT
-  - name: concurrency_sharded_sccc_WT
-  - name: replicasets_WT
-  - name: replication_WT
-  - name: sharding_WT
-  - name: sharding_csrs_upgrade_WT
-  - name: slow1_WT
-  - name: slow2_WT
-  - name: sharded_collections_jscore_passthrough_WT
-  - name: sharding_jscore_passthrough_WT
-  - name: sharding_jscore_passthrough_wire_ops_WT
-  - name: tool_WT
-  - name: unittests
-
 - name: ubuntu1404
   display_name: SSL Ubuntu 14.04
   modules:
@@ -5088,58 +5078,6 @@ buildvariants:
 #          Debian buildvariants           #
 ###########################################
 
-- name: enterprise-debian71-64
-  display_name: Enterprise Debian 7.1
-  modules:
-  - enterprise
-  - mongo-tools
-  run_on:
-  - debian71-test
-  batchtime: 1440 # 1 day
-  expansions:
-    gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
-    tooltags: "-tags 'ssl sasl'"
-    push_path: linux
-    push_bucket: downloads.10gen.com
-    push_name: linux
-    push_arch: x86_64-enterprise-debian71
-    compile_flags: --ssl MONGO_DISTMOD=debian71 --release -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++
-    num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
-    has_packages: true
-    packager_script: packager-enterprise.py
-    packager_arch: x86_64
-    packager_distro: debian71
-    repo_edition: enterprise
-  tasks:
-  - name: compile
-    distros:
-    - debian71-build
-  - name: audit
-  - name: audit_WT
-  - name: dbtest
-  - name: dbtest_WT
-  - name: ese_WT
-  - name: jsCore
-  - name: jsCore_auth
-  - name: jstestfuzz
-  - name: jstestfuzz_WT
-  - name: jstestfuzz_replication_WT
-  - name: jstestfuzz_sharded_WT
-  - name: mmap
-  - name: replicasets_auth
-  - name: sasl
-  - name: sharding_auth
-  - name: sharding_csrs_upgrade_auth_WT
-  - name: snmp
-  - name: snmp_WT
-  - name: ssl
-  - name: sslSpecial
-  - name: unittests
-  - name: package
-    distros:
-    - ubuntu1604-packer
-  - name: push
-
 - name: enterprise-debian81-64
   display_name: Enterprise Debian 8.1
   modules:
@@ -5193,121 +5131,6 @@ buildvariants:
   - name: push
 
 
-- name: debian71
-  display_name: SSL Debian 7.1
-  modules:
-  - mongo-tools
-  run_on:
-  - debian71-test
-  batchtime: 1440 # 1 day
-  expansions:
-    gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
-    tooltags: "-tags ssl"
-    push_path: linux
-    push_bucket: downloads.mongodb.org
-    push_name: linux
-    push_arch: x86_64-debian71
-    compile_flags: --ssl MONGO_DISTMOD=debian71 -j$(grep -c ^processor /proc/cpuinfo) --release CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++
-    num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
-    has_packages: true
-    packager_script: packager.py
-    packager_arch: x86_64
-    packager_distro: debian71
-    repo_edition: org
-  tasks:
-  - name: compile
-    distros:
-    - debian71-build
-  - name: aggregation
-  - name: aggregation_WT
-  - name: aggregation_auth
-  - name: auth
-  - name: auth_WT
-  - name: dbtest
-  - name: dbtest_WT
-  - name: disk
-  - name: durability
-  - name: dur_jscore_passthrough
-  - name: failpoints
-  - name: failpoints_auth
-  - name: gle_auth
-  - name: gle_auth_WT
-  - name: gle_auth_write_cmd
-  - name: gle_auth_write_cmd_WT
-  - name: gle_auth_basics_passthrough
-  - name: gle_auth_basics_passthrough_WT
-  - name: gle_auth_basics_passthrough_write_cmd
-  - name: gle_auth_basics_passthrough_write_cmd_WT
-  - name: sharding_gle_auth_basics_passthrough
-  - name: sharding_gle_auth_basics_passthrough_WT
-  - name: sharding_gle_auth_basics_passthrough_write_cmd
-  - name: sharding_gle_auth_basics_passthrough_write_cmd_WT
-  - name: jsCore
-  - name: jsCore_WT
-  - name: jsCore_auth
-  - name: jsCore_compatibility
-  - name: jsCore_compatibility_WT
-  - name: jsCore_small_oplog
-  - name: jsCore_small_oplog_WT
-  - name: jsCore_small_oplog_rs
-  - name: jsCore_small_oplog_rs_WT
-  - name: jstestfuzz
-  - name: jstestfuzz_WT
-  - name: jstestfuzz_replication_WT
-  - name: jstestfuzz_sharded_WT
-  - name: mmap
-  - name: mongosTest
-  - name: multiversion
-  - name: noPassthrough
-  - name: noPassthrough_WT
-  - name: noPassthroughWithMongod
-  - name: noPassthroughWithMongod_WT
-  - name: bulk_gle_passthrough
-  - name: bulk_gle_passthrough_WT
-  - name: parallel
-  - name: parallel_WT
-  - name: parallel_compatibility
-  - name: parallel_compatibility_WT
-  - name: concurrency
-  - name: concurrency_WT
-  - name: concurrency_replication
-  - name: concurrency_replication_WT
-  - name: concurrency_sharded
-  - name: concurrency_sharded_WT
-  - name: concurrency_sharded_sccc
-  - name: concurrency_sharded_sccc_WT
-  - name: replicasets
-  - name: replicasets_WT
-  - name: replicasets_auth
-  - name: replication
-  - name: replication_WT
-  - name: replication_auth
-  - name: sharding
-  - name: sharding_WT
-  - name: sharding_auth
-  - name: sharding_csrs_upgrade
-  - name: sharding_csrs_upgrade_WT
-  - name: sharding_csrs_upgrade_auth_WT
-  - name: sharding_legacy_multiversion
-  - name: slow1
-  - name: slow1_WT
-  - name: slow2
-  - name: slow2_WT
-  - name: sharded_collections_jscore_passthrough
-  - name: sharded_collections_jscore_passthrough_WT
-  - name: sharding_jscore_passthrough
-  - name: sharding_jscore_passthrough_WT
-  - name: sharding_jscore_passthrough_wire_ops_WT
-  - name: ssl
-  - name: sslSpecial
-  - name: tool
-  - name: tool_WT
-  - name: unittests
-  - name: package
-    distros:
-    - ubuntu1604-packer
-  - name: push
-
 - name: debian81
   display_name: SSL Debian 8.1
   modules:

etc/longevity.yml

@@ -176,19 +176,7 @@ functions:
           cat perf.json | egrep "name|ops_per_sec"
 
   "analyze":
-    - command: json.get_history
-      params:
-        task: ${task_name}
-        file: "work/history.json"
-        name: "perf"
-    - command: json.get_history
-      params:
-        tags: true
-        task: ${task_name}
-        file: "work/tags.json"
-        name: "perf"
     - command: shell.exec
-      # post_run_check.py and longevity_override.json for DSI tests are part of dsi repo
       type : test
       params:
         working_dir: work
@@ -198,7 +186,7 @@ functions:
           TAG="3.0.12-Baseline"
           PROJECT="mongo-longevity"
           OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/longevity_override.json"
-          python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id $PROJECT --task_name ${task_name} --variant ${build_variant}
+          python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} --refTag $TAG --overrideFile $OVERRIDEFILE --project_id $PROJECT --task_name ${task_name} --variant ${build_variant}
 
 tasks:
 - name: compile

etc/system_perf.yml

@@ -16,7 +16,7 @@ post:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: work/dsi-artifacts.tgz
-        remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.${ext|tgz}
+        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.${ext|tgz}
         bucket: mciuploads
         permissions: public-read
         content_type: ${content_type|application/x-gzip}
@@ -26,7 +26,7 @@ post:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: src/workloads/workloads/jsdoc/jsdocs-redirect.html
-        remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/workloads-${task_name}-${build_id}.html
+        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/workloads-${task_name}-${build_id}.html
         bucket: mciuploads
         permissions: public-read
         content_type: text/html
@@ -61,6 +61,44 @@ post:
     - command: shell.cleanup
 
 functions:
+  "compile mongodb":
+    - command: shell.exec
+      params:
+        working_dir: src
+        script: |
+          set -o errexit
+          set -o verbose
+          ${scons|scons} ${compile_flags|} mongo mongod mongos
+          mkdir -p mongodb/bin
+          mv mongo mongodb/bin
+          mv mongod mongodb/bin
+          mv mongos mongodb/bin
+          tar czf mongodb${wtdevelop|}.tar.gz mongodb
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/mongodb${wtdevelop|}.tar.gz
+        remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/x-gzip}
+        display_name: mongodb${wtdevelop|}.tar.gz
+
+  # NOTE: Unlike evergreen.yml, there's no conditional here. If called, this is never a noop!
+  "use WiredTiger develop" :
+    command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -o errexit
+        set -o verbose
+        cd src/third_party
+        for wtdir in api dist examples ext lang src test tools ; do
+          rm -rf wiredtiger/$wtdir
+          mv wtdevelop/$wtdir wiredtiger/
+        done
+
   "prepare environment":
     - command: shell.exec
       params:
@@ -86,7 +124,7 @@ functions:
           storageEngine: ${storageEngine}
           test_control: ${test}
           production: true
-          mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz"
+          mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project_dir}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz"
           workloads_dir: ../src/workloads/workloads
           ycsb_dir: ../src/YCSB/YCSB
           EOF
@@ -103,6 +141,7 @@ functions:
           workdir: ${workdir}
           revision: ${revision}
           project: ${project}
+          project_dir: ${project_dir}
           branch_name: ${branch_name}
 
           # sys-perf expansions
@@ -112,6 +151,7 @@ functions:
           dsi_rev: ${dsi_rev}
           workloads_rev: ${workloads_rev}
           EOF
+
     - command: shell.exec
       params:
         silent: true
@@ -128,6 +168,8 @@ functions:
           aws_secret_key: "${terraform_secret}"
           perf_jira_user: "${perf_jira_user}"
           perf_jira_pw: "${perf_jira_pw}"
+          dsi_analysis_atlas_user: "${dsi_analysis_atlas_user}"
+          dsi_analysis_atlas_pw: "${dsi_analysis_atlas_pw}"
           EOF
           chmod 400 runtime_secret.yml
     - command: shell.exec
@@ -137,10 +179,7 @@ functions:
         # configure environment, has private information, no logging
         script: |
           set -e
-          virtualenv ./venv
-          source ./venv/bin/activate
-          pip install -r ../src/dsi/dsi/requirements.txt
-          python ../src/dsi/dsi/bin/bootstrap.py
+          ../src/dsi/dsi/run-dsi python ../src/dsi/dsi/bin/bootstrap.py
     - command: shell.exec
       params:
         script: |
@@ -158,10 +197,9 @@ functions:
           set -e
           set -v
           source ./dsienv.sh
-          source ./venv/bin/activate
-          infrastructure_provisioning.py
-          workload_setup.py
-          mongodb_setup.py
+          ../src/dsi/dsi/run-dsi infrastructure_provisioning.py
+          ../src/dsi/dsi/run-dsi workload_setup.py
+          ../src/dsi/dsi/run-dsi mongodb_setup.py
 
   "run test":
     - command: shell.exec
@@ -172,51 +210,28 @@ functions:
           set -e
           set -v
           source ./dsienv.sh
-          source ./venv/bin/activate
-          test_control.py
+          ../src/dsi/dsi/run-dsi test_control.py
     - command: "json.send"
       params:
          name: "perf"
          file: "work/perf.json"
 
   "analyze":
-    - command: json.get_history
-      params:
-        task: ${task_name}
-        file: "work/history.json"
-        name: "perf"
-    - command: json.get_history
-      params:
-        tags: true
-        task: ${task_name}
-        file: "work/tags.json"
-        name: "perf"
     - command: shell.exec
-      # generate dashboard data
-      type : test
       params:
         working_dir: work
-        silent: true
         script: |
-          set -o errexit
-          TAGS="3.2.17-Baseline"
-          OVERRIDEFILE="../src/dsi/dsi/analysis/v3.2/system_perf_override.json"
-          python -u ../src/dsi/dsi/analysis/dashboard_gen.py --rev ${revision} -f history.json -t tags.json --refTag $TAGS --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} --jira-user ${perf_jira_user} --jira-password ${perf_jira_pw} || true
-    - command: "json.send"
-      params:
-         name: "dashboard"
-         file: "work/dashboard.json"
+          ../src/dsi/dsi/run-dsi detect-changes
     - command: shell.exec
-      # post_run_check.py and override.json for DSI tests are part of dsi repo
       type : test
       params:
         working_dir: work
         script: |
           set -o errexit
           set -o verbose
-          TAG="3.2.17-Baseline"
+          TAG="3.2.21-Baseline"
           OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json"
-          python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name}
+          python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name}
 
 #######################################
 #               Tasks                 #
@@ -228,42 +243,13 @@ tasks:
     - command: git.get_project
       params:
         directory: src
-    - command: git.apply_patch
-      params:
-        directory: src
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          set -o errexit
-          set -o verbose
-          ${scons|scons} ${compile_flags|} mongo mongod mongos
-          mkdir -p mongodb/bin
-          mv mongo mongodb/bin
-          mv mongod mongodb/bin
-          mv mongos mongodb/bin
-          tar cvf mongodb.tar mongodb
-          gzip mongodb.tar
-    - command: s3.put
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: src/mongodb.tar.gz
-        remote_file: ${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz
-        bucket: mciuploads
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: mongodb.tar.gz
+    - func: "compile mongodb"
 
-- name: industry_benchmarks_WT
-  depends_on:
-    - name: compile
-      variant: linux-standalone
+- name: industry_benchmarks
   priority: 5
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "wiredTiger"
         test: "ycsb"
     - func: "deploy cluster"
     - func: "run test"
@@ -271,31 +257,11 @@ tasks:
       vars:
         script_flags: --ycsb-throughput-analysis reports
 
-- name: industry_benchmarks_MMAPv1
+- name: industry_benchmarks_wmajority
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "mmapv1"
-        test: "ycsb"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-      vars:
-        script_flags: --ycsb-throughput-analysis reports
-
-- name: industry_benchmarks_wmajority_WT
-  priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "wiredTiger"
         test: "ycsb-wmajority"
     - func: "deploy cluster"
     - func: "run test"
@@ -303,73 +269,41 @@ tasks:
       vars:
         script_flags: --ycsb-throughput-analysis reports
 
-- name: industry_benchmarks_wmajority_MMAPv1
+- name: crud_workloads
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "mmapv1"
-        test: "ycsb-wmajority"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-      vars:
-        script_flags: --ycsb-throughput-analysis reports
-
-- name: crud_workloads_WT
-  priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "wiredTiger"
         test: "crud_workloads"
     - func: "deploy cluster"
     - func: "run test"
     - func: "analyze"
 
-- name: mixed_workloads_WT
+- name: mixed_workloads
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "wiredTiger"
         test: "mixed_workloads"
     - func: "deploy cluster"
     - func: "run test"
     - func: "analyze"
 
-- name: misc_workloads_WT
+- name: misc_workloads
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "wiredTiger"
         test: "misc_workloads"
     - func: "deploy cluster"
     - func: "run test"
     - func: "analyze"
 
-- name: map_reduce_workloads_WT
+- name: map_reduce_workloads
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "wiredTiger"
         test: "map_reduce_workloads"
     - func: "deploy cluster"
     - func: "run test"
@@ -377,186 +311,59 @@ tasks:
 
 - name: smoke_test
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "wiredTiger"
         test: "short"
     - func: "deploy cluster"
     - func: "run test"
     - func: "analyze"
 
-- name: crud_workloads_MMAPv1
+- name: non_sharded_workloads
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "mmapv1"
-        test: "crud_workloads"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
-- name: mixed_workloads_MMAPv1
-  priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "mmapv1"
-        test: "mixed_workloads"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
-- name: misc_workloads_MMAPv1
-  priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "mmapv1"
-        test: "misc_workloads"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
-- name: map_reduce_workloads_MMAPv1
-  priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "mmapv1"
-        test: "map_reduce_workloads"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
-- name: non_sharded_workloads_WT
-  priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "wiredTiger"
         test: "non_sharded"
     - func: "deploy cluster"
     - func: "run test"
     - func: "analyze"
 
-- name: non_sharded_workloads_MMAPv1
+- name: mongos_workloads
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "mmapv1"
-        test: "non_sharded"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
-- name: mongos_workloads_WT
-  priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "wiredTiger"
         test: "mongos"
     - func: "deploy cluster"
     - func: "run test"
     - func: "analyze"
 
-- name: mongos_workloads_MMAPv1
+- name: move_chunk_workloads
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  exec_timeout_secs: 32400 # 9 hours
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "mmapv1"
-        test: "mongos"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
-- name: move_chunk_workloads_WT
-  priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "wiredTiger"
         test: "move_chunk"
     - func: "deploy cluster"
     - func: "run test"
     - func: "analyze"
 
-- name: move_chunk_workloads_MMAPv1
+- name: move_chunk_waiting_workloads
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "mmapv1"
-        test: "move_chunk"
+        test: "move_chunk_waiting"
     - func: "deploy cluster"
     - func: "run test"
     - func: "analyze"
 
-- name: initialsync_WT
+- name: initialsync
   priority: 5
-  depends_on:
-    - name: compile
-      variant: linux-standalone
   commands:
     - func: "prepare environment"
       vars:
-        storageEngine: "wiredTiger"
-        test: "initialsync"
-    - func: "deploy cluster"
-    - func: "run test"
-    - func: "analyze"
-
-- name: initialsync_MMAPv1
-  priority: 5
-  depends_on:
-    - name: compile
-      variant : linux-standalone
-  commands:
-    - func: "prepare environment"
-      vars:
-        storageEngine: "mmapv1"
         test: "initialsync"
     - func: "deploy cluster"
     - func: "run test"
@@ -579,11 +386,34 @@ modules:
   branch: master
 
 
+#######################################
+#         Buildvariants               #
+#######################################
+buildvariants:
+
+# We are explicitly tracking the rhel70 variant compile options from evergreen.yml.  If we can get
+# proper artifacts directly from that project, we should do that and remove these tasks.
+- name: compile-rhel70
+  display_name: Compile on rhel70
+  batchtime: 1440 # 24 hours
+  modules:
+    - wtdevelop
+  expansions:
+    compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars
+    gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH"
+    platform: linux
+    project_dir: &project_dir dsi
+    tooltags: ""
+    use_scons_cache: true
+  run_on:
+      - "rhel70"
+  tasks:
+    - name: compile
+
+
 #######################################
 #         Linux Buildvariants         #
 #######################################
-
-buildvariants:
 - name: linux-1-node-replSet
   display_name: Linux 1-Node ReplSet
   batchtime: 10080 # 7 days
@@ -595,53 +425,45 @@ buildvariants:
     setup: single-replica
     cluster: single
     platform: linux
-    project: &project dsi-v3.2
+    project_dir: *project_dir
+    storageEngine: wiredTiger
   run_on:
       - "rhel70-perf-single"
-  tasks:
-    - name: industry_benchmarks_WT
-    - name: crud_workloads_WT
-    - name: mixed_workloads_WT
-    - name: misc_workloads_WT
-    - name: map_reduce_workloads_WT
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
+  tasks: &1nodetasks
+    - name: industry_benchmarks
+    - name: crud_workloads
+    - name: mixed_workloads
+    - name: misc_workloads
+    - name: map_reduce_workloads
     - name: smoke_test
-    - name: industry_benchmarks_MMAPv1
-    - name: crud_workloads_MMAPv1
-    - name: mixed_workloads_MMAPv1
-    - name: misc_workloads_MMAPv1
-    - name: map_reduce_workloads_MMAPv1
-    - name: non_sharded_workloads_WT
-    - name: non_sharded_workloads_MMAPv1
+    - name: non_sharded_workloads
 
 - name: linux-standalone
   display_name: Linux Standalone
   batchtime: 10080 # 7 days
   modules: *modules
   expansions:
-    compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
     setup: standalone
     cluster: single
     platform: linux
-    project: *project
+    project_dir: *project_dir
+    storageEngine: wiredTiger
   run_on:
       - "rhel70-perf-single"
-  tasks:
-    - name: compile
-      distros:
-      - rhel70
-    - name: industry_benchmarks_WT
-    - name: crud_workloads_WT
-    - name: mixed_workloads_WT
-    - name: misc_workloads_WT
-    - name: map_reduce_workloads_WT
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
+  tasks: &standalonetasks
+    - name: industry_benchmarks
+    - name: crud_workloads
+    - name: mixed_workloads
+    - name: misc_workloads
+    - name: map_reduce_workloads
     - name: smoke_test
-    - name: industry_benchmarks_MMAPv1
-    - name: crud_workloads_MMAPv1
-    - name: mixed_workloads_MMAPv1
-    - name: misc_workloads_MMAPv1
-    - name: map_reduce_workloads_MMAPv1
-    - name: non_sharded_workloads_WT
-    - name: non_sharded_workloads_MMAPv1
+    - name: non_sharded_workloads
 
 - name: linux-3-shard
   display_name: Linux 3-Shard Cluster
@@ -652,27 +474,23 @@ buildvariants:
     setup: shard
     cluster: shard
     platform: linux
-    project: *project
+    project_dir: *project_dir
+    storageEngine: wiredTiger
   run_on:
       - "rhel70-perf-shard"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
   tasks:
-    - name: industry_benchmarks_WT
-    - name: crud_workloads_WT
-    - name: mixed_workloads_WT
-    - name: misc_workloads_WT
-    - name: map_reduce_workloads_WT
+    - name: industry_benchmarks
+    - name: crud_workloads
+    - name: mixed_workloads
+    - name: misc_workloads
+    - name: map_reduce_workloads
     - name: smoke_test
-    - name: industry_benchmarks_MMAPv1
-    - name: industry_benchmarks_wmajority_WT
-    - name: industry_benchmarks_wmajority_MMAPv1
-    - name: crud_workloads_MMAPv1
-    - name: mixed_workloads_MMAPv1
-    - name: misc_workloads_MMAPv1
-    - name: map_reduce_workloads_MMAPv1
-    - name: mongos_workloads_WT
-    - name: mongos_workloads_MMAPv1
-    - name: move_chunk_workloads_WT
-    - name: move_chunk_workloads_MMAPv1
+    - name: industry_benchmarks_wmajority
+    - name: mongos_workloads
+    - name: move_chunk_workloads
 
 - name: linux-3-node-replSet
   display_name: Linux 3-Node ReplSet
@@ -683,25 +501,22 @@ buildvariants:
     setup: replica
     cluster: replica
     platform: linux
-    project: *project
+    project_dir: *project_dir
+    storageEngine: wiredTiger
   run_on:
       - "rhel70-perf-replset"
-  tasks:
-    - name: industry_benchmarks_WT
-    - name: crud_workloads_WT
-    - name: mixed_workloads_WT
-    - name: misc_workloads_WT
-    - name: map_reduce_workloads_WT
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
+  tasks: &3nodetasks
+    - name: industry_benchmarks
+    - name: crud_workloads
+    - name: mixed_workloads
+    - name: misc_workloads
+    - name: map_reduce_workloads
     - name: smoke_test
-    - name: industry_benchmarks_MMAPv1
-    - name: industry_benchmarks_wmajority_WT
-    - name: industry_benchmarks_wmajority_MMAPv1
-    - name: crud_workloads_MMAPv1
-    - name: mixed_workloads_MMAPv1
-    - name: misc_workloads_MMAPv1
-    - name: map_reduce_workloads_MMAPv1
-    - name: non_sharded_workloads_WT
-    - name: non_sharded_workloads_MMAPv1
+    - name: industry_benchmarks_wmajority
+    - name: non_sharded_workloads
 
 - name: linux-3-node-replSet-initialsync
   display_name: Linux 3-Node ReplSet Initial Sync
@@ -712,10 +527,132 @@ buildvariants:
     setup: replica-2node
     cluster: replica
     platform: linux
-    project: *project
+    project_dir: *project_dir
+    storageEngine: wiredTiger
   run_on:
       - "rhel70-perf-replset"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
   tasks:
-    - name: initialsync_WT
-    - name: initialsync_MMAPv1
+    - name: initialsync
 
+#######################################
+#         MMAP Buildvariants         #
+#######################################
+- name: mmap-1-node-replSet
+  display_name: MMAP 1-Node ReplSet
+  batchtime: 10080 # 7 days
+  modules: *modules
+  expansions:
+    setup: single-replica
+    cluster: single
+    platform: linux
+    storageEngine: "mmapv1"
+    project_dir: *project_dir
+  run_on:
+      - "rhel70-perf-single"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
+  tasks:
+    - name: smoke_test
+    - name: industry_benchmarks
+    - name: crud_workloads
+    - name: mixed_workloads
+    - name: misc_workloads
+    - name: map_reduce_workloads
+    - name: non_sharded_workloads
+
+- name: mmap-standalone
+  display_name: MMAP Standalone
+  batchtime: 10080 # 7 days
+  modules: *modules
+  expansions:
+    setup: standalone
+    cluster: single
+    platform: linux
+    storageEngine: "mmapv1"
+    project_dir: *project_dir
+  run_on:
+      - "rhel70-perf-single"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
+  tasks:
+    - name: smoke_test
+    - name: industry_benchmarks
+    - name: crud_workloads
+    - name: mixed_workloads
+    - name: misc_workloads
+    - name: map_reduce_workloads
+    - name: non_sharded_workloads
+
+- name: mmap-3-shard
+  display_name: MMAP 3-Shard Cluster
+  batchtime: 10080 # 7 days
+  modules: *modules
+  expansions:
+    setup: shard
+    cluster: shard
+    platform: linux
+    storageEngine: "mmapv1"
+    project_dir: *project_dir
+  run_on:
+      - "rhel70-perf-shard"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
+  tasks:
+    - name: smoke_test
+    - name: industry_benchmarks
+    - name: industry_benchmarks_wmajority
+    - name: crud_workloads
+    - name: mixed_workloads
+    - name: misc_workloads
+    - name: map_reduce_workloads
+    - name: mongos_workloads
+    - name: move_chunk_workloads
+
+- name: mmap-3-node-replSet
+  display_name: MMAP 3-Node ReplSet
+  batchtime: 10080 # 7 days
+  modules: *modules
+  expansions:
+    setup: replica
+    cluster: replica
+    platform: linux
+    storageEngine: "mmapv1"
+    project_dir: *project_dir
+  run_on:
+      - "rhel70-perf-replset"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
+  tasks:
+    - name: smoke_test
+    - name: industry_benchmarks
+    - name: industry_benchmarks_wmajority
+    - name: crud_workloads
+    - name: mixed_workloads
+    - name: misc_workloads
+    - name: map_reduce_workloads
+    - name: non_sharded_workloads
+
+- name: mmap-3-node-replSet-initialsync
+  display_name: MMAP 3-Node ReplSet Initial Sync
+  batchtime: 10080 # 7 days
+  modules: *modules
+  expansions:
+    setup: replica-2node
+    cluster: replica
+    platform: linux
+    storageEngine: "mmapv1"
+    project_dir: *project_dir
+  run_on:
+      - "rhel70-perf-replset"
+  depends_on:
+      - name: compile
+        variant: compile-rhel70
+  tasks:
+    - name: initialsync

jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js

@@ -48,6 +48,7 @@ var blacklist = [
     'convert_to_capped_collection_index.js',  // convertToCapped can't be run on mongos processes
     'findAndModify_remove_queue.js',          // remove cannot be {} for findAndModify
     'findAndModify_update_collscan.js',       // findAndModify requires a shard key
+    'findAndModify_update_grow.js',           // can cause OOM kills on test hosts
     'findAndModify_update_queue.js',          // findAndModify requires a shard key
     'group.js',                // the group command cannot be issued against a sharded cluster
     'group_cond.js',           // the group command cannot be issued against a sharded cluster

jstests/core/minmax.js

@@ -1,97 +1,142 @@
-// test min / max query parameters
+// Test min / max query parameters.
+(function() {
+    "use strict";
 
-addData = function() {
-    t.save({a: 1, b: 1});
-    t.save({a: 1, b: 2});
-    t.save({a: 2, b: 1});
-    t.save({a: 2, b: 2});
-};
+    load("jstests/aggregation/extras/utils.js");  // For resultsEq.
 
-t = db.jstests_minmax;
-t.drop();
-t.ensureIndex({a: 1, b: 1});
-addData();
+    const coll = db.jstests_minmax;
+    coll.drop();
 
-printjson(t.find().min({a: 1, b: 2}).max({a: 2, b: 1}).toArray());
-assert.eq(1, t.find().min({a: 1, b: 2}).max({a: 2, b: 1}).toArray().length);
-assert.eq(2, t.find().min({a: 1, b: 2}).max({a: 2, b: 1.5}).toArray().length);
-assert.eq(2, t.find().min({a: 1, b: 2}).max({a: 2, b: 2}).toArray().length);
+    function addData() {
+        assert.writeOK(coll.save({a: 1, b: 1}));
+        assert.writeOK(coll.save({a: 1, b: 2}));
+        assert.writeOK(coll.save({a: 2, b: 1}));
+        assert.writeOK(coll.save({a: 2, b: 2}));
+    }
 
-// just one bound
-assert.eq(3, t.find().min({a: 1, b: 2}).toArray().length);
-assert.eq(3, t.find().max({a: 2, b: 1.5}).toArray().length);
-assert.eq(3, t.find().min({a: 1, b: 2}).hint({a: 1, b: 1}).toArray().length);
-assert.eq(3, t.find().max({a: 2, b: 1.5}).hint({a: 1, b: 1}).toArray().length);
+    assert.commandWorked(coll.ensureIndex({a: 1, b: 1}));
+    addData();
 
-t.drop();
-t.ensureIndex({a: 1, b: -1});
-addData();
-assert.eq(4, t.find().min({a: 1, b: 2}).toArray().length);
-assert.eq(4, t.find().max({a: 2, b: 0.5}).toArray().length);
-assert.eq(1, t.find().min({a: 2, b: 1}).toArray().length);
-assert.eq(1, t.find().max({a: 1, b: 1.5}).toArray().length);
-assert.eq(4, t.find().min({a: 1, b: 2}).hint({a: 1, b: -1}).toArray().length);
-assert.eq(4, t.find().max({a: 2, b: 0.5}).hint({a: 1, b: -1}).toArray().length);
-assert.eq(1, t.find().min({a: 2, b: 1}).hint({a: 1, b: -1}).toArray().length);
-assert.eq(1, t.find().max({a: 1, b: 1.5}).hint({a: 1, b: -1}).toArray().length);
+    assert.eq(1, coll.find().min({a: 1, b: 2}).max({a: 2, b: 1}).toArray().length);
+    assert.eq(2, coll.find().min({a: 1, b: 2}).max({a: 2, b: 1.5}).toArray().length);
+    assert.eq(2, coll.find().min({a: 1, b: 2}).max({a: 2, b: 2}).toArray().length);
 
-// hint doesn't match
-assert.throws(function() {
-    t.find().min({a: 1}).hint({a: 1, b: -1}).toArray();
-});
-assert.throws(function() {
-    t.find().min({a: 1, b: 1}).max({a: 1}).hint({a: 1, b: -1}).toArray();
-});
-assert.throws(function() {
-    t.find().min({b: 1}).max({a: 1, b: 2}).hint({a: 1, b: -1}).toArray();
-});
-assert.throws(function() {
-    t.find().min({a: 1}).hint({$natural: 1}).toArray();
-});
-assert.throws(function() {
-    t.find().max({a: 1}).hint({$natural: 1}).toArray();
-});
+    // Single bound.
+    assert.eq(3, coll.find().min({a: 1, b: 2}).toArray().length);
+    assert.eq(3, coll.find().max({a: 2, b: 1.5}).toArray().length);
+    assert.eq(3, coll.find().min({a: 1, b: 2}).hint({a: 1, b: 1}).toArray().length);
+    assert.eq(3, coll.find().max({a: 2, b: 1.5}).hint({a: 1, b: 1}).toArray().length);
 
-// Reverse direction scan of the a:1 index between a:6 (inclusive) and a:3 (exclusive).
-t.drop();
-t.ensureIndex({a: 1});
-for (i = 0; i < 10; ++i) {
-    t.save({_id: i, a: i});
-}
-if (0) {  // SERVER-3766
-    reverseResult = t.find().min({a: 6}).max({a: 3}).sort({a: -1}).hint({a: 1}).toArray();
-    assert.eq([{_id: 6, a: 6}, {_id: 5, a: 5}, {_id: 4, a: 4}], reverseResult);
-}
+    coll.drop();
+    assert.commandWorked(coll.ensureIndex({a: 1, b: -1}));
+    addData();
+    assert.eq(4, coll.find().min({a: 1, b: 2}).toArray().length);
+    assert.eq(4, coll.find().max({a: 2, b: 0.5}).toArray().length);
+    assert.eq(1, coll.find().min({a: 2, b: 1}).toArray().length);
+    assert.eq(1, coll.find().max({a: 1, b: 1.5}).toArray().length);
+    assert.eq(4, coll.find().min({a: 1, b: 2}).hint({a: 1, b: -1}).toArray().length);
+    assert.eq(4, coll.find().max({a: 2, b: 0.5}).hint({a: 1, b: -1}).toArray().length);
+    assert.eq(1, coll.find().min({a: 2, b: 1}).hint({a: 1, b: -1}).toArray().length);
+    assert.eq(1, coll.find().max({a: 1, b: 1.5}).hint({a: 1, b: -1}).toArray().length);
 
-//
-// SERVER-15015.
-//
+    // Hint doesn't match.
+    assert.throws(function() {
+        coll.find().min({a: 1}).hint({a: 1, b: -1}).toArray();
+    });
+    assert.throws(function() {
+        coll.find().min({a: 1, b: 1}).max({a: 1}).hint({a: 1, b: -1}).toArray();
+    });
+    assert.throws(function() {
+        coll.find().min({b: 1}).max({a: 1, b: 2}).hint({a: 1, b: -1}).toArray();
+    });
+    assert.throws(function() {
+        coll.find().min({a: 1}).hint({$natural: 1}).toArray();
+    });
+    assert.throws(function() {
+        coll.find().max({a: 1}).hint({$natural: 1}).toArray();
+    });
 
-// Test ascending index.
-t.drop();
-t.ensureIndex({a: 1});
-t.insert({a: 3});
-t.insert({a: 4});
-t.insert({a: 5});
+    coll.drop();
+    assert.commandWorked(coll.ensureIndex({a: 1}));
+    for (var i = 0; i < 10; ++i) {
+        assert.writeOK(coll.save({_id: i, a: i}));
+    }
 
-var cursor = t.find().min({a: 4});
-assert.eq(4, cursor.next()["a"]);
-assert.eq(5, cursor.next()["a"]);
-assert(!cursor.hasNext());
+    // Reverse direction scan of the a:1 index between a:6 (inclusive) and a:3 (exclusive) is
+    // expected to fail, as max must be > min.
+    var error = assert.throws(function() {
+        coll.find().min({a: 6}).max({a: 3}).sort({a: -1}).toArray();
+    });
+    assert.eq(error.code, ErrorCodes.BadValue);
 
-cursor = t.find().max({a: 4});
-assert.eq(3, cursor.next()["a"]);
-assert(!cursor.hasNext());
+    error = assert.throws(function() {
+        coll.find().min({a: 6}).max({a: 3}).sort({a: -1}).hint({a: 1}).toArray();
+    });
+    assert.eq(error.code, ErrorCodes.BadValue);
 
-// Test descending index.
-t.dropIndexes();
-t.ensureIndex({a: -1});
+    // A find with identical min and max values is expected to fail, as max is exclusive.
+    error = assert.throws(function() {
+        coll.find().min({a: 2}).max({a: 2}).toArray();
+    });
+    assert.eq(error.code, ErrorCodes.BadValue);
 
-cursor = t.find().min({a: 4});
-assert.eq(4, cursor.next()["a"]);
-assert.eq(3, cursor.next()["a"]);
-assert(!cursor.hasNext());
+    error = assert.throws(function() {
+        coll.find().min({a: 2}).max({a: 2}).hint({a: 1}).toArray();
+    });
+    assert.eq(error.code, ErrorCodes.BadValue);
 
-cursor = t.find().max({a: 4});
-assert.eq(5, cursor.next()["a"]);
-assert(!cursor.hasNext());
+    error = assert.throws(function() {
+        coll.find().min({a: 2}).max({a: 2}).sort({a: -1}).toArray();
+    });
+    assert.eq(error.code, ErrorCodes.BadValue);
+
+    error = assert.throws(function() {
+        coll.find().min({a: 2}).max({a: 2}).sort({a: -1}).hint({a: 1}).toArray();
+    });
+    assert.eq(error.code, ErrorCodes.BadValue);
+
+    coll.drop();
+    addData();
+    assert.commandWorked(coll.ensureIndex({a: 1, b: 1}));
+
+    error = assert.throws(function() {
+        coll.find().min({a: 1, b: 2}).max({a: 1, b: 2}).toArray();
+    });
+    assert.eq(error.code, ErrorCodes.BadValue);
+
+    error = assert.throws(function() {
+        coll.find().min({a: 1, b: 2}).max({a: 1, b: 2}).hint({a: 1, b: 1}).toArray();
+    });
+    assert.eq(error.code, ErrorCodes.BadValue);
+
+    // Test ascending index.
+    coll.drop();
+    assert.commandWorked(coll.ensureIndex({a: 1}));
+    assert.writeOK(coll.insert({a: 3}));
+    assert.writeOK(coll.insert({a: 4}));
+    assert.writeOK(coll.insert({a: 5}));
+
+    var cursor = coll.find().min({a: 4});
+    assert.eq(4, cursor.next().a);
+    assert.eq(5, cursor.next().a);
+
+    assert(!cursor.hasNext());
+
+    cursor = coll.find().max({a: 4});
+    assert.eq(3, cursor.next()["a"]);
+    assert(!cursor.hasNext());
+
+    // Test descending index.
+    assert.commandWorked(coll.dropIndexes());
+    assert.commandWorked(coll.ensureIndex({a: -1}));
+
+    cursor = coll.find().min({a: 4});
+    assert.eq(4, cursor.next().a);
+    assert.eq(3, cursor.next().a);
+
+    assert(!cursor.hasNext());
+
+    cursor = coll.find().max({a: 4});
+    assert.eq(5, cursor.next()["a"]);
+    assert(!cursor.hasNext());
+}());

jstests/core/minmax_edge.js

@@ -79,8 +79,6 @@ verifyMax({a: {a: 1}}, [0, 1, 2, 3, 4, 5, 6, 7, 8]);
 verifyMin({a: 'a'}, []);
 verifyMax({a: 'a'}, [0, 1, 2, 3, 4, 5, 6, 7, 8]);
 
-verifyResultIds(t.find().min({a: 4}).max({a: 4}).toArray(), []);
-
 // Now with a compound index.
 reset(t);
 assert.commandWorked(t.ensureIndex({a: 1, b: -1}));
@@ -101,8 +99,6 @@ verifyMax({a: {a: 1}, b: 1}, [0, 1, 2, 3, 4, 5, 6, 7, 8]);
 verifyMin({a: 'a', b: 1}, []);
 verifyMax({a: 'a', b: 1}, [0, 1, 2, 3, 4, 5, 6, 7, 8]);
 
-verifyResultIds(t.find().min({a: 4, b: 1}).max({a: 4, b: 1}).toArray(), []);
-
 // Edge cases on b values
 verifyMin({a: 1, b: Infinity}, [0, 1, 2, 3, 4, 5, 6, 7, 8]);
 verifyMin({a: 2, b: Infinity}, [3, 4, 5, 6, 7, 8]);
@@ -146,8 +142,6 @@ verifyMax({a: {a: 1}}, []);
 verifyMin({a: 'a'}, [0, 1, 2, 3, 4, 5, 6, 7, 8]);
 verifyMax({a: 'a'}, []);
 
-verifyResultIds(t.find().min({a: 4}).max({a: 4}).toArray(), []);
-
 // Now with a compound index.
 reset(t);
 t.ensureIndex({a: -1, b: -1});

jstests/libs/check_log.js

@@ -29,7 +29,7 @@ var checkLog;
          * the provided 'msg' is found in the logs, or 5 minutes have elapsed. Throws an exception
          * on timeout.
          */
-        var contains = function(conn, msg) {
+        var contains = function(conn, msg, timeoutSeconds = 5 * 60) {
             assert.soon(
                 function() {
                     var logMessages = getGlobalLog(conn);
@@ -44,7 +44,7 @@ var checkLog;
                     return false;
                 },
                 'Could not find log entries containing the following message: ' + msg,
-                5 * 60 * 1000,
+                timeoutSeconds * 1000,
                 300);
         };
 

jstests/libs/client-self-signed.pem

@@ -0,0 +1,47 @@
+-----BEGIN CERTIFICATE-----
+MIIDXDCCAkQCCQCDkcJhHXlrtzANBgkqhkiG9w0BAQsFADBwMQ8wDQYDVQQDDAZj
+bGllbnQxEzARBgNVBAsMCktlcm5lbFVzZXIxEDAOBgNVBAoMB01vbmdvREIxFjAU
+BgNVBAcMDU5ldyBZb3JrIENpdHkxETAPBgNVBAgMCE5ldyBZb3JrMQswCQYDVQQG
+EwJVUzAeFw0xODA1MDgxOTA2MTdaFw0yMDA3MTcxOTA2MTdaMHAxDzANBgNVBAMM
+BmNsaWVudDETMBEGA1UECwwKS2VybmVsVXNlcjEQMA4GA1UECgwHTW9uZ29EQjEW
+MBQGA1UEBwwNTmV3IFlvcmsgQ2l0eTERMA8GA1UECAwITmV3IFlvcmsxCzAJBgNV
+BAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxIP1BCsp6e+B
+5BpefH+yDvdVwDteI3H7CKh8X0TJYAYHAy5bOJ9fS0PB50gAXYqDFTHWTnGGxTaq
+67d4Hc5ZB+UgnNralBvhVYB6jFpV4oBqdht//VhcdLBq8n6aRmM63zP82dKA5Ehz
+q7jhwLI0v/m3lZfkE+qKWb+E87KS47BstOHDIqAUIeUJutrC/usS7aatRhHqtmg2
+8pEVJV8/YBPL4DWZkKoqrg+hJjXNjpy7ec2GAvgk+5ZWvjmNUBTtrE+47l8Eq3nM
+TLsTDqtPHqv/udZScaCltPkw2ffGMhhp/eFLF+F3E9NpZC5mpUfe4+E5iZAXL9/r
+IRO4ESbI9wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB4DscFT2z5hgzHC64SVjHk
+Rpodjct0/kLkcJpaAZ/VTU6J6JI0TRDI0lsYV+UlogPxNWHgo7OGmG26tUPh61TX
+XCq0Y1SbUsJFJa61d4fl30cBsA/w+eN1GRMua+eo1DTeFC8n0bVUqCjaMzumZWnX
+bOR5eUUGw2r/JbCvCU2QNuL6/LKg2e2/C6pwkxQzFpKVTgvc0mq9+rifAIo0a9JV
+41FIpgbJrojXCjP1gs2On3Uvr0XJsiMd9+T6duZt5MNh4M3Mfrr+tzATM61JM0Mg
+VryKakzERFMLnTH3aS/JfTMGlCvUJEiMU2N9mePmSucNnJHX6wYjmOr++6RoLBhc
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAxIP1BCsp6e+B5BpefH+yDvdVwDteI3H7CKh8X0TJYAYHAy5b
+OJ9fS0PB50gAXYqDFTHWTnGGxTaq67d4Hc5ZB+UgnNralBvhVYB6jFpV4oBqdht/
+/VhcdLBq8n6aRmM63zP82dKA5Ehzq7jhwLI0v/m3lZfkE+qKWb+E87KS47BstOHD
+IqAUIeUJutrC/usS7aatRhHqtmg28pEVJV8/YBPL4DWZkKoqrg+hJjXNjpy7ec2G
+Avgk+5ZWvjmNUBTtrE+47l8Eq3nMTLsTDqtPHqv/udZScaCltPkw2ffGMhhp/eFL
+F+F3E9NpZC5mpUfe4+E5iZAXL9/rIRO4ESbI9wIDAQABAoIBAEV/SUJNJYiG3E/Y
+B3xOCKGEVP4jwLoWUCoSs9FZWUItP4OffEE9E41hjAyICSELoCn5swmQ2pGaML4l
+50PbOt8w5+Sw/UYNyhXb1vsV43Oz6HKWbSAihKf70AasQVENPL+8+fnqmQyils2v
+F593JnKSFJvB2LY2ZvEwawtoJsyAdENw+NacVnEoMbaJqn0x0XsFb0RAvV9PQaFD
+U29yg3T3TxnoDHxmnPaLUhn4ht7bOc66PjeJEl2jvnhNdSwPND9dg+hF/hX966Io
+vNfKyp5q5nr6dnshjMWciyAu3v0FhaQ+7MTG2pij1yR1L1XM/gxL7u65bCCw3D2d
+BEiiB2ECgYEA7HoiLTImR2IN3QSkKi/j6YzLIhpZgfl1XW3P2EKXcbToghDsbZtV
+81ZtPwX5R5gCSKR7c05hPOgio2ext71jomRQYDHHTFu2gbKyk0iC3fhY7u44ugRQ
+KOMqQE2WZUV4Ll9QOflMoSWypDmSzgG5hYF86dSW1Leuoo8nDUnvcocCgYEA1L0/
+YNZoF+h4OsO3t7zJ38X6vmn7vwkWwlFC8puaseUmL0ENWGZxtBZzuzcVHWzSTxzk
++lpKNUloXeEE39btVWonDUx4uPajHsXE3i8/jjgOxden36cGxXxV1yWaSGTC7ikM
+TPSzhtyMxlsmG7cXgIELCiWF0H2Mt+RGwTnz4hECgYEApQD/keu5O/vWUo7ngaxS
+6mbcYppiNEf9Ncjmyy6D/8pMplzqq/xEd/upVN11OBiAF3H11douDVKowL4dCmky
++V/nmYeCWexwp4UqB40EC6A7ZdSBboKN8Em59I5uz5Eur6TAFQO4DYWBZgfqQKaB
+bf+RSpOU/y5/w1wB0srxMkcCgYEAyq36geIDjjOWyDXLBKxCkIVoXJ1QkTKxkjQl
+WkpRWYb9mCub8Uq4rWBupkDWRxB0VZcruDDpXlcvrbFCGZgEyUm3Jv5iTaX8xsaY
+xy2wmWhi+q55a6UP2Hqatb1Hfg0xggFtjKvdlnDtVdVlOyZ2p6FJyULyeC4FMPW1
+S0ZRSmECgYEAyjTgEjvIUsd4NkVdMcBBHO/WKl7gMQDcUGOnlrpyEPumXC1+H2jy
+GWbnBoWyXJ+SrMP3MFpCEG7BZ7P5oFjLDBJ8+xUiHWPLLGRRhp2tyySeR/urjgRO
+kLX6W+WWErw2V5+KmgEJb/SpbAGcJhDyLCN56L/Osqrod9avQ4/YW60=
+-----END RSA PRIVATE KEY-----

jstests/replsets/initial_sync_oplog_visibility.js

@@ -0,0 +1,66 @@
+/**
+ * Test that we wait for oplog visibility before beginning initial sync.
+ */
+(function() {
+    'use strict';
+    load("jstests/libs/check_log.js");
+
+    var name = 'initial_sync_oplog_visibility';
+
+    var replTest = new ReplSetTest({name: name, nodes: 1});
+    replTest.startSet();
+    replTest.initiate();
+    var primary = replTest.getPrimary();
+    assert.commandWorked(
+        primary.adminCommand({setParameter: 1, logComponentVerbosity: {query: 3}}));
+
+    var firstColl = "hangColl";
+    var secondColl = "secondColl";
+
+    // Create both collections.
+    assert.writeOK(primary.getDB(name)[firstColl].insert({init: 1}));
+    assert.writeOK(primary.getDB(name)[secondColl].insert({init: 1}));
+
+    jsTestLog("Add a node to initial sync.");
+    var secondary = replTest.add({});
+    assert.commandWorked(secondary.adminCommand(
+        {configureFailPoint: 'initialSyncHangBeforeOplogVisibility', mode: 'alwaysOn'}));
+    replTest.reInitiate();
+    checkLog.contains(secondary,
+                      "initial sync - initialSyncHangBeforeOplogVisibility fail point enabled");
+
+    // Start an insert that will hang in a parallel shell.
+    assert.commandWorked(
+        primary.adminCommand({configureFailPoint: 'hangOnInsertObserver', mode: 'alwaysOn'}));
+    const awaitInsertShell = startParallelShell(function() {
+        var name = 'initial_sync_oplog_visibility';
+        var firstColl = "hangColl";
+        assert.writeOK(db.getSiblingDB(name)[firstColl].insert({a: 1}));
+    }, primary.port);
+    checkLog.contains(primary, "op observer - hangOnInsertObserver fail point enabled");
+
+    // Let initial sync finish and fail.
+    assert.commandWorked(secondary.adminCommand(
+        {configureFailPoint: 'initialSyncHangBeforeOplogVisibility', mode: 'off'}));
+    checkLog.contains(secondary, "initial sync attempt failed");
+
+    // Let the insert and the initial sync finish.
+    assert.commandWorked(
+        primary.adminCommand({configureFailPoint: 'hangOnInsertObserver', mode: 'off'}));
+    awaitInsertShell();
+    replTest.awaitSecondaryNodes();
+
+    // The oplog visibility query should use an oplog-optimized plan. Check vaguely for this by
+    // awaiting a characteristic log message for each storage engine - we at least know that *some*
+    // query used the optimal plan around the time of the visibility query.
+    const timeoutSeconds = 30;
+    if (primary.adminCommand("serverStatus").storageEngine.name === "wiredTiger") {
+        jsTestLog("Checking for log message about 'direct oplog seek' query plan.");
+        checkLog.contains(primary, "Using direct oplog seek", timeoutSeconds);
+    } else if (primary.adminCommand("serverStatus").storageEngine.name === "mmapv1") {
+        jsTestLog("Checking for log message about 'Using OplogStart stage'.");
+        checkLog.contains(primary, "Using OplogStart stage", timeoutSeconds);
+    }
+
+    replTest.stopSet();
+})();

jstests/ssl/x509_invalid.js

@@ -0,0 +1,59 @@
+// Test X509 auth when --sslAllowInvalidCertificates is enabled
+
+(function() {
+    'use strict';
+
+    const CLIENT_NAME = 'C=US,ST=New York,L=New York City,O=MongoDB,OU=KernelUser,CN=client';
+    const CLIENT_CERT = 'jstests/libs/client.pem';
+    const SERVER_CERT = 'jstests/libs/server.pem';
+    const CA_CERT = 'jstests/libs/ca.pem';
+    const SELF_SIGNED_CERT = 'jstests/libs/client-self-signed.pem';
+
+    function testClient(conn, cert, name, shouldSucceed) {
+        const auth = {
+            mechanism: 'MONGODB-X509',
+            user: name
+        };
+        const script = 'assert(db.getSiblingDB(\'$external\').auth(' + tojson(auth) + '));';
+        clearRawMongoProgramOutput();
+        const exitCode = runMongoProgram('mongo',
+                                         '--ssl',
+                                         '--sslAllowInvalidHostnames',
+                                         '--sslPEMKeyFile',
+                                         cert,
+                                         '--sslCAFile',
+                                         CA_CERT,
+                                         '--port',
+                                         conn.port,
+                                         '--eval',
+                                         script);
+
+        assert.eq(shouldSucceed, exitCode === 0, "exitCode = " + tojson(exitCode));
+        assert.eq(
+            !shouldSucceed,
+            rawMongoProgramOutput().includes('No verified subject name available from client'));
+    }
+
+    function runTest(conn) {
+        const admin = conn.getDB('admin');
+        admin.createUser({user: "admin", pwd: "admin", roles: ["root"]});
+        admin.auth('admin', 'admin');
+
+        const external = conn.getDB('$external');
+        external.createUser({user: CLIENT_NAME, roles: [{'role': 'readWrite', 'db': 'test'}]});
+
+        testClient(conn, CLIENT_CERT, CLIENT_NAME, true);
+        testClient(conn, SELF_SIGNED_CERT, CLIENT_NAME, false);
+    }
+
+    // Standalone.
+    const mongod = MongoRunner.runMongod({
+        auth: '',
+        sslMode: 'requireSSL',
+        sslPEMKeyFile: SERVER_CERT,
+        sslCAFile: CA_CERT,
+        sslAllowInvalidCertificates: '',
+    });
+    runTest(mongod);
+    MongoRunner.stopMongod(mongod);
+})();

jstests/ssl/x509_startup_warning.js

@@ -0,0 +1,35 @@
+// Test for startuo warning when X509 auth and sslAllowInvalidCertificates are enabled
+
+(function() {
+    'use strict';
+
+    function runTest(opts, expectWarning) {
+        clearRawMongoProgramOutput();
+        const mongod = MongoRunner.runMongod(Object.assign(
+            {
+              auth: '',
+              sslMode: 'requireSSL',
+              sslPEMKeyFile: 'jstests/libs/server.pem',
+              sslCAFile: 'jstests/libs/ca.pem',
+            },
+            opts));
+        assert.eq(expectWarning,
+                  rawMongoProgramOutput().includes(
+                      'WARNING: While invalid X509 certificates may be used'));
+        MongoRunner.stopMongod(mongod);
+    }
+
+    // Don't expect a warning when we're not using both options together.
+    runTest({}, false);
+    runTest(
+        {sslAllowInvalidCertificates: '', setParameter: 'authenticationMechanisms=SCRAM-SHA-1'},
+        false);
+    runTest({setParameter: 'authenticationMechanisms=MONGODB-X509'}, false);
+    runTest({clusterAuthMode: 'x509'}, false);
+
+    // Do expect a warning when we're combining options.
+    runTest(
+        {sslAllowInvalidCertificates: '', setParameter: 'authenticationMechanisms=MONGODB-X509'},
+        true);
+    runTest({sslAllowInvalidCertificates: '', clusterAuthMode: 'x509'}, true);
+})();

src/mongo/db/commands/authentication_commands.cpp

@@ -162,6 +162,8 @@ bool CmdAuthenticate::run(OperationContext* txn,
     }
 
     UserName user(cmdObj.getStringField("user"), dbname);
+    uassert(ErrorCodes::AuthenticationFailed, "No user name provided", !user.getUser().empty());
+
     if (Command::testCommandsEnabled && user.getDB() == "admin" &&
         user.getUser() == internalSecurity.user->getName().getUser()) {
         // Allows authenticating as the internal user against the admin database.  This is to
@@ -315,6 +317,9 @@ Status CmdAuthenticate::_authenticateX509(OperationContext* txn,
     ClientBasic* client = ClientBasic::getCurrent();
     AuthorizationSession* authorizationSession = AuthorizationSession::get(client);
     std::string clientSubjectName = client->port()->getX509SubjectName();
+    uassert(ErrorCodes::AuthenticationFailed,
+            "No verified subject name available from client",
+            !clientSubjectName.empty());
 
     if (!getSSLManager()->getSSLConfiguration().hasCA) {
         return Status(ErrorCodes::AuthenticationFailed,

src/mongo/db/db.cpp

@@ -47,6 +47,7 @@
 #include "mongo/db/auth/auth_index_d.h"
 #include "mongo/db/auth/authorization_manager.h"
 #include "mongo/db/auth/authorization_manager_global.h"
+#include "mongo/db/auth/sasl_options.h"
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/catalog/collection_catalog_entry.h"
 #include "mongo/db/catalog/database.h"
@@ -120,6 +121,7 @@
 #include "mongo/util/quick_exit.h"
 #include "mongo/util/ramlog.h"
 #include "mongo/util/scopeguard.h"
+#include "mongo/util/sequence_util.h"
 #include "mongo/util/signal_handlers.h"
 #include "mongo/util/stacktrace.h"
 #include "mongo/util/startup_test.h"
@@ -128,6 +130,10 @@
 #include "mongo/util/time_support.h"
 #include "mongo/util/version.h"
 
+#ifdef MONGO_CONFIG_SSL
+#include "mongo/util/net/ssl_options.h"
+#endif
+
 #if !defined(_WIN32)
 #include <sys/file.h>
 #endif
@@ -645,6 +651,18 @@ void _initAndListen(int listenPort) {
 
     logMongodStartupWarnings(storageGlobalParams, serverGlobalParams);
 
+#if MONGO_CONFIG_SSL
+    if (sslGlobalParams.sslAllowInvalidCertificates &&
+        ((serverGlobalParams.clusterAuthMode.load() == ServerGlobalParams::ClusterAuthMode_x509) ||
+         sequenceContains(saslGlobalParams.authenticationMechanisms, "MONGODB-X509"))) {
+        log() << "** WARNING: While invalid X509 certificates may be used to" << startupWarningsLog;
+        log() << "**          connect to this server, they will not be considered"
+              << startupWarningsLog;
+        log() << "**          permissible for authentication." << startupWarningsLog;
+        log() << startupWarningsLog;
+    }
+#endif
+
     {
         stringstream ss;
         ss << endl;

src/mongo/db/matcher/expression_leaf.cpp

@@ -551,7 +551,10 @@ Status ArrayFilterEntries::setEqualities(std::vector<BSONElement> equalities) {
         }
     }
 
-    _equalities = BSONElementFlatSet(equalities.begin(), equalities.end());
+    // Sort the list of equalities to work around https://svn.boost.org/trac10/ticket/13140.
+    std::sort(equalities.begin(), equalities.end(), BSONElementCmpWithoutField());
+    _equalities =
+        BSONElementFlatSet(equalities.begin(), equalities.end(), BSONElementCmpWithoutField());
     return Status::OK();
 }
 

src/mongo/db/op_observer.cpp

@@ -25,6 +25,7 @@
 *    exception statement from all source files in the program, then also delete
 *    it in the license file.
 */
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kReplication
 
 #include "mongo/platform/basic.h"
 
@@ -36,16 +37,20 @@
 #include "mongo/db/dbdirectclient.h"
 #include "mongo/db/service_context.h"
 #include "mongo/db/namespace_string.h"
+#include "mongo/db/operation_context.h"
 #include "mongo/db/repl/oplog.h"
 #include "mongo/db/repl/replication_coordinator_global.h"
 #include "mongo/s/d_state.h"
 #include "mongo/scripting/engine.h"
-#include "mongo/db/operation_context.h"
+#include "mongo/util/fail_point_service.h"
+#include "mongo/util/log.h"
 
 namespace mongo {
 
 using std::vector;
 
+MONGO_FP_DECLARE(hangOnInsertObserver);
+
 void OpObserver::onCreateIndex(OperationContext* txn,
                                const std::string& ns,
                                BSONObj indexDoc,
@@ -64,6 +69,16 @@ void OpObserver::onInserts(OperationContext* txn,
                            bool fromMigrate) {
     repl::logOps(txn, "i", nss, begin, end, fromMigrate);
 
+    if (MONGO_FAIL_POINT(hangOnInsertObserver)) {
+        // This log output is used in js tests so please leave it.
+        log() << "op observer - hangOnInsertObserver fail point enabled. Blocking until fail point "
+                 "is disabled.";
+        while (MONGO_FAIL_POINT(hangOnInsertObserver)) {
+            mongo::sleepsecs(1);
+            txn->checkForInterrupt();
+        }
+    }
+
     const char* ns = nss.ns().c_str();
     for (auto it = begin; it != end; it++) {
         getGlobalAuthorizationManager()->logOp(txn, "i", ns, *it, nullptr);

src/mongo/db/query/query_planner.cpp

@@ -621,7 +621,7 @@ Status QueryPlanner::plan(const CanonicalQuery& query,
 
                     // Now we have the final min and max. This index is only relevant for
                     // the min/max query if min < max.
-                    if (0 >= finishedMinObj.woCompare(finishedMaxObj, kp, false)) {
+                    if (0 > finishedMinObj.woCompare(finishedMaxObj, kp, false)) {
                         // Found a relevant index.
                         idxNo = i;
                         break;

src/mongo/db/query/query_planner_common.cpp

@@ -61,8 +61,8 @@ void QueryPlannerCommon::reverseScans(QuerySolutionNode* node) {
         }
 
         if (!isn->bounds.isValidFor(isn->indexKeyPattern, isn->direction)) {
-            LOG(5) << "Invalid bounds: " << isn->bounds.toString() << std::endl;
-            invariant(0);
+            severe() << "Invalid bounds: " << isn->bounds.toString();
+            MONGO_UNREACHABLE;
         }
 
         // TODO: we can just negate every value in the already computed properties.

src/mongo/db/query/query_planner_test.cpp

@@ -1053,12 +1053,7 @@ TEST_F(QueryPlannerTest, MaxValid) {
 
 TEST_F(QueryPlannerTest, MinMaxSameValue) {
     addIndex(BSON("a" << 1));
-    runQueryHintMinMax(BSONObj(), BSONObj(), fromjson("{a: 1}"), fromjson("{a: 1}"));
-
-    assertNumSolutions(1U);
-    assertSolutionExists(
-        "{fetch: {filter: null, "
-        "node: {ixscan: {filter: null, pattern: {a: 1}}}}}");
+    runInvalidQueryHintMinMax(BSONObj(), BSONObj(), fromjson("{a: 1}"), fromjson("{a: 1}"));
 }
 
 TEST_F(QueryPlannerTest, MaxWithoutIndex) {

src/mongo/db/repl/oplogreader.h

@@ -84,8 +84,10 @@ public:
     DBClientConnection* conn() {
         return _conn.get();
     }
-    BSONObj findOne(const char* ns, const Query& q) {
-        return conn()->findOne(ns, q, 0, QueryOption_SlaveOk);
+    BSONObj findOne(const char* ns,
+                    const Query& q,
+                    QueryOptions options = static_cast<QueryOptions>(0)) {
+        return conn()->findOne(ns, q, 0, QueryOption_SlaveOk | options);
     }
     BSONObj getLastOp(const std::string& ns) {
         return findOne(ns.c_str(), Query().sort(reverseNaturalObj));

src/mongo/db/repl/replication_info.cpp

@@ -46,6 +46,7 @@
 #include "mongo/db/repl/replication_coordinator_global.h"
 #include "mongo/db/storage/storage_options.h"
 #include "mongo/db/wire_version.h"
+#include "mongo/rpc/metadata/client_metadata_naive.h"
 #include "mongo/s/write_ops/batched_command_request.h"
 
 namespace mongo {
@@ -225,6 +226,8 @@ public:
         if (cmdObj["forShell"].trueValue())
             LastError::get(txn->getClient()).disable();
 
+        logClientMetadata(txn->getClient(), cmdObj);
+
         appendReplicationInfo(txn, result, 0);
 
         if (serverGlobalParams.configsvrMode == CatalogManager::ConfigServerMode::CSRS) {

src/mongo/db/repl/rs_initialsync.cpp

@@ -72,6 +72,7 @@ MONGO_FP_DECLARE(failInitSyncWithBufferedEntriesLeft);
 // Failpoint which causes the initial sync function to hang before copying databases.
 MONGO_FP_DECLARE(initialSyncHangBeforeCopyingDatabases);
 
+MONGO_FP_DECLARE(initialSyncHangBeforeOplogVisibility);
 
 /**
  * Truncates the oplog (removes any documents) and resets internal variables that were
@@ -344,6 +345,39 @@ Status _initialSync() {
         sleepsecs(15);
         return Status(ErrorCodes::InitialSyncFailure, msg);
     }
+    OpTime firstLastOpTime = fassertStatusOK(40420, OpTime::parseFromOplogEntry(lastOp));
+
+    log() << "initial sync ensuring correct oplog visibility. Starting at "
+          << firstLastOpTime.toBSON();
+
+    if (MONGO_FAIL_POINT(initialSyncHangBeforeOplogVisibility)) {
+        // This log output is used in js tests so please leave it.
+        log() << "initial sync - initialSyncHangBeforeOplogVisibility fail point "
+                 "enabled. Blocking until fail point is disabled.";
+        while (MONGO_FAIL_POINT(initialSyncHangBeforeOplogVisibility) && !inShutdown()) {
+            mongo::sleepsecs(1);
+        }
+        log() << "initial sync - initialSyncHangBeforeOplogVisibility fail point disabled.";
+    }
+
+    // We do a forward scan starting at the lastOp we just fetched to ensure that all operations
+    // with earlier OpTimes are committed. This will block until all earlier oplog entries are
+    // visible and should either throw a socket exception or return lastOp.
+    BSONObjBuilder gte;
+    gte.append("$gte", firstLastOpTime.getTimestamp());
+    BSONObjBuilder queryBob;
+    queryBob.append("ts", gte.done());
+    const BSONObj& query = queryBob.done();
+    BSONObj lastOpConfirm = r.findOne(rsOplogName.c_str(), query, QueryOption_OplogReplay);
+    invariant(!lastOpConfirm.isEmpty());
+
+    OpTime lastOpConfirmTime = fassertStatusOK(40421, OpTime::parseFromOplogEntry(lastOpConfirm));
+    if (lastOpConfirmTime != firstLastOpTime) {
+        return Status(ErrorCodes::InitialSyncFailure,
+                      str::stream()
+                          << "Last op was not confirmed. last op: " << firstLastOpTime.toBSON()
+                          << ". confirmation: " << lastOpConfirmTime.toBSON());
+    }
 
     log() << "initial sync drop all databases";
     dropAllDatabasesExceptLocal(&txn);

src/mongo/dbtests/basictests.cpp

@@ -217,73 +217,6 @@ public:
 };
 }
 
-class sleeptest {
-public:
-    void run() {
-        Timer t;
-        int matches = 0;
-        for (int p = 0; p < 3; p++) {
-            sleepsecs(1);
-            int sec = (t.millis() + 2) / 1000;
-            if (sec == 1)
-                matches++;
-            else
-                mongo::unittest::log() << "temp millis: " << t.millis() << endl;
-            ASSERT(sec >= 0 && sec <= 2);
-            t.reset();
-        }
-        if (matches < 2)
-            mongo::unittest::log() << "matches:" << matches << endl;
-        ASSERT(matches >= 2);
-
-        sleepmicros(1527123);
-        ASSERT(t.micros() > 1000000);
-        ASSERT(t.micros() < 2000000);
-
-        t.reset();
-        sleepmillis(1727);
-        ASSERT(t.millis() >= 1000);
-        ASSERT(t.millis() <= 2500);
-
-        {
-            int total = 1200;
-            int ms = 2;
-            t.reset();
-            for (int i = 0; i < (total / ms); i++) {
-                sleepmillis(ms);
-            }
-            {
-                int x = t.millis();
-                if (x < 1000 || x > 2500) {
-                    cout << "sleeptest finds sleep accuracy to be not great. x: " << x << endl;
-                    ASSERT(x >= 1000);
-                    ASSERT(x <= 20000);
-                }
-            }
-        }
-
-#ifdef __linux__
-        {
-            int total = 1200;
-            int micros = 100;
-            t.reset();
-            int numSleeps = 1000 * (total / micros);
-            for (int i = 0; i < numSleeps; i++) {
-                sleepmicros(micros);
-            }
-            {
-                int y = t.millis();
-                if (y < 1000 || y > 2500) {
-                    cout << "sleeptest y: " << y << endl;
-                    ASSERT(y >= 1000);
-                    /* ASSERT( y <= 100000 ); */
-                }
-            }
-        }
-#endif
-    }
-};
-
 class SleepBackoffTest {
 public:
     void run() {
@@ -494,7 +427,6 @@ public:
         add<stringbuildertests::reset1>();
         add<stringbuildertests::reset2>();
 
-        add<sleeptest>();
         add<SleepBackoffTest>();
         add<AssertTests>();
 

src/mongo/rpc/SConscript

@@ -139,6 +139,7 @@ env.Library(
     source=[
         'metadata.cpp',
         'metadata/audit_metadata.cpp',
+        'metadata/client_metadata_naive.cpp',
         'metadata/config_server_metadata.cpp',
         'metadata/server_selection_metadata.cpp',
         'metadata/sharding_metadata.cpp',
@@ -150,6 +151,7 @@ env.Library(
         '$BUILD_DIR/mongo/client/read_preference',
         '$BUILD_DIR/mongo/db/repl/optime',
         '$BUILD_DIR/mongo/util/decorable',
+        '$BUILD_DIR/mongo/util/net/hostandport',
     ],
 )
 

src/mongo/rpc/metadata/client_metadata_naive.cpp

@@ -0,0 +1,47 @@
+/**
+ * Copyright (C) 2018 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or  modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kNetwork
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/rpc/metadata/client_metadata_naive.h"
+
+#include "mongo/util/log.h"
+
+namespace mongo {
+
+void logClientMetadata(const Client* const client, const BSONObj& isMasterCmdObj) {
+    BSONElement elem = isMasterCmdObj[kMetadataDocumentName];
+    if (!elem.eoo()) {
+        log() << "received client metadata from " << client->getRemote().toString() << " "
+              << client->desc() << ": " << elem.Obj();
+    }
+}
+
+}  // namespace mongo

src/mongo/rpc/metadata/client_metadata_naive.h

@@ -0,0 +1,43 @@
+/**
+ * Copyright (C) 2018 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or  modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/bson/bsonobj.h"
+#include "mongo/db/client.h"
+
+namespace mongo {
+
+const char kMetadataDocumentName[] = "client";
+
+/**
+ * Log client and client metadata information to disk.
+ */
+void logClientMetadata(const Client*, const BSONObj& isMasterCmdObj);
+
+}  // namespace mongo

src/mongo/s/commands/cluster_is_master_cmd.cpp

@@ -29,7 +29,9 @@
 #include "mongo/platform/basic.h"
 
 #include "mongo/db/commands.h"
+#include "mongo/db/operation_context.h"
 #include "mongo/db/wire_version.h"
+#include "mongo/rpc/metadata/client_metadata_naive.h"
 #include "mongo/s/catalog/forwarding_catalog_manager.h"
 #include "mongo/s/grid.h"
 #include "mongo/s/write_ops/batched_command_request.h"
@@ -65,6 +67,8 @@ public:
                      int options,
                      std::string& errmsg,
                      BSONObjBuilder& result) {
+        logClientMetadata(txn->getClient(), cmdObj);
+
         result.appendBool("ismaster", true);
         result.append("msg", "isdbgrid");
         result.appendNumber("maxBsonObjectSize", BSONObjMaxUserSize);

src/mongo/util/exception_filter_win32.cpp

@@ -129,8 +129,8 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) {
               sizeof(addressString),
               "0x%p",
               excPointers->ExceptionRecord->ExceptionAddress);
-    log() << "*** unhandled exception " << exceptionString << " at " << addressString
-          << ", terminating" << std::endl;
+    severe() << "*** unhandled exception " << exceptionString << " at " << addressString
+             << ", terminating" << std::endl;
     if (excPointers->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
         ULONG acType = excPointers->ExceptionRecord->ExceptionInformation[0];
         const char* acTypeString;
@@ -152,10 +152,10 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) {
                   sizeof(addressString),
                   " 0x%p",
                   excPointers->ExceptionRecord->ExceptionInformation[1]);
-        log() << "*** access violation was a " << acTypeString << addressString << std::endl;
+        severe() << "*** access violation was a " << acTypeString << addressString << std::endl;
     }
 
-    log() << "*** stack trace for unhandled exception:" << std::endl;
+    severe() << "*** stack trace for unhandled exception:" << std::endl;
 
     // Create a copy of context record because printWindowsStackTrace will mutate it.
     CONTEXT contextCopy(*(excPointers->ContextRecord));
@@ -165,7 +165,7 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) {
     doMinidumpWithException(excPointers);
 
     // Don't go through normal shutdown procedure. It may make things worse.
-    log() << "*** immediate exit due to unhandled exception" << std::endl;
+    severe() << "*** immediate exit due to unhandled exception" << std::endl;
     quickExit(EXIT_ABRUPT);
 
     // We won't reach here

src/mongo/util/net/ssl_manager.cpp

@@ -1152,6 +1152,7 @@ StatusWith<boost::optional<std::string>> SSLManager::parseAndValidatePeerCertifi
         if (_allowInvalidCertificates) {
             warning() << "SSL peer certificate validation failed: "
                       << X509_verify_cert_error_string(result);
+            return {boost::none};
         } else {
             str::stream msg;
             msg << "SSL peer certificate validation failed: "

src/third_party/wiredtiger/dist/s_string.ok

@@ -143,6 +143,7 @@ Fprintf
 FreeBSD
 FreeBSD's
 FreeLibrary
+Fsync
 Fuerst
 GCC
 GIDs
@@ -207,6 +208,7 @@ LSM
 LSN
 LSNs
 LTE
+LWN
 LZ
 LZO
 LeafGreen
@@ -285,6 +287,7 @@ Pandis
 Phong
 PlatformSDK
 Posix
+PostgreSQL
 PowerPC
 Pre
 Preload

src/third_party/wiredtiger/import.data

@@ -1,5 +1,5 @@
 {
-    "commit": "4babae8093f7a3cb05226abe959e10e0bb6b2716", 
+    "commit": "264bc732b0b25f84e4c3af70220d3cedb34bc551", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-3.2"

src/third_party/wiredtiger/src/btree/bt_split.c

@@ -462,6 +462,9 @@ __split_ref_prepare(WT_SESSION_IMPL *session,
 	 * ascend into the created children, but eventually fail as that parent
 	 * page won't yet know about the created children pages. That's OK, we
 	 * spin there until the parent's page index is updated.
+	 *
+	 * Lock the newly created page to ensure it doesn't split until all
+	 * child pages have been updated.
 	 */
 	for (i = skip_first ? 1 : 0; i < pindex->entries; ++i) {
 		ref = pindex->index[i];
@@ -495,12 +498,14 @@ __split_ref_prepare(WT_SESSION_IMPL *session,
 		 * reading the child's page index structure is safe.
 		 */
 		j = 0;
+		WT_PAGE_LOCK(session, child);
 		WT_ENTER_PAGE_INDEX(session);
 		WT_INTL_FOREACH_BEGIN(session, child, child_ref) {
 			child_ref->home = child;
 			child_ref->pindex_hint = j++;
 		} WT_INTL_FOREACH_END;
 		WT_LEAVE_PAGE_INDEX(session);
+		WT_PAGE_UNLOCK(session, child);
 
 #ifdef HAVE_DIAGNOSTIC
 		WT_WITH_PAGE_INDEX(session,

src/third_party/wiredtiger/src/os_posix/os_fs.c

@@ -31,6 +31,23 @@
 /*
  * __posix_sync --
  *	Underlying support function to flush a file descriptor.
+ *
+ * Fsync calls (or fsync-style calls, for example, fdatasync) are not retried
+ * on failure, and failure halts the system.
+ *
+ * Excerpted from the LWN.net article https://lwn.net/Articles/752063/:
+ * In short, PostgreSQL assumes that a successful call to fsync() indicates
+ * that all data written since the last successful call made it safely to
+ * persistent storage. But that is not what the kernel actually does. When
+ * a buffered I/O write fails due to a hardware-level error, filesystems
+ * will respond differently, but that behavior usually includes discarding
+ * the data in the affected pages and marking them as being clean. So a read
+ * of the blocks that were just written will likely return something other
+ * than the data that was written.
+ *
+ * Given the shared history of UNIX filesystems, and the difficulty of knowing
+ * what specific error will be returned under specific circumstances, we don't
+ * retry fsync-style calls and panic if a flush operation fails.
  */
 static int
 __posix_sync(
@@ -52,25 +69,49 @@ __posix_sync(
 	 * OS X F_FULLFSYNC fcntl documentation:
 	 * "This is currently implemented on HFS, MS-DOS (FAT), and Universal
 	 * Disk Format (UDF) file systems."
+	 *
+	 * See comment in __posix_sync(): sync cannot be retried or fail.
 	 */
-	WT_SYSCALL_RETRY(fcntl(fd, F_FULLFSYNC, 0) == -1 ? -1 : 0, ret);
-	if (ret == 0)
-		return (0);
-	/*
-	 * Assume F_FULLFSYNC failed because the file system doesn't support it
-	 * and fallback to fsync.
-	 */
+	static enum { FF_NOTSET, FF_IGNORE, FF_OK } ff_status = FF_NOTSET;
+	switch (ff_status) {
+	case FF_NOTSET:
+		WT_SYSCALL(fcntl(fd, F_FULLFSYNC, 0) == -1 ? -1 : 0, ret);
+		if (ret == 0) {
+			ff_status = FF_OK;
+			return (0);
+		}
+
+		/*
+		 * If the first F_FULLFSYNC fails, assume the file system
+		 * doesn't support it and fallback to fdatasync or fsync.
+		 */
+		ff_status = FF_IGNORE;
+		__wt_err(session, ret,
+		    "fcntl(F_FULLFSYNC) failed, falling back to fdatasync "
+		    "or fsync");
+		break;
+	case FF_IGNORE:
+		break;
+	case FF_OK:
+		WT_SYSCALL(fcntl(fd, F_FULLFSYNC, 0) == -1 ? -1 : 0, ret);
+		if (ret == 0)
+			return (0);
+		WT_PANIC_RET(session,
+		    ret, "%s: %s: fcntl(F_FULLFSYNC)", name, func);
+	}
 #endif
 #if defined(HAVE_FDATASYNC)
-	WT_SYSCALL_RETRY(fdatasync(fd), ret);
+	/* See comment in __posix_sync(): sync cannot be retried or fail. */
+	WT_SYSCALL(fdatasync(fd), ret);
 	if (ret == 0)
 		return (0);
-	WT_RET_MSG(session, ret, "%s: %s: fdatasync", name, func);
+	WT_PANIC_RET(session, ret, "%s: %s: fdatasync", name, func);
 #else
-	WT_SYSCALL_RETRY(fsync(fd), ret);
+	/* See comment in __posix_sync(): sync cannot be retried or fail. */
+	WT_SYSCALL(fsync(fd), ret);
 	if (ret == 0)
 		return (0);
-	WT_RET_MSG(session, ret, "%s: %s: fsync", name, func);
+	WT_PANIC_RET(session, ret, "%s: %s: fsync", name, func);
 #endif
 }
 
@@ -108,12 +149,15 @@ __posix_directory_sync(WT_SESSION_IMPL *session, const char *path)
 	WT_SYSCALL(close(fd), tret);
 	if (tret != 0) {
 		__wt_err(session, tret, "%s: directory-sync: close", dir);
-		if (ret == 0)
-			ret = tret;
+		WT_TRET(tret);
 	}
 
 err:	__wt_scr_free(session, &tmp);
-	return (ret);
+	if (ret == 0)
+		return (ret);
+
+	/* See comment in __posix_sync(): sync cannot be retried or fail. */
+	WT_PANIC_RET(session, ret, "%s: directory-sync", path);
 }
 #endif
 
@@ -460,11 +504,13 @@ __posix_file_sync_nowait(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session)
 	session = (WT_SESSION_IMPL *)wt_session;
 	pfh = (WT_FILE_HANDLE_POSIX *)file_handle;
 
-	WT_SYSCALL_RETRY(sync_file_range(pfh->fd,
+	/* See comment in __posix_sync(): sync cannot be retried or fail. */
+	WT_SYSCALL(sync_file_range(pfh->fd,
 	    (off64_t)0, (off64_t)0, SYNC_FILE_RANGE_WRITE), ret);
 	if (ret == 0)
 		return (0);
-	WT_RET_MSG(session, ret,
+
+	WT_PANIC_RET(session, ret,
 	    "%s: handle-sync-nowait: sync_file_range", file_handle->name);
 }
 #endif

src/third_party/wiredtiger/src/reconcile/rec_write.c

@@ -388,6 +388,18 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref,
 	 */
 	WT_PAGE_LOCK(session, page);
 
+	/*
+	 * Now that the page is locked, if attempting to evict it, check again
+	 * whether eviction is permitted. The page's state could have changed
+	 * while we were waiting to acquire the lock (e.g., the page could have
+	 * split).
+	 */
+	if (LF_ISSET(WT_EVICTING) &&
+	    !__wt_page_can_evict(session, ref, NULL)) {
+		WT_PAGE_UNLOCK(session, page);
+		return (EBUSY);
+	}
+
 	oldest_id = __wt_txn_oldest_id(session);
 	if (LF_ISSET(WT_EVICTING))
 		mod->last_eviction_id = oldest_id;