Compare commits

...

6 Commits

Author SHA1 Message Date
Jon Streets
d5900efb03 Update the directory name with correct version 2021-02-04 11:38:21 -05:00
Jon Streets
25527deb17 Update version to 3.6.17-linux-splunk-v2 2021-02-01 12:42:01 -05:00
Jon Streets
87d6e14760 Update version to splunk-3.6.17-linux-v2 2021-01-29 14:47:32 -05:00
Jon Streets
d2cb50e0d5 Update version string to splunk1 2021-01-13 16:26:52 -05:00
Geert Bosch
5932ecc473 SERVER-52902 Remove assertion !haveJournalFiles()
(cherry picked from commit 80083d984b)
2021-01-13 16:23:45 -05:00
Brian McCarthy
40fff57cbb BUILD-10833 Splunk custom build of 3.6.17 with BACKPORT-6439 Linux 2021-01-11 17:04:48 -05:00
6 changed files with 187 additions and 307 deletions

43
README-SPLUNK-LINUX Normal file
View File

@@ -0,0 +1,43 @@
Splunk Custom Build Linux
=========================
NOTE: We use 3 separate branches to build and test builds for Splunk: Linux, Windows, and MacOS.
This is because signficant changes have to be made to build the tools on RHEL 5.5 Linux. We also need to make some code changes to account for features
that are not available in the older OS.
Finally, Splunk requires different versions of openssl (and a specific location requirement on macOS requiring its own distro in Evergreen)
This would be hard to generalize in a single evergreen.yml in one branch.
Current branches:
splunk-linux-v3.6.17-BACKPORT-6439
splunk-macos-v3.6.17-BACKPORT-6439
splunk-windows-v3.6.17-BACKPORT-6439
We build on RHEL 5.5
Use the "linux64" variant for patches:
```
evergreen patch -p "mongodb-mongo-v3.6" -t "compile" -d "BUILD-10833 Splunk custom build of 3.6.17 with BACKPORT-6439" -v "linux-64" -u -f -y
```
Here is the last full patch build
https://evergreen.mongodb.com/version/5e839c9c0305b97fc1bafa03
We expect the below tests to fail:
https://jira.mongodb.org/browse/BUILD-10428?focusedCommentId=2947636&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-2947636
jstestfuzz_*
multiverson_*
sharding_last_stable_mongos_and_mixed_shards
compile_all
serial_run_WT
In the near future, we will probably change the way we build and test these Linux versions per:
https://jira.mongodb.org/browse/WRITING-6736
https://docs.google.com/document/d/1kDfQ_1k0OHTueH_kleOF17G7ZWvECde_IvCLLM2f47g/edit#

View File

@@ -90,6 +90,8 @@ variables:
# This script converts the generated version string into a sanitized version string for
# use by scons and uploading artifacts as well as information about for the scons cache.
${activate_virtualenv}
### SPLUNK Since we are using a virtualenv from the toolchain now, we need pyyaml
pip install pyyaml
MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} USE_SCONS_CACHE=${use_scons_cache|false} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
- &apply_compile_expansions
@@ -257,7 +259,9 @@ functions:
${activate_virtualenv}
bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]')
# Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();"
mongo_ver=$($mongo_binary --version | perl -pe '/version v(.*)$/; $_ = $1;' | tr -d '[ \r\n]')
### SPLUNK
### mongo_ver=$($mongo_binary --version | perl -pe '/version v(.*)$/; $_ = $1;' | tr -d '[ \r\n]')
mongo_ver=$(LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/openssl-1.0.2/lib $mongo_binary --version | perl -pe '/version v(.*)$/; $_ = $1;' | tr -d '[ \r\n]')
# The versions must match
if [ "$bin_ver" != "$mongo_ver" ]; then
echo "The mongo version is $mongo_ver, expected version is $bin_ver"
@@ -527,21 +531,56 @@ functions:
set -o igncr
fi;
### SPLUNK
# We need pcap headers
rpm -ivh http://boxes.10gen.com/build/libpcap-devel-0.9.4-15.el5.x86_64.rpm
### SPLUNK
#sed -i.bak "s/built-without-version-string/3.6.17-SERVER-42525-splunk/" common/options/options.go
sed -i.bak "s/built-without-version-string/3.6.17-linux-splunk-v2/" common/options/options.go
sed -i.bak "s/built-without-git-spec/$(git rev-parse HEAD)/" common/options/options.go
# Move the vendor source to make compatible with gccgo
mkdir /data/go
mkdir /data/go/src
cp -r vendor/* /data/go/src
export GOROOT=/data/go
export PATH=/opt/mongodbtoolchain/v2/bin/:$PATH
### SPLUNK
# set_goenv provides set_goenv(), print_ldflags() and print_tags() used below
. ./set_goenv.sh
GOROOT="" set_goenv || exit
### SPLUNK
set_goenv || exit
env | grep ^GO
go version
### SPLUNK
# In RHEL 5.5, /usr/bin/ld can't handle --build-id parameters, so
# use a wrapper if it's present on the system
#
if [ -d /opt/ldwrapper/bin ]
then
export PATH=/opt/ldwrapper/bin:$PATH
fi
build_tools="bsondump mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop"
# SPLUNK - Don't build mongoreplay
if [ "${build_mongoreplay}" = "true" ]; then
build_tools="$build_tools mongoreplay"
fi
### SPLUNK
export CGO_CPPFLAGS=-I/tmp/openssl-1.0.2/include
export CGO_LDFLAGS="-L/tmp/openssl-1.0.2/lib -lssl -lcrypto"
for i in $build_tools; do
go build $(buildflags) -ldflags "$(print_ldflags)" ${args} -tags "$(print_tags ${tooltags})" -o "../../../../../../mongo-tools/$i${exe|}" $i/main/$i.go
"../../../../../../mongo-tools/$i${exe|}" --version
### SPLUNK
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/openssl-1.0.2/lib "../../../../../../mongo-tools/$i${exe|}" --version
file "../../../../../../mongo-tools/$i${exe|}"
done
@@ -570,10 +609,25 @@ functions:
name: perf
file: src/perf.json
### SPLUNK openssl 1.0.2
"install openssl 1.0.2" : &install_openssl
command: shell.exec
params:
working_dir: src
script: |
set -o errexit
set -o verbose
# if hostname | grep macos # MAC
if [ `cat /etc/redhat-release | awk '{print $7}'` == '5.7' ]
then
curl http://boxes.10gen.com/build/openssl-1.0.2l.tar.gz | tar -zxf- -C /tmp
fi
"do setup" :
- *fetch_artifacts
- *fetch_binaries
- *extract_binaries
- *install_openssl
- *check_binary_version
- *get_buildnumber
- *set_up_credentials
@@ -596,16 +650,24 @@ functions:
set -o errexit
set -o verbose
### SPLUNK
### Forget old python, it fails on RHEL 5.5
### Stick with the toolchain python 2
### There is no python3 in the RHEL 5.5 toolchain. It's only used for lint with mypy
python_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2})
python3_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python3})
### python3_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python3})
if [ "Windows_NT" = "$OS" ]; then
python_loc=$(cygpath -w $python_loc)
python3_loc=$(cygpath -w c:/python/Python36/python.exe)
fi
# Set up virtualenv in ${workdir}
virtualenv --python "$python_loc" --system-site-packages "${workdir}/venv"
# Add virtualenv for python3 in ${workdir}
virtualenv --python "$python3_loc" --system-site-packages "${workdir}/venv_3"
### SPLUNK
### virtualenv --python "$python_loc" --system-site-packages "${workdir}/venv"
virtualenv --python "$python_loc" "${workdir}/venv"
### SPLUNK, don't add this for RHEL 5.5
### Add virtualenv for python3 in ${workdir}
### SPLUNK - No python 3
### virtualenv --python "$python3_loc" --system-site-packages "${workdir}/venv_3"
"run tests" :
- command: expansions.update
@@ -723,6 +785,7 @@ functions:
extra_args="$extra_args --tagFile=etc/test_retrial.yml"
fi
### SPLUNK add ld_library_path
# The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py
# invocation. It doesn't set any environment variables and should therefore come last in
# this list of expansions.
@@ -734,6 +797,7 @@ functions:
${lang_environment} \
${san_options} \
${san_symbolizer} \
${ld_library_path} \
${snmp_config_path} \
${resmoke_wrapper} \
$python buildscripts/evergreen_run_tests.py \
@@ -791,6 +855,15 @@ functions:
rm -rf ${install_directory|/data/mongo-install-directory}
### SPLUNK
### Since we aren't building the tools, create some placeholders
### mkdir -p src/mongo-tools
### for i in mongodump mongorestore mongoexport mongoimport mongostat mongotop bsondump mongofiles mongooplog mongoreplay;do
### echo a > src/mongo-tools/$i
### done
extra_args=""
if [ "${targets}" = "all" ] && [ -n "${num_scons_compile_all_jobs_available|}" ]; then
echo "Changing SCons to run with --jobs=${num_scons_compile_all_jobs_available|}"
@@ -809,12 +882,15 @@ functions:
set -o verbose
# We get the raw version string (r1.2.3-45-gabcdef) from git
MONGO_VERSION=$(git describe)
### MONGO_VERSION=$(git describe)
### SPLUNK
MONGO_VERSION=3.6.17-linux-splunk-v2
# If this is a patch build, we add the patch version id to the version string so we know
# this build was a patch, and which evergreen task it came from
if [ "${is_patch|}" = "true" ]; then
MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
fi
### SPLUNK Get rid of the below block
### if [ "${is_patch|}" = "true" ]; then
### MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
### fi
# This script converts the generated version string into a sanitized version string for
# use by scons and uploading artifacts as well as information about for the scons cache.
@@ -1007,7 +1083,9 @@ functions:
${add_nodejs_to_path}
npm run ${npm_command|jstestfuzz} -- ${jstestfuzz_vars} --branch ${branch_name}
### SPLUNK - Fix up the npm runs
### npm run ${npm_command|jstestfuzz} -- --jsTestsDir ../jstests ${jstestfuzz_vars}
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/openssl-1.0.2/lib npm run ${npm_command|jstestfuzz} -- --jsTestsDir ../jstests ${jstestfuzz_vars}
- command: archive.targz_pack
params:
@@ -1211,6 +1289,10 @@ functions:
set -o verbose
rm -rf src /data/db/* mongo-diskstats* mongo-*.tgz ~/.aws ~/.boto venv
### SPLUNK
### Remove openssl 1.0.2 from /tmp when finished
echo "deleting openssl openssl 1.0.2"
rm -rf /tmp/openssl-1.0.2
"kill processes" :
command: shell.exec
@@ -2009,6 +2091,13 @@ pre:
python=${python|/opt/mongodbtoolchain/v2/bin/python2}
fi
echo "python set to $(which $python)"
### pip install pyyaml Cheetah typing requests pymongo urllib3
### SPLUNK we need to lock down some versions
pip install argparse==1.4.0 certifi==2018.1.18 chardet==3.0.4 \
cpplint==1.3.0 enum34==1.1.6 idna==2.6 poster==0.8.1 \
psutil==5.4.3 pycrypto==2.6.1 PyKMIP==0.4.0 pymongo==3.5.1 PyYAML==3.11 \
requests==2.18.4 simples3==1.0 six==1.11.0 urllib3==1.22 Cheetah typing
- key: activate_virtualenv_3
value: |
# check if virtualenv for python3 is set up
@@ -2511,6 +2600,8 @@ tasks:
- func: "get buildnumber"
- func: "set up credentials"
- func: "fetch and build OpenSSL"
### SPLUNK Install openssl
- func: "install openssl 1.0.2"
- func: "build new tools" # noop if ${newtools} is not "true"
- func: "generate compile expansions"
# Then we load the generated version data into the agent so we can use it in task definitions
@@ -2675,6 +2766,8 @@ tasks:
- func: "get buildnumber"
- func: "set up credentials"
- func: "fetch and build OpenSSL"
### SPLUNK Install openssl 1.0.2
- func: "install openssl 1.0.2"
- func: "build new tools" # noop if ${newtools} is not "true"
- func: "generate compile expansions"
# Then we load the generated version data into the agent so we can use it in task definitions.
@@ -5945,28 +6038,38 @@ buildvariants:
# Linux buildvariants #
###########################################
### SPLUNK
- name: linux-64
display_name: Linux
display_name: Linux Splunk
run_on:
- rhel62-small
### - rhel62-small
### SPLUNK
- rhel55-test
batchtime: 1440 # 1 day
expansions:
push_path: linux
push_bucket: downloads.mongodb.org
push_name: linux
push_arch: x86_64
compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars --release
### SPLUNK
### compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars --release
ld_library_path: LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp/openssl-1.0.2/lib
compile_flags: --ssl -j$(grep -c ^processor /proc/cpuinfo) LINKFLAGS=-static-libgcc --variables-files=etc/scons/mongodbtoolchain_gcc.vars --release
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
use_scons_cache: true
tooltags: ""
build_mongoreplay: true
tooltags: "ssl"
build_mongoreplay: false
tasks:
- name: compile
distros:
- rhel62-large
### SPLUNK
### - rhel62-large
- rhel55-build
- name: compile_all
distros:
- rhel62-large
### SPLUNK
### - rhel62-large
- rhel55-build
- name: aggregation_WT
- name: aggregation_auth
- name: aggregation_read_concern_majority_passthrough_WT

View File

@@ -20,3 +20,5 @@ ENV = {
CC = '/opt/mongodbtoolchain/v2/bin/gcc'
CXX = '/opt/mongodbtoolchain/v2/bin/g++'
OBJCOPY = '/opt/mongodbtoolchain/v2/bin/objcopy'
LIBPATH = '/tmp/openssl-1.0.2/lib'
CPPPATH = '/tmp/openssl-1.0.2/include'

View File

@@ -262,7 +262,6 @@ void removeJournalFiles() {
log() << "error removing journal files " << e.what() << endl;
throw;
}
verify(!haveJournalFiles());
flushMyDirectory(getJournalDir() /
"file"); // flushes parent of argument (in this case journal dir)

View File

@@ -111,7 +111,8 @@ buildflags() {
UNAME_S=$(PATH="/usr/bin:/bin" uname -s)
case $UNAME_S in
Linux)
flags="-buildmode=pie"
# SPLUNK - gcc go does not have support for buildmode
#flags="-buildmode=pie"
;;
esac
echo "$flags"

View File

@@ -32,320 +32,52 @@
#error Do not include the DNS Query platform implementation headers. Please use "mongo/util/dns_query.h" instead.
#endif
// DNS Headers for POSIX/libresolv have to be included in a specific order
// clang-format off
#include <sys/types.h>
#include <netinet/in.h>
#include <arpa/nameser.h>
#include <resolv.h>
// clang-format on
#include <stdio.h>
#include <iostream>
#include <cassert>
#include <sstream>
#include <string>
#include <cstdint>
#include <vector>
#include <array>
#include <stdexcept>
#include <memory>
#include <exception>
#include <string>
#include <vector>
#include <boost/noncopyable.hpp>
#include "mongo/util/assert_util.h"
namespace mongo {
namespace dns {
// The anonymous namespace is safe, in this header, as it is not really a header. It is only used
// in the `dns_query.cpp` TU.
namespace {
using std::begin;
using std::end;
using namespace std::literals::string_literals;
enum class DNSQueryClass { kInternet };
const std::size_t kMaxExpectedDNSResponseSize = 65536;
const std::size_t kMaxSRVHostNameSize = 8192;
enum class DNSQueryType { kSRV, kTXT, kAddress };
enum class DNSQueryClass {
kInternet = ns_c_in,
};
[[noreturn]] void throwNotSupported() {
uasserted(ErrorCodes::BadValue, "srv_nsearch not supported on android");
}
enum class DNSQueryType {
kSRV = ns_t_srv,
kTXT = ns_t_txt,
kAddress = ns_t_a,
};
/**
* A `ResourceRecord` represents a single DNS entry as parsed by the resolver API.
* It can be viewed as one of various record types, using the member functions.
* It roughly corresponds to the DNS RR data structure
*/
class ResourceRecord {
public:
explicit ResourceRecord() = default;
explicit ResourceRecord(std::string initialService, ns_msg& ns_answer, const int initialPos)
: _service(std::move(initialService)),
_answerStart(ns_msg_base(ns_answer)),
_answerEnd(ns_msg_end(ns_answer)),
_pos(initialPos) {
if (ns_parserr(&ns_answer, ns_s_an, initialPos, &this->_resource_record))
this->_badRecord();
}
/**
* View this record as a DNS TXT record.
*/
std::vector<std::string> txtEntry() const {
const auto data = this->_rawData();
if (data.empty()) {
uasserted(ErrorCodes::DNSProtocolError, "DNS TXT Record is not correctly sized");
}
const std::size_t amount = data.front();
const auto first = begin(data) + 1;
std::vector<std::string> rv;
if (data.size() - 1 < amount) {
uasserted(ErrorCodes::DNSProtocolError, "DNS TXT Record is not correctly sized");
}
rv.emplace_back(first, first + amount);
return rv;
throwNotSupported();
}
/**
* View this record as a DNS A record.
*/
std::string addressEntry() const {
std::string rv;
auto data = _rawData();
if (data.size() != 4) {
uasserted(ErrorCodes::DNSProtocolError, "DNS A Record is not correctly sized");
}
for (const std::uint8_t& ch : data) {
std::ostringstream oss;
oss << int(ch);
rv += oss.str() + ".";
}
rv.pop_back();
return rv;
throwNotSupported();
}
/**
* View this record as a DNS SRV record.
*/
SRVHostEntry srvHostEntry() const {
const std::size_t kPortOffsetInPacket = 4;
const std::uint8_t* const data = ns_rr_rdata(this->_resource_record);
if (data < this->_answerStart ||
data + kPortOffsetInPacket + sizeof(std::uint16_t) > this->_answerEnd) {
std::ostringstream oss;
oss << "Invalid record " << this->_pos << " of SRV answer for \"" << this->_service
<< "\": Incorrect result size";
uasserted(ErrorCodes::DNSProtocolError, oss.str());
}
const std::uint16_t port = [data] {
std::uint16_t tmp;
memcpy(&tmp, data + kPortOffsetInPacket, sizeof(tmp));
return ntohs(tmp);
}();
// The '@' is an impermissible character in a host name, so we populate the string we'll
// return with it, such that a failure in string manipulation or corrupted dns packets will
// cause an illegal hostname.
std::string name(kMaxSRVHostNameSize, '@');
const auto size = dn_expand(this->_answerStart,
this->_answerEnd,
data + kPortOffsetInPacket + sizeof(port),
&name[0],
name.size());
if (size < 1)
this->_badRecord();
// Trim the expanded name
name.resize(name.find('\0'));
name += '.';
// return by copy is equivalent to a `shrink_to_fit` and `move`.
return {name, port};
throwNotSupported();
}
private:
void _badRecord() const {
std::ostringstream oss;
oss << "Invalid record " << this->_pos << " of DNS answer for \"" << this->_service
<< "\": \"" << strerror(errno) << "\"";
uasserted(ErrorCodes::DNSProtocolError, oss.str());
};
std::vector<std::uint8_t> _rawData() const {
const std::uint8_t* const data = ns_rr_rdata(this->_resource_record);
const std::size_t length = ns_rr_rdlen(this->_resource_record);
return {data, data + length};
}
std::string _service;
ns_rr _resource_record;
const std::uint8_t* _answerStart;
const std::uint8_t* _answerEnd;
int _pos;
};
/**
* The `DNSResponse` class represents a response to a DNS query.
* It has STL-compatible iterators to view individual DNS Resource Records within a response.
*/
class DNSResponse {
using DNSResponse = std::vector<ResourceRecord>;
class DNSQueryState {
public:
explicit DNSResponse(std::string initialService, std::vector<std::uint8_t> initialData)
: _service(std::move(initialService)), _data(std::move(initialData)) {
if (ns_initparse(this->_data.data(), this->_data.size(), &this->_ns_answer)) {
std::ostringstream oss;
oss << "Invalid SRV answer for \"" << this->_service << "\"";
uasserted(ErrorCodes::DNSProtocolError, oss.str());
}
this->_nRecords = ns_msg_count(this->_ns_answer, ns_s_an);
if (!this->_nRecords) {
std::ostringstream oss;
oss << "No SRV records for \"" << this->_service << "\"";
uasserted(ErrorCodes::DNSProtocolError, oss.str());
}
DNSResponse lookup(const std::string&, const DNSQueryClass, const DNSQueryType) {
throwNotSupported();
}
class iterator {
public:
auto makeRelopsLens() const {
return std::tie(this->_response, this->_pos);
}
inline friend bool operator==(const iterator& lhs, const iterator& rhs) {
return lhs.makeRelopsLens() == rhs.makeRelopsLens();
}
inline friend bool operator<(const iterator& lhs, const iterator& rhs) {
return lhs.makeRelopsLens() < rhs.makeRelopsLens();
}
inline friend bool operator!=(const iterator& lhs, const iterator& rhs) {
return !(lhs == rhs);
}
const ResourceRecord& operator*() {
this->_populate();
return this->_record;
}
const ResourceRecord* operator->() {
this->_populate();
return &this->_record;
}
iterator& operator++() {
this->_advance();
return *this;
}
iterator operator++(int) {
iterator tmp = *this;
this->_advance();
return tmp;
}
private:
friend DNSResponse;
explicit iterator(DNSResponse* const r)
: _response(r), _record(this->_response->_service, this->_response->_ns_answer, 0) {}
explicit iterator(DNSResponse* const initialResponse, const int initialPos)
: _response(initialResponse), _pos(initialPos) {}
void _populate() {
if (this->_ready) {
return;
}
this->_record =
ResourceRecord(this->_response->_service, this->_response->_ns_answer, this->_pos);
this->_ready = true;
}
void _advance() {
++this->_pos;
this->_ready = false;
}
DNSResponse* _response;
int _pos = 0;
ResourceRecord _record;
bool _ready = false;
};
auto begin() {
return iterator(this);
DNSQueryState() {
throwNotSupported();
}
auto end() {
return iterator(this, this->_nRecords);
}
std::size_t size() const {
return this->_nRecords;
}
private:
std::string _service;
std::vector<std::uint8_t> _data;
ns_msg _ns_answer;
std::size_t _nRecords;
};
/**
* The `DNSQueryState` object represents the state of a DNS query interface, on Unix-like systems.
*/
class DNSQueryState : boost::noncopyable {
public:
std::vector<std::uint8_t> raw_lookup(const std::string& service,
const DNSQueryClass class_,
const DNSQueryType type) {
std::vector<std::uint8_t> result(kMaxExpectedDNSResponseSize);
const int size = res_nsearch(
&_state, service.c_str(), int(class_), int(type), &result[0], result.size());
if (size < 0) {
std::ostringstream oss;
oss << "Failed to look up service \"" << service << "\": " << strerror(errno);
uasserted(ErrorCodes::DNSHostNotFound, oss.str());
}
result.resize(size);
return result;
}
DNSResponse lookup(const std::string& service,
const DNSQueryClass class_,
const DNSQueryType type) {
return DNSResponse(service, raw_lookup(service, class_, type));
}
public:
~DNSQueryState() {
res_nclose(&_state);
}
DNSQueryState() : _state() {
res_ninit(&_state);
}
private:
struct __res_state _state;
};
} // namespace
} // namespace dns