xuemingqiang/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update test/3rdparty with the packages required to run the test suite in parallel mode. Change the short command line flag to "-j", matching make.

Browse Source 
--HG--
rename : test/3rdparty/testscenarios-0.2/.bzrignore => test/3rdparty/testscenarios-0.4/.bzrignore
rename : test/3rdparty/testscenarios-0.2/Apache-2.0 => test/3rdparty/testscenarios-0.4/Apache-2.0
rename : test/3rdparty/testscenarios-0.2/BSD => test/3rdparty/testscenarios-0.4/BSD
rename : test/3rdparty/testscenarios-0.2/COPYING => test/3rdparty/testscenarios-0.4/COPYING
rename : test/3rdparty/testscenarios-0.2/GOALS => test/3rdparty/testscenarios-0.4/GOALS
rename : test/3rdparty/testscenarios-0.2/HACKING => test/3rdparty/testscenarios-0.4/HACKING
rename : test/3rdparty/testscenarios-0.2/MANIFEST.in => test/3rdparty/testscenarios-0.4/MANIFEST.in
rename : test/3rdparty/testscenarios-0.2/Makefile => test/3rdparty/testscenarios-0.4/Makefile
rename : test/3rdparty/testscenarios-0.2/doc/__init__.py => test/3rdparty/testscenarios-0.4/doc/__init__.py
rename : test/3rdparty/testscenarios-0.2/doc/example.py => test/3rdparty/testscenarios-0.4/doc/example.py
rename : test/3rdparty/testscenarios-0.2/doc/test_sample.py => test/3rdparty/testscenarios-0.4/doc/test_sample.py
rename : test/3rdparty/testtools-0.9.12/doc/conf.py => test/3rdparty/testtools-0.9.34/doc/conf.py
rename : test/3rdparty/testtools-0.9.12/doc/make.bat => test/3rdparty/testtools-0.9.34/doc/make.bat
rename : test/3rdparty/testtools-0.9.12/testtools/_compat2x.py => test/3rdparty/testtools-0.9.34/testtools/_compat2x.py
rename : test/3rdparty/testtools-0.9.12/testtools/_spinner.py => test/3rdparty/testtools-0.9.34/testtools/_spinner.py
rename : test/3rdparty/testtools-0.9.12/testtools/distutilscmd.py => test/3rdparty/testtools-0.9.34/testtools/distutilscmd.py
rename : test/3rdparty/testtools-0.9.12/testtools/monkey.py => test/3rdparty/testtools-0.9.34/testtools/monkey.py
rename : test/3rdparty/testtools-0.9.12/testtools/tests/test_monkey.py => test/3rdparty/testtools-0.9.34/testtools/tests/test_monkey.py
rename : test/3rdparty/testtools-0.9.12/testtools/tests/test_runtest.py => test/3rdparty/testtools-0.9.34/testtools/tests/test_runtest.py
rename : test/3rdparty/testtools-0.9.12/testtools/utils.py => test/3rdparty/testtools-0.9.34/testtools/utils.py
This commit is contained in:
Michael Cahill
2013-12-20 10:27:28 +11:00
parent e2fcf1a851
commit fa69e2a994
194 changed files with 24285 additions and 5494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

495
test/3rdparty/python-subunit-0.0.16/python/subunit/v2.py vendored Normal file
View File

@@ -0,0 +1,495 @@
#
# subunit: extensions to Python unittest to get test results from subprocesses.
# Copyright (C) 2013 Robert Collins <robertc@robertcollins.net>
#
# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause
# license at the users choice. A copy of both licenses are available in the
# project source as Apache-2.0 and BSD. You may not use this file except in
# compliance with one of these two licences.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# license you chose for the specific language governing permissions and
# limitations under that license.
#
import codecs
utf_8_decode = codecs.utf_8_decode
import datetime
from io import UnsupportedOperation
import os
import select
import struct
import zlib
from extras import safe_hasattr, try_imports
builtins = try_imports(['__builtin__', 'builtins'])
import subunit
import subunit.iso8601 as iso8601
__all__ = [
'ByteStreamToStreamResult',
'StreamResultToBytes',
]
SIGNATURE = b'\xb3'
FMT_8 = '>B'
FMT_16 = '>H'
FMT_24 = '>HB'
FMT_32 = '>I'
FMT_TIMESTAMP = '>II'
FLAG_TEST_ID = 0x0800
FLAG_ROUTE_CODE = 0x0400
FLAG_TIMESTAMP = 0x0200
FLAG_RUNNABLE = 0x0100
FLAG_TAGS = 0x0080
FLAG_MIME_TYPE = 0x0020
FLAG_EOF = 0x0010
FLAG_FILE_CONTENT = 0x0040
EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=iso8601.Utc())
NUL_ELEMENT = b'\0'[0]
# Contains True for types for which 'nul in thing' falsely returns false.
_nul_test_broken = {}
def has_nul(buffer_or_bytes):
"""Return True if a null byte is present in buffer_or_bytes."""
# Simple "if NUL_ELEMENT in utf8_bytes:" fails on Python 3.1 and 3.2 with
# memoryviews. See https://bugs.launchpad.net/subunit/+bug/1216246
buffer_type = type(buffer_or_bytes)
broken = _nul_test_broken.get(buffer_type)
if broken is None:
reference = buffer_type(b'\0')
broken = not NUL_ELEMENT in reference
_nul_test_broken[buffer_type] = broken
if broken:
return b'\0' in buffer_or_bytes
else:
return NUL_ELEMENT in buffer_or_bytes
class ParseError(Exception):
"""Used to pass error messages within the parser."""
class StreamResultToBytes(object):
"""Convert StreamResult API calls to bytes.
The StreamResult API is defined by testtools.StreamResult.
"""
status_mask = {
None: 0,
'exists': 0x1,
'inprogress': 0x2,
'success': 0x3,
'uxsuccess': 0x4,
'skip': 0x5,
'fail': 0x6,
'xfail': 0x7,
}
zero_b = b'\0'[0]
def __init__(self, output_stream):
"""Create a StreamResultToBytes with output written to output_stream.
:param output_stream: A file-like object. Must support write(bytes)
and flush() methods. Flush will be called after each write.
The stream will be passed through subunit.make_stream_binary,
to handle regular cases such as stdout.
"""
self.output_stream = subunit.make_stream_binary(output_stream)
def startTestRun(self):
pass
def stopTestRun(self):
pass
def status(self, test_id=None, test_status=None, test_tags=None,
runnable=True, file_name=None, file_bytes=None, eof=False,
mime_type=None, route_code=None, timestamp=None):
self._write_packet(test_id=test_id, test_status=test_status,
test_tags=test_tags, runnable=runnable, file_name=file_name,
file_bytes=file_bytes, eof=eof, mime_type=mime_type,
route_code=route_code, timestamp=timestamp)
def _write_utf8(self, a_string, packet):
utf8 = a_string.encode('utf-8')
self._write_number(len(utf8), packet)
packet.append(utf8)
def _write_len16(self, length, packet):
assert length < 65536
packet.append(struct.pack(FMT_16, length))
def _write_number(self, value, packet):
packet.extend(self._encode_number(value))
def _encode_number(self, value):
assert value >= 0
if value < 64:
return [struct.pack(FMT_8, value)]
elif value < 16384:
value = value | 0x4000
return [struct.pack(FMT_16, value)]
elif value < 4194304:
value = value | 0x800000
return [struct.pack(FMT_16, value >> 8),
struct.pack(FMT_8, value & 0xff)]
elif value < 1073741824:
value = value | 0xc0000000
return [struct.pack(FMT_32, value)]
else:
raise ValueError('value too large to encode: %r' % (value,))
def _write_packet(self, test_id=None, test_status=None, test_tags=None,
runnable=True, file_name=None, file_bytes=None, eof=False,
mime_type=None, route_code=None, timestamp=None):
packet = [SIGNATURE]
packet.append(b'FF') # placeholder for flags
# placeholder for length, but see below as length is variable.
packet.append(b'')
flags = 0x2000 # Version 0x2
if timestamp is not None:
flags = flags | FLAG_TIMESTAMP
since_epoch = timestamp - EPOCH
nanoseconds = since_epoch.microseconds * 1000
seconds = (since_epoch.seconds + since_epoch.days * 24 * 3600)
packet.append(struct.pack(FMT_32, seconds))
self._write_number(nanoseconds, packet)
if test_id is not None:
flags = flags | FLAG_TEST_ID
self._write_utf8(test_id, packet)
if test_tags:
flags = flags | FLAG_TAGS
self._write_number(len(test_tags), packet)
for tag in test_tags:
self._write_utf8(tag, packet)
if runnable:
flags = flags | FLAG_RUNNABLE
if mime_type:
flags = flags | FLAG_MIME_TYPE
self._write_utf8(mime_type, packet)
if file_name is not None:
flags = flags | FLAG_FILE_CONTENT
self._write_utf8(file_name, packet)
self._write_number(len(file_bytes), packet)
packet.append(file_bytes)
if eof:
flags = flags | FLAG_EOF
if route_code is not None:
flags = flags | FLAG_ROUTE_CODE
self._write_utf8(route_code, packet)
# 0x0008 - not used in v2.
flags = flags | self.status_mask[test_status]
packet[1] = struct.pack(FMT_16, flags)
base_length = sum(map(len, packet)) + 4
if base_length <= 62:
# one byte to encode length, 62+1 = 63
length_length = 1
elif base_length <= 16381:
# two bytes to encode length, 16381+2 = 16383
length_length = 2
elif base_length <= 4194300:
# three bytes to encode length, 419430+3=4194303
length_length = 3
else:
# Longer than policy:
# TODO: chunk the packet automatically?
# - strip all but file data
# - do 4M chunks of that till done
# - include original data in final chunk.
raise ValueError("Length too long: %r" % base_length)
packet[2:3] = self._encode_number(base_length + length_length)
# We could either do a partial application of crc32 over each chunk
# or a single join to a temp variable then a final join
# or two writes (that python might then split).
# For now, simplest code: join, crc32, join, output
content = b''.join(packet)
self.output_stream.write(content + struct.pack(
FMT_32, zlib.crc32(content) & 0xffffffff))
self.output_stream.flush()
class ByteStreamToStreamResult(object):
"""Parse a subunit byte stream.
Mixed streams that contain non-subunit content is supported when a
non_subunit_name is passed to the contructor. The default is to raise an
error containing the non-subunit byte after it has been read from the
stream.
Typical use:
>>> case = ByteStreamToStreamResult(sys.stdin.buffer)
>>> result = StreamResult()
>>> result.startTestRun()
>>> case.run(result)
>>> result.stopTestRun()
"""
status_lookup = {
0x0: None,
0x1: 'exists',
0x2: 'inprogress',
0x3: 'success',
0x4: 'uxsuccess',
0x5: 'skip',
0x6: 'fail',
0x7: 'xfail',
}
def __init__(self, source, non_subunit_name=None):
"""Create a ByteStreamToStreamResult.
:param source: A file like object to read bytes from. Must support
read(<count>) and return bytes. The file is not closed by
ByteStreamToStreamResult. subunit.make_stream_binary() is
called on the stream to get it into bytes mode.
:param non_subunit_name: If set to non-None, non subunit content
encountered in the stream will be converted into file packets
labelled with this name.
"""
self.non_subunit_name = non_subunit_name
self.source = subunit.make_stream_binary(source)
self.codec = codecs.lookup('utf8').incrementaldecoder()
def run(self, result):
"""Parse source and emit events to result.
This is a blocking call: it will run until EOF is detected on source.
"""
self.codec.reset()
mid_character = False
while True:
# We're in blocking mode; read one char
content = self.source.read(1)
if not content:
# EOF
return
if not mid_character and content[0] == SIGNATURE[0]:
self._parse_packet(result)
continue
if self.non_subunit_name is None:
raise Exception("Non subunit content", content)
try:
if self.codec.decode(content):
# End of a character
mid_character = False
else:
mid_character = True
except UnicodeDecodeError:
# Bad unicode, not our concern.
mid_character = False
# Aggregate all content that is not subunit until either
# 1MiB is accumulated or 50ms has passed with no input.
# Both are arbitrary amounts intended to give a simple
# balance between efficiency (avoiding death by a thousand
# one-byte packets), buffering (avoiding overlarge state
# being hidden on intermediary nodes) and interactivity
# (when driving a debugger, slow response to typing is
# annoying).
buffered = [content]
while len(buffered[-1]):
try:
self.source.fileno()
except:
# Won't be able to select, fallback to
# one-byte-at-a-time.
break
# Note: this has a very low timeout because with stdin, the
# BufferedIO layer typically has all the content available
# from the stream when e.g. pdb is dropped into, leading to
# select always timing out when in fact we could have read
# (from the buffer layer) - we typically fail to aggregate
# any content on 3.x Pythons.
readable = select.select([self.source], [], [], 0.000001)[0]
if readable:
content = self.source.read(1)
if not len(content):
# EOF - break and emit buffered.
break
if not mid_character and content[0] == SIGNATURE[0]:
# New packet, break, emit buffered, then parse.
break
buffered.append(content)
# Feed into the codec.
try:
if self.codec.decode(content):
# End of a character
mid_character = False
else:
mid_character = True
except UnicodeDecodeError:
# Bad unicode, not our concern.
mid_character = False
if not readable or len(buffered) >= 1048576:
# timeout or too much data, emit what we have.
break
result.status(
file_name=self.non_subunit_name,
file_bytes=b''.join(buffered))
if mid_character or not len(content) or content[0] != SIGNATURE[0]:
continue
# Otherwise, parse a data packet.
self._parse_packet(result)
def _parse_packet(self, result):
try:
packet = [SIGNATURE]
self._parse(packet, result)
except ParseError as error:
result.status(test_id="subunit.parser", eof=True,
file_name="Packet data", file_bytes=b''.join(packet),
mime_type="application/octet-stream")
result.status(test_id="subunit.parser", test_status='fail',
eof=True, file_name="Parser Error",
file_bytes=(error.args[0]).encode('utf8'),
mime_type="text/plain;charset=utf8")
def _to_bytes(self, data, pos, length):
"""Return a slice of data from pos for length as bytes."""
# memoryview in 2.7.3 and 3.2 isn't directly usable with struct :(.
# see https://bugs.launchpad.net/subunit/+bug/1216163
result = data[pos:pos+length]
if type(result) is not bytes:
return result.tobytes()
return result
def _parse_varint(self, data, pos, max_3_bytes=False):
# because the only incremental IO we do is at the start, and the 32 bit
# CRC means we can always safely read enough to cover any varint, we
# can be sure that there should be enough data - and if not it is an
# error not a normal situation.
data_0 = struct.unpack(FMT_8, self._to_bytes(data, pos, 1))[0]
typeenum = data_0 & 0xc0
value_0 = data_0 & 0x3f
if typeenum == 0x00:
return value_0, 1
elif typeenum == 0x40:
data_1 = struct.unpack(FMT_8, self._to_bytes(data, pos+1, 1))[0]
return (value_0 << 8) | data_1, 2
elif typeenum == 0x80:
data_1 = struct.unpack(FMT_16, self._to_bytes(data, pos+1, 2))[0]
return (value_0 << 16) | data_1, 3
else:
if max_3_bytes:
raise ParseError('3 byte maximum given but 4 byte value found.')
data_1, data_2 = struct.unpack(FMT_24, self._to_bytes(data, pos+1, 3))
result = (value_0 << 24) | data_1 << 8 | data_2
return result, 4
def _parse(self, packet, result):
# 2 bytes flags, at most 3 bytes length.
packet.append(self.source.read(5))
flags = struct.unpack(FMT_16, packet[-1][:2])[0]
length, consumed = self._parse_varint(
packet[-1], 2, max_3_bytes=True)
remainder = self.source.read(length - 6)
if len(remainder) != length - 6:
raise ParseError(
'Short read - got %d bytes, wanted %d bytes' % (
len(remainder), length - 6))
if consumed != 3:
# Avoid having to parse torn values
packet[-1] += remainder
pos = 2 + consumed
else:
# Avoid copying potentially lots of data.
packet.append(remainder)
pos = 0
crc = zlib.crc32(packet[0])
for fragment in packet[1:-1]:
crc = zlib.crc32(fragment, crc)
crc = zlib.crc32(packet[-1][:-4], crc) & 0xffffffff
packet_crc = struct.unpack(FMT_32, packet[-1][-4:])[0]
if crc != packet_crc:
# Bad CRC, report it and stop parsing the packet.
raise ParseError(
'Bad checksum - calculated (0x%x), stored (0x%x)'
% (crc, packet_crc))
if safe_hasattr(builtins, 'memoryview'):
body = memoryview(packet[-1])
else:
body = packet[-1]
# Discard CRC-32
body = body[:-4]
# One packet could have both file and status data; the Python API
# presents these separately (perhaps it shouldn't?)
if flags & FLAG_TIMESTAMP:
seconds = struct.unpack(FMT_32, self._to_bytes(body, pos, 4))[0]
nanoseconds, consumed = self._parse_varint(body, pos+4)
pos = pos + 4 + consumed
timestamp = EPOCH + datetime.timedelta(
seconds=seconds, microseconds=nanoseconds/1000)
else:
timestamp = None
if flags & FLAG_TEST_ID:
test_id, pos = self._read_utf8(body, pos)
else:
test_id = None
if flags & FLAG_TAGS:
tag_count, consumed = self._parse_varint(body, pos)
pos += consumed
test_tags = set()
for _ in range(tag_count):
tag, pos = self._read_utf8(body, pos)
test_tags.add(tag)
else:
test_tags = None
if flags & FLAG_MIME_TYPE:
mime_type, pos = self._read_utf8(body, pos)
else:
mime_type = None
if flags & FLAG_FILE_CONTENT:
file_name, pos = self._read_utf8(body, pos)
content_length, consumed = self._parse_varint(body, pos)
pos += consumed
file_bytes = self._to_bytes(body, pos, content_length)
if len(file_bytes) != content_length:
raise ParseError('File content extends past end of packet: '
'claimed %d bytes, %d available' % (
content_length, len(file_bytes)))
pos += content_length
else:
file_name = None
file_bytes = None
if flags & FLAG_ROUTE_CODE:
route_code, pos = self._read_utf8(body, pos)
else:
route_code = None
runnable = bool(flags & FLAG_RUNNABLE)
eof = bool(flags & FLAG_EOF)
test_status = self.status_lookup[flags & 0x0007]
result.status(test_id=test_id, test_status=test_status,
test_tags=test_tags, runnable=runnable, mime_type=mime_type,
eof=eof, file_name=file_name, file_bytes=file_bytes,
route_code=route_code, timestamp=timestamp)
__call__ = run
def _read_utf8(self, buf, pos):
length, consumed = self._parse_varint(buf, pos)
pos += consumed
utf8_bytes = buf[pos:pos+length]
if length != len(utf8_bytes):
raise ParseError(
'UTF8 string at offset %d extends past end of packet: '
'claimed %d bytes, %d available' % (pos - 2, length,
len(utf8_bytes)))
if has_nul(utf8_bytes):
raise ParseError('UTF8 string at offset %d contains NUL byte' % (
pos-2,))
try:
utf8, decoded_bytes = utf_8_decode(utf8_bytes)
if decoded_bytes != length:
raise ParseError("Invalid (partially decodable) string at "
"offset %d, %d undecoded bytes" % (
pos-2, length - decoded_bytes))
return utf8, length+pos
except UnicodeDecodeError:
raise ParseError('UTF8 string at offset %d is not UTF8' % (pos-2,))