WT-0000 Cut WiredTiger WT-2.9.1 release (#3215)

Fix setting the Autoconf options too late for strict checking.

NEWS

@@ -1,3 +1,53 @@
+WiredTiger release 2.9.1, 2016-12-22
+------------------------------------
+
+New features and API changes; refer to the API documentation for full details:
+
+* SERVER-26545	Remove fixed-size limitation on WiredTiger hazard pointers. See the upgrading documentation for details
+* WT-283	Add a new WT_SESSION::alter method that can be used to reconfigure table metadata
+* WT-2670	Change the default file system access pattern advice for data files from random to no advice. Add access_pattern_hint configuration option for WT_SESSION::create API that can be used to advise the file system of expected access semantics. See the upgrading documentation for details.
+* WT-3034	Add support for including updates when reading from named snapshots
+
+Significant changes and bug fixes:
+
+* WT-2960	Reduce likelihood of using the lookaside file, especially when inserting multi-megabyte values
+* WT-3056	Allow projected table and join cursors to use primary keys
+* WT-3070	Fix a bug in search_near on indexes
+
+Other noteworthy changes since the previous release:
+
+* WT-2336	Add a test validating schema operations via file system call monitoring
+* WT-2402	Pad structures to avoid cache line sharing
+* WT-2771	Add a statistic to track per-btree dirty cache usage
+* WT-2833	Add projections to wt dump utility
+* WT-2969	Possible snapshot corruption during compaction
+* WT-3014	Add GCC/clang support for ELF symbol visibility
+* WT-3021	Fixes for java log example, raw mode in java, and raw mode in log cursors
+* WT-3025	Fix error path in log_force_sync
+* WT-3028	Don't check for blocked eviction with in-memory workloads
+* WT-3030	Fix a race between scans and splits reading the index hint
+* WT-3037	Clean up some log slot comments
+* WT-3048	WiredTiger maximum size warning uses the wrong format
+* WT-3051	Remove external __wt_hex symbol
+* WT-3052	Improve search if index hint is wrong
+* WT-3053	Make Python use internal memory allocation again
+* WT-3054	Make a PackOutputStream constructor that is compatible with the previous interface.
+* WT-3055	When an AsyncOp is created, cache the whether the cursor is "raw"
+* WT-3057	WiredTiger hazard pointers should use the WT_REF, not the WT_PAGE
+* WT-3061	Syscall testing should support pwrite64 on Linux
+* WT-3064	Minor tree cleanups: .gitignore, NEWS misspelling
+* WT-3066	Minor code cleanups
+* WT-3068	Copy artifacts of test runs in wtperf_run script
+* WT-3068	Have Jenkins include specific files for copy rather than exclude
+* WT-3069	Fix LevelDB APIs build failures
+* WT-3071	Fixed sign-conversion compiler errors in Java and Python SWIG code
+* WT-3075	Document and enforce that WiredTiger now depends on Python 2.7
+* WT-3078	Test reconfiguration hang in the statlog server
+* WT-3080	Python test suite: add elapsed time for tests
+* WT-3082	Python test suite: shorten default run to avoid timeouts
+* WT-3084	Fix Coverity resource leak complaint
+* WT-3091	Add stats to test_perf001 test, so we can investigate what happened when it failed
+
 WiredTiger release 2.9.0, 2016-09-06
 ------------------------------------
 

README

@@ -1,4 +1,4 @@
-WiredTiger 2.9.1: (December  7, 2016)
+WiredTiger 2.9.1: (December 23, 2016)
 
 This is version 2.9.1 of WiredTiger.
 

build_posix/aclocal/version-set.m4

@@ -3,7 +3,7 @@ dnl build by dist/s_version
 VERSION_MAJOR=2
 VERSION_MINOR=9
 VERSION_PATCH=1
-VERSION_STRING='"WiredTiger 2.9.1: (December  7, 2016)"'
+VERSION_STRING='"WiredTiger 2.9.1: (December 23, 2016)"'
 
 AC_SUBST(VERSION_MAJOR)
 AC_SUBST(VERSION_MINOR)

build_posix/configure.ac.in

@@ -91,6 +91,9 @@ fi
 # Linux requires _GNU_SOURCE to be defined
 AS_CASE([$host_os], [linux*], [AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE"])
 
+# Configure options.
+AM_OPTIONS
+
 # If enable-strict is configured, turn on as much error checking as we can for
 # this compiler. Intended for developers, and only works for gcc/clang, but it
 # fills a need.
@@ -109,9 +112,6 @@ if test "$wt_cv_enable_strict" = "yes"; then
 	AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings"
 fi
 
-# Configure options.
-AM_OPTIONS
-
 # Java and Python APIs
 if test "$wt_cv_enable_java" = "yes" -o "$wt_cv_enable_python" = "yes"; then
 	# Only a warning, we need to build release packages without SWIG.
@@ -133,7 +133,7 @@ if test "$wt_cv_enable_java" = "yes"; then
 fi
 
 if test "$wt_cv_enable_python" = "yes"; then
-	AM_PATH_PYTHON([2.7])
+	AM_PATH_PYTHON([2.6])
 	if test -n "$with_python_prefix" ; then
 		PYTHON_INSTALL_ARG="-d $with_python_prefix"
 	fi

dist/s_string.ok

@@ -21,6 +21,7 @@ Alakuijala
 Alexandrescu's
 Alloc
 Async
+AsyncOp
 Athanassoulis
 Athlon
 BBBBB
@@ -276,6 +277,7 @@ PRNG
 PTHREAD
 PTR
 PackInputStream
+PackOutputStream
 Pandis
 Phong
 PlatformSDK
@@ -339,6 +341,7 @@ Split's
 Stoica
 StoreLoad
 StoreStore
+Syscall
 TAILQ
 TCMalloc
 TESTUTIL

dist/stat_data.py

@@ -477,6 +477,7 @@ dsrc_stats = [
     ##########################################
     # Cache and eviction statistics
     ##########################################
+    CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale,size'),
     CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale,size'),
     CacheStat('cache_bytes_read', 'bytes read into cache', 'size'),
     CacheStat('cache_bytes_write', 'bytes written from cache', 'size'),

src/btree/bt_stat.c

@@ -40,6 +40,8 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst)
 	WT_STAT_SET(session, stats, btree_maxleafpage, btree->maxleafpage);
 	WT_STAT_SET(session, stats, btree_maxleafvalue, btree->maxleafvalue);
 
+	WT_STAT_SET(session, stats, cache_bytes_dirty,
+	    __wt_btree_dirty_inuse(session));
 	WT_STAT_SET(session, stats, cache_bytes_inuse,
 	    __wt_btree_bytes_inuse(session));
 

src/cursor/cur_index.c

@@ -520,8 +520,8 @@ __wt_curindex_open(WT_SESSION_IMPL *session,
 	WT_ERR(__curindex_open_colgroups(session, cindex, cfg));
 
 	if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON))
-		__wt_json_column_init(
-		    cursor, table->key_format, &idx->colconf, &table->colconf);
+		__wt_json_column_init(cursor, uri, table->key_format,
+		    &idx->colconf, &table->colconf);
 
 	if (0) {
 err:		WT_TRET(__curindex_close(cursor));

src/cursor/cur_json.c

@@ -369,11 +369,11 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode)
  *	of column names.
  */
 void
-__wt_json_column_init(WT_CURSOR *cursor, const char *keyformat,
+__wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat,
     const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf)
 {
 	WT_CURSOR_JSON *json;
-	const char *p, *end, *beginkey;
+	const char *beginkey, *end, *lparen, *p;
 	uint32_t keycnt, nkeys;
 
 	json = (WT_CURSOR_JSON *)cursor->json_private;
@@ -400,8 +400,16 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat,
 			keycnt++;
 		p++;
 	}
-	json->value_names.str = p;
-	json->value_names.len = WT_PTRDIFF(end, p);
+	if ((lparen = strchr(uri, '(')) != NULL) {
+		/* This cursor is a projection. */
+		json->value_names.str = lparen;
+		json->value_names.len = strlen(lparen) - 1;
+		WT_ASSERT((WT_SESSION_IMPL *)cursor->session,
+		    json->value_names.str[json->value_names.len] == ')');
+	} else {
+		json->value_names.str = p;
+		json->value_names.len = WT_PTRDIFF(end, p);
+	}
 	if (idxconf == NULL) {
 		if (p > beginkey)
 			p--;

src/cursor/cur_table.c

@@ -951,7 +951,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session,
 
 	if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON))
 		__wt_json_column_init(
-		    cursor, table->key_format, NULL, &table->colconf);
+		    cursor, uri, table->key_format, NULL, &table->colconf);
 
 	/*
 	 * Open the colgroup cursors immediately: we're going to need them for

src/docs/build-pydoc.sh

@@ -3,4 +3,4 @@ TOP=$DOCS/..
 . $TOP/config.sh
 
 cd python
-PYTHONPATH=../../lang/python/src:$THRIFT_HOME/lib/python2.7/site-packages pydoc -w wiredtiger
+PYTHONPATH=../../lang/python/src:$THRIFT_HOME/lib/python2.6/site-packages pydoc -w wiredtiger

src/docs/testing.dox

@@ -27,7 +27,7 @@ The WiredTiger unit test suite includes tests that cover:
 
 The WiredTiger Python test suite is built using the WiredTiger Python
 API and the Python unittest functionality (the test suite requires at
-least Python version 2.7).
+least Python version 2.6).
 
 The WiredTiger test suite automatically runs as part of every commit
 into the WiredTiger GitHub source tree.

src/docs/top/main.dox

@@ -6,12 +6,12 @@ WiredTiger is an high performance, scalable, production quality, NoSQL,
 @section releases Releases
 
 <table>
-@row{<b>WiredTiger 2.9.0</b> (current),
+@row{<b>WiredTiger 2.9.1</b> (current),
+	<a href="releases/wiredtiger-2.9.1.tar.bz2"><b>[Release package]</b></a>,
+	<a href="2.9.1/index.html"><b>[Documentation]</b></a>}
+@row{<b>WiredTiger 2.9.0</b> (previous),
 	<a href="releases/wiredtiger-2.9.0.tar.bz2"><b>[Release package]</b></a>,
 	<a href="2.9.0/index.html"><b>[Documentation]</b></a>}
-@row{<b>WiredTiger 2.8.0</b> (previous),
-	<a href="releases/wiredtiger-2.8.0.tar.bz2"><b>[Release package]</b></a>,
-	<a href="2.8.0/index.html"><b>[Documentation]</b></a>}
 @row{<b>Development branch</b>,
 	<a href="https://github.com/wiredtiger/wiredtiger"><b>[Source code]</b></a>,
 	<a href="develop/index.html"><b>[Documentation]</b></a>}

src/docs/upgrading.dox

@@ -1,19 +1,24 @@
 /*! @page upgrading Upgrading WiredTiger applications
-@section version_291 Upgrading to Version 2.9.1
-<dl>
-<dt>WiredTiger now requires Python 2.7 at minimum</dt>
-<dd>
-The minimum version of Python supported by WiredTiger is now 2.7 up from the
-previous version of 2.6. This is due to extra unit tests added in this release
-that depend on 2.7. This is not due to a change in the Python API.
-</dd>
 
+@section version_291 Upgrading to Version 2.9.1
+
+<dl>
 <dt>Changes to hazard pointer configuration</dt>
 <dd>
 The \c hazard_max parameter to ::wiredtiger_open is now ignored.  Memory is
 allocated for hazard pointers as required by each session.
 </dd>
-</dl><hr>
+
+<dt>Change to the default fadvise behavior for data files</dt>
+<dd>
+The old default behavior was to advise the file system that access would be
+random for data files, and there was no way to alter that. We no longer
+call advise the file system of expected access patterns by default, and
+have added a new \c access_pattern_hint configuration option available for
+WT_SESSION::create that can be used to restore the old default by setting
+the value to "random".
+</dd>
+</dl>
 
 @section version_290 Upgrading to Version 2.9.0
 <dl>

src/include/btree.h

@@ -131,6 +131,7 @@ struct __wt_btree {
 	uint64_t write_gen;		/* Write generation */
 
 	uint64_t    bytes_inmem;	/* Cache bytes in memory. */
+	uint64_t    bytes_dirty_intl;	/* Bytes in dirty internal pages. */
 	uint64_t    bytes_dirty_leaf;	/* Bytes in dirty leaf pages. */
 
 	WT_REF	   *evict_ref;		/* Eviction thread's location */

src/include/btree.i

@@ -70,6 +70,23 @@ __wt_btree_bytes_inuse(WT_SESSION_IMPL *session)
 	return (__wt_cache_bytes_plus_overhead(cache, btree->bytes_inmem));
 }
 
+/*
+ * __wt_btree_dirty_inuse --
+ *	Return the number of dirty bytes in use.
+ */
+static inline uint64_t
+__wt_btree_dirty_inuse(WT_SESSION_IMPL *session)
+{
+	WT_BTREE *btree;
+	WT_CACHE *cache;
+
+	btree = S2BT(session);
+	cache = S2C(session)->cache;
+
+	return (__wt_cache_bytes_plus_overhead(cache,
+	    btree->bytes_dirty_intl + btree->bytes_dirty_leaf));
+}
+
 /*
  * __wt_btree_dirty_leaf_inuse --
  *	Return the number of bytes in use by dirty leaf pages.
@@ -105,11 +122,12 @@ __wt_cache_page_inmem_incr(WT_SESSION_IMPL *session, WT_PAGE *page, size_t size)
 	(void)__wt_atomic_addsize(&page->memory_footprint, size);
 	if (__wt_page_is_modified(page)) {
 		(void)__wt_atomic_addsize(&page->modify->bytes_dirty, size);
-		if (WT_PAGE_IS_INTERNAL(page))
+		if (WT_PAGE_IS_INTERNAL(page)) {
+			(void)__wt_atomic_add64(&btree->bytes_dirty_intl, size);
 			(void)__wt_atomic_add64(&cache->bytes_dirty_intl, size);
-		else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
-			(void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size);
+		} else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
 			(void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size);
+			(void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size);
 		}
 	}
 	/* Track internal size in cache. */
@@ -238,10 +256,12 @@ __wt_cache_page_byte_dirty_decr(
 	if (i == 5)
 		return;
 
-	if (WT_PAGE_IS_INTERNAL(page))
+	if (WT_PAGE_IS_INTERNAL(page)) {
+		__wt_cache_decr_check_uint64(session, &btree->bytes_dirty_intl,
+		    decr, "WT_BTREE.bytes_dirty_intl");
 		__wt_cache_decr_check_uint64(session, &cache->bytes_dirty_intl,
 		    decr, "WT_CACHE.bytes_dirty_intl");
-	else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
+	} else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
 		__wt_cache_decr_check_uint64(session, &btree->bytes_dirty_leaf,
 		    decr, "WT_BTREE.bytes_dirty_leaf");
 		__wt_cache_decr_check_uint64(session, &cache->bytes_dirty_leaf,
@@ -297,6 +317,7 @@ __wt_cache_dirty_incr(WT_SESSION_IMPL *session, WT_PAGE *page)
 	 */
 	size = page->memory_footprint;
 	if (WT_PAGE_IS_INTERNAL(page)) {
+		(void)__wt_atomic_add64(&btree->bytes_dirty_intl, size);
 		(void)__wt_atomic_add64(&cache->bytes_dirty_intl, size);
 		(void)__wt_atomic_add64(&cache->pages_dirty_intl, 1);
 	} else {
@@ -392,17 +413,20 @@ __wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page)
 
 	/* Update the cache's dirty-byte count. */
 	if (modify != NULL && modify->bytes_dirty != 0) {
-		if (WT_PAGE_IS_INTERNAL(page))
+		if (WT_PAGE_IS_INTERNAL(page)) {
+			__wt_cache_decr_zero_uint64(session,
+			    &btree->bytes_dirty_intl,
+			    modify->bytes_dirty, "WT_BTREE.bytes_dirty_intl");
 			__wt_cache_decr_zero_uint64(session,
 			    &cache->bytes_dirty_intl,
 			    modify->bytes_dirty, "WT_CACHE.bytes_dirty_intl");
-		else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
-			__wt_cache_decr_zero_uint64(session,
-			    &cache->bytes_dirty_leaf,
-			    modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf");
+		} else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
 			__wt_cache_decr_zero_uint64(session,
 			    &btree->bytes_dirty_leaf,
 			    modify->bytes_dirty, "WT_BTREE.bytes_dirty_leaf");
+			__wt_cache_decr_zero_uint64(session,
+			    &cache->bytes_dirty_leaf,
+			    modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf");
 		}
 	}
 

src/include/extern.h

@@ -294,7 +294,7 @@ extern int __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT
 extern int __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, WT_CURSOR_JSON *json, bool iskey, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
-extern void __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
+extern void __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern const char *__wt_json_tokname(int toktype) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
 extern int __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, const char *format, WT_CURSOR_JSON *json, bool iskey, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));

src/include/stat.h

@@ -564,6 +564,7 @@ struct __wt_dsrc_stats {
 	int64_t cache_pages_requested;
 	int64_t cache_write;
 	int64_t cache_write_restore;
+	int64_t cache_bytes_dirty;
 	int64_t cache_eviction_clean;
 	int64_t cache_state_gen_avg_gap;
 	int64_t cache_state_avg_written_size;

src/include/wiredtiger.in

@@ -4978,181 +4978,183 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
 #define	WT_STAT_DSRC_CACHE_WRITE			2059
 /*! cache: pages written requiring in-memory restoration */
 #define	WT_STAT_DSRC_CACHE_WRITE_RESTORE		2060
+/*! cache: tracked dirty bytes in the cache */
+#define	WT_STAT_DSRC_CACHE_BYTES_DIRTY			2061
 /*! cache: unmodified pages evicted */
-#define	WT_STAT_DSRC_CACHE_EVICTION_CLEAN		2061
+#define	WT_STAT_DSRC_CACHE_EVICTION_CLEAN		2062
 /*!
  * cache_walk: Average difference between current eviction generation
  * when the page was last considered, only reported if cache_walk or all
  * statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP		2062
+#define	WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP		2063
 /*!
  * cache_walk: Average on-disk page image size seen, only reported if
  * cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE	2063
+#define	WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE	2064
 /*!
  * cache_walk: Clean pages currently in cache, only reported if
  * cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN		2064
+#define	WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN		2065
 /*!
  * cache_walk: Current eviction generation, only reported if cache_walk
  * or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT		2065
+#define	WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT		2066
 /*!
  * cache_walk: Dirty pages currently in cache, only reported if
  * cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY		2066
+#define	WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY		2067
 /*!
  * cache_walk: Entries in the root page, only reported if cache_walk or
  * all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES		2067
+#define	WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES		2068
 /*!
  * cache_walk: Internal pages currently in cache, only reported if
  * cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL		2068
+#define	WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL		2069
 /*!
  * cache_walk: Leaf pages currently in cache, only reported if cache_walk
  * or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF		2069
+#define	WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF		2070
 /*!
  * cache_walk: Maximum difference between current eviction generation
  * when the page was last considered, only reported if cache_walk or all
  * statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP		2070
+#define	WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP		2071
 /*!
  * cache_walk: Maximum page size seen, only reported if cache_walk or all
  * statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE		2071
+#define	WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE		2072
 /*!
  * cache_walk: Minimum on-disk page image size seen, only reported if
  * cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE	2072
+#define	WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE	2073
 /*!
  * cache_walk: On-disk page image sizes smaller than a single allocation
  * unit, only reported if cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE	2073
+#define	WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE	2074
 /*!
  * cache_walk: Pages created in memory and never written, only reported
  * if cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_MEMORY			2074
+#define	WT_STAT_DSRC_CACHE_STATE_MEMORY			2075
 /*!
  * cache_walk: Pages currently queued for eviction, only reported if
  * cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_QUEUED			2075
+#define	WT_STAT_DSRC_CACHE_STATE_QUEUED			2076
 /*!
  * cache_walk: Pages that could not be queued for eviction, only reported
  * if cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE		2076
+#define	WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE		2077
 /*!
  * cache_walk: Refs skipped during cache traversal, only reported if
  * cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED		2077
+#define	WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED		2078
 /*!
  * cache_walk: Size of the root page, only reported if cache_walk or all
  * statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE		2078
+#define	WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE		2079
 /*!
  * cache_walk: Total number of pages currently in cache, only reported if
  * cache_walk or all statistics are enabled
  */
-#define	WT_STAT_DSRC_CACHE_STATE_PAGES			2079
+#define	WT_STAT_DSRC_CACHE_STATE_PAGES			2080
 /*! compression: compressed pages read */
-#define	WT_STAT_DSRC_COMPRESS_READ			2080
+#define	WT_STAT_DSRC_COMPRESS_READ			2081
 /*! compression: compressed pages written */
-#define	WT_STAT_DSRC_COMPRESS_WRITE			2081
+#define	WT_STAT_DSRC_COMPRESS_WRITE			2082
 /*! compression: page written failed to compress */
-#define	WT_STAT_DSRC_COMPRESS_WRITE_FAIL		2082
+#define	WT_STAT_DSRC_COMPRESS_WRITE_FAIL		2083
 /*! compression: page written was too small to compress */
-#define	WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL		2083
+#define	WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL		2084
 /*! compression: raw compression call failed, additional data available */
-#define	WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY	2084
+#define	WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY	2085
 /*! compression: raw compression call failed, no additional data available */
-#define	WT_STAT_DSRC_COMPRESS_RAW_FAIL			2085
+#define	WT_STAT_DSRC_COMPRESS_RAW_FAIL			2086
 /*! compression: raw compression call succeeded */
-#define	WT_STAT_DSRC_COMPRESS_RAW_OK			2086
+#define	WT_STAT_DSRC_COMPRESS_RAW_OK			2087
 /*! cursor: bulk-loaded cursor-insert calls */
-#define	WT_STAT_DSRC_CURSOR_INSERT_BULK			2087
+#define	WT_STAT_DSRC_CURSOR_INSERT_BULK			2088
 /*! cursor: create calls */
-#define	WT_STAT_DSRC_CURSOR_CREATE			2088
+#define	WT_STAT_DSRC_CURSOR_CREATE			2089
 /*! cursor: cursor-insert key and value bytes inserted */
-#define	WT_STAT_DSRC_CURSOR_INSERT_BYTES		2089
+#define	WT_STAT_DSRC_CURSOR_INSERT_BYTES		2090
 /*! cursor: cursor-remove key bytes removed */
-#define	WT_STAT_DSRC_CURSOR_REMOVE_BYTES		2090
+#define	WT_STAT_DSRC_CURSOR_REMOVE_BYTES		2091
 /*! cursor: cursor-update value bytes updated */
-#define	WT_STAT_DSRC_CURSOR_UPDATE_BYTES		2091
+#define	WT_STAT_DSRC_CURSOR_UPDATE_BYTES		2092
 /*! cursor: insert calls */
-#define	WT_STAT_DSRC_CURSOR_INSERT			2092
+#define	WT_STAT_DSRC_CURSOR_INSERT			2093
 /*! cursor: next calls */
-#define	WT_STAT_DSRC_CURSOR_NEXT			2093
+#define	WT_STAT_DSRC_CURSOR_NEXT			2094
 /*! cursor: prev calls */
-#define	WT_STAT_DSRC_CURSOR_PREV			2094
+#define	WT_STAT_DSRC_CURSOR_PREV			2095
 /*! cursor: remove calls */
-#define	WT_STAT_DSRC_CURSOR_REMOVE			2095
+#define	WT_STAT_DSRC_CURSOR_REMOVE			2096
 /*! cursor: reset calls */
-#define	WT_STAT_DSRC_CURSOR_RESET			2096
+#define	WT_STAT_DSRC_CURSOR_RESET			2097
 /*! cursor: restarted searches */
-#define	WT_STAT_DSRC_CURSOR_RESTART			2097
+#define	WT_STAT_DSRC_CURSOR_RESTART			2098
 /*! cursor: search calls */
-#define	WT_STAT_DSRC_CURSOR_SEARCH			2098
+#define	WT_STAT_DSRC_CURSOR_SEARCH			2099
 /*! cursor: search near calls */
-#define	WT_STAT_DSRC_CURSOR_SEARCH_NEAR			2099
+#define	WT_STAT_DSRC_CURSOR_SEARCH_NEAR			2100
 /*! cursor: truncate calls */
-#define	WT_STAT_DSRC_CURSOR_TRUNCATE			2100
+#define	WT_STAT_DSRC_CURSOR_TRUNCATE			2101
 /*! cursor: update calls */
-#define	WT_STAT_DSRC_CURSOR_UPDATE			2101
+#define	WT_STAT_DSRC_CURSOR_UPDATE			2102
 /*! reconciliation: dictionary matches */
-#define	WT_STAT_DSRC_REC_DICTIONARY			2102
+#define	WT_STAT_DSRC_REC_DICTIONARY			2103
 /*! reconciliation: fast-path pages deleted */
-#define	WT_STAT_DSRC_REC_PAGE_DELETE_FAST		2103
+#define	WT_STAT_DSRC_REC_PAGE_DELETE_FAST		2104
 /*!
  * reconciliation: internal page key bytes discarded using suffix
  * compression
  */
-#define	WT_STAT_DSRC_REC_SUFFIX_COMPRESSION		2104
+#define	WT_STAT_DSRC_REC_SUFFIX_COMPRESSION		2105
 /*! reconciliation: internal page multi-block writes */
-#define	WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL		2105
+#define	WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL		2106
 /*! reconciliation: internal-page overflow keys */
-#define	WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL		2106
+#define	WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL		2107
 /*! reconciliation: leaf page key bytes discarded using prefix compression */
-#define	WT_STAT_DSRC_REC_PREFIX_COMPRESSION		2107
+#define	WT_STAT_DSRC_REC_PREFIX_COMPRESSION		2108
 /*! reconciliation: leaf page multi-block writes */
-#define	WT_STAT_DSRC_REC_MULTIBLOCK_LEAF		2108
+#define	WT_STAT_DSRC_REC_MULTIBLOCK_LEAF		2109
 /*! reconciliation: leaf-page overflow keys */
-#define	WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF		2109
+#define	WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF		2110
 /*! reconciliation: maximum blocks required for a page */
-#define	WT_STAT_DSRC_REC_MULTIBLOCK_MAX			2110
+#define	WT_STAT_DSRC_REC_MULTIBLOCK_MAX			2111
 /*! reconciliation: overflow values written */
-#define	WT_STAT_DSRC_REC_OVERFLOW_VALUE			2111
+#define	WT_STAT_DSRC_REC_OVERFLOW_VALUE			2112
 /*! reconciliation: page checksum matches */
-#define	WT_STAT_DSRC_REC_PAGE_MATCH			2112
+#define	WT_STAT_DSRC_REC_PAGE_MATCH			2113
 /*! reconciliation: page reconciliation calls */
-#define	WT_STAT_DSRC_REC_PAGES				2113
+#define	WT_STAT_DSRC_REC_PAGES				2114
 /*! reconciliation: page reconciliation calls for eviction */
-#define	WT_STAT_DSRC_REC_PAGES_EVICTION			2114
+#define	WT_STAT_DSRC_REC_PAGES_EVICTION			2115
 /*! reconciliation: pages deleted */
-#define	WT_STAT_DSRC_REC_PAGE_DELETE			2115
+#define	WT_STAT_DSRC_REC_PAGE_DELETE			2116
 /*! session: object compaction */
-#define	WT_STAT_DSRC_SESSION_COMPACT			2116
+#define	WT_STAT_DSRC_SESSION_COMPACT			2117
 /*! session: open cursor count */
-#define	WT_STAT_DSRC_SESSION_CURSOR_OPEN		2117
+#define	WT_STAT_DSRC_SESSION_CURSOR_OPEN		2118
 /*! transaction: update conflicts */
-#define	WT_STAT_DSRC_TXN_UPDATE_CONFLICT		2118
+#define	WT_STAT_DSRC_TXN_UPDATE_CONFLICT		2119
 
 /*!
  * @}

src/support/stat.c

@@ -64,6 +64,7 @@ static const char * const __stats_dsrc_desc[] = {
 	"cache: pages requested from the cache",
 	"cache: pages written from cache",
 	"cache: pages written requiring in-memory restoration",
+	"cache: tracked dirty bytes in the cache",
 	"cache: unmodified pages evicted",
 	"cache_walk: Average difference between current eviction generation when the page was last considered",
 	"cache_walk: Average on-disk page image size seen",
@@ -225,6 +226,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats)
 	stats->cache_pages_requested = 0;
 	stats->cache_write = 0;
 	stats->cache_write_restore = 0;
+		/* not clearing cache_bytes_dirty */
 	stats->cache_eviction_clean = 0;
 		/* not clearing cache_state_gen_avg_gap */
 		/* not clearing cache_state_avg_written_size */
@@ -372,6 +374,7 @@ __wt_stat_dsrc_aggregate_single(
 	to->cache_pages_requested += from->cache_pages_requested;
 	to->cache_write += from->cache_write;
 	to->cache_write_restore += from->cache_write_restore;
+	to->cache_bytes_dirty += from->cache_bytes_dirty;
 	to->cache_eviction_clean += from->cache_eviction_clean;
 	to->cache_state_gen_avg_gap += from->cache_state_gen_avg_gap;
 	to->cache_state_avg_written_size +=
@@ -535,6 +538,7 @@ __wt_stat_dsrc_aggregate(
 	    WT_STAT_READ(from, cache_pages_requested);
 	to->cache_write += WT_STAT_READ(from, cache_write);
 	to->cache_write_restore += WT_STAT_READ(from, cache_write_restore);
+	to->cache_bytes_dirty += WT_STAT_READ(from, cache_bytes_dirty);
 	to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean);
 	to->cache_state_gen_avg_gap +=
 	    WT_STAT_READ(from, cache_state_gen_avg_gap);

src/utilities/util_dump.c

@@ -6,10 +6,14 @@
  * See the file LICENSE for redistribution information.
  */
 
+#include <assert.h>
 #include "util.h"
 #include "util_dump.h"
 
-static int dump_config(WT_SESSION *, const char *, bool, bool);
+#define	STRING_MATCH_CONFIG(s, item)					\
+	(strncmp(s, (item).str, (item).len) == 0 && (s)[(item).len] == '\0')
+
+static int dump_config(WT_SESSION *, const char *, WT_CURSOR *, bool, bool);
 static int dump_json_begin(WT_SESSION *);
 static int dump_json_end(WT_SESSION *);
 static int dump_json_separator(WT_SESSION *);
@@ -17,7 +21,8 @@ static int dump_json_table_end(WT_SESSION *);
 static int dump_prefix(WT_SESSION *, bool, bool);
 static int dump_record(WT_CURSOR *, bool, bool);
 static int dump_suffix(WT_SESSION *, bool);
-static int dump_table_config(WT_SESSION *, WT_CURSOR *, const char *, bool);
+static int dump_table_config(
+    WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, bool);
 static int dump_table_parts_config(
     WT_SESSION *, WT_CURSOR *, const char *, const char *, bool);
 static int dup_json_string(const char *, char **);
@@ -32,10 +37,11 @@ util_dump(WT_SESSION *session, int argc, char *argv[])
 	size_t len;
 	int ch, i;
 	bool hex, json, reverse;
-	char *checkpoint, *config, *name;
+	char *checkpoint, *config, *name, *p, *simplename;
 
 	hex = json = reverse = false;
-	checkpoint = config = name = NULL;
+	checkpoint = config = name = simplename = NULL;
+	cursor = NULL;
 	while ((ch = __wt_getopt(progname, argc, argv, "c:f:jrx")) != EOF)
 		switch (ch) {
 		case 'c':
@@ -84,14 +90,12 @@ util_dump(WT_SESSION *session, int argc, char *argv[])
 			if ((ret = dump_json_separator(session)) != 0)
 				goto err;
 		free(name);
-		name = NULL;
+		free(simplename);
+		name = simplename = NULL;
 
 		if ((name = util_name(session, argv[i], "table")) == NULL)
 			goto err;
 
-		if (dump_config(session, name, hex, json) != 0)
-			goto err;
-
 		len =
 		    checkpoint == NULL ? 0 : strlen("checkpoint=") +
 		    strlen(checkpoint) + 1;
@@ -115,10 +119,26 @@ util_dump(WT_SESSION *session, int argc, char *argv[])
 			goto err;
 		}
 
+		if ((simplename = strdup(name)) == NULL) {
+			ret = util_err(session, errno, NULL);
+			goto err;
+		}
+		if ((p = strchr(simplename, '(')) != NULL)
+			*p = '\0';
+		if (dump_config(session, simplename, cursor, hex, json) != 0)
+			goto err;
+
 		if ((ret = dump_record(cursor, reverse, json)) != 0)
 			goto err;
 		if (json && (ret = dump_json_table_end(session)) != 0)
 			goto err;
+
+		ret = cursor->close(cursor);
+		cursor = NULL;
+		if (ret != 0) {
+			ret = util_err(session, ret, NULL);
+			goto err;
+		}
 	}
 	if (json && ((ret = dump_json_end(session)) != 0))
 		goto err;
@@ -129,7 +149,11 @@ err:		ret = 1;
 
 	free(config);
 	free(name);
-
+	free(simplename);
+	if (cursor != NULL && (ret = cursor->close(cursor)) != 0) {
+		(void)util_err(session, ret, NULL);
+		ret = 1;
+	}
 	return (ret);
 }
 
@@ -138,15 +162,16 @@ err:		ret = 1;
  *	Dump the config for the uri.
  */
 static int
-dump_config(WT_SESSION *session, const char *uri, bool hex, bool json)
+dump_config(WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool hex,
+    bool json)
 {
-	WT_CURSOR *cursor;
+	WT_CURSOR *mcursor;
 	WT_DECL_RET;
 	int tret;
 
 	/* Open a metadata cursor. */
 	if ((ret = session->open_cursor(
-	    session, "metadata:create", NULL, NULL, &cursor)) != 0) {
+	    session, "metadata:create", NULL, NULL, &mcursor)) != 0) {
 		fprintf(stderr, "%s: %s: session.open_cursor: %s\n", progname,
 		    "metadata:create", session->strerror(session, ret));
 		return (1);
@@ -156,10 +181,11 @@ dump_config(WT_SESSION *session, const char *uri, bool hex, bool json)
 	 * want to output a header if the user entered the wrong name. This is
 	 * where we find out a table doesn't exist, use a simple error message.
 	 */
-	cursor->set_key(cursor, uri);
-	if ((ret = cursor->search(cursor)) == 0) {
+	mcursor->set_key(mcursor, uri);
+	if ((ret = mcursor->search(mcursor)) == 0) {
 		if ((!json && dump_prefix(session, hex, json) != 0) ||
-		    dump_table_config(session, cursor, uri, json) != 0 ||
+		    dump_table_config(session, mcursor, cursor,
+		    uri, json) != 0 ||
 		    dump_suffix(session, json) != 0)
 			ret = 1;
 	} else if (ret == WT_NOTFOUND)
@@ -167,8 +193,8 @@ dump_config(WT_SESSION *session, const char *uri, bool hex, bool json)
 	else
 		ret = util_err(session, ret, "%s", uri);
 
-	if ((tret = cursor->close(cursor)) != 0) {
-		tret = util_cerr(cursor, "close", tret);
+	if ((tret = mcursor->close(mcursor)) != 0) {
+		tret = util_cerr(mcursor, "close", tret);
 		if (ret == 0)
 			ret = tret;
 	}
@@ -224,17 +250,126 @@ dump_json_table_end(WT_SESSION *session)
 	return (0);
 }
 
+/*
+ * dump_add_config
+ *	Add a formatted config string to an output buffer.
+ */
+static int
+dump_add_config(WT_SESSION *session, char **bufp, size_t *leftp,
+    const char *fmt, ...)
+	WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5)))
+{
+	int n;
+	va_list ap;
+
+	va_start(ap, fmt);
+	n = vsnprintf(*bufp, *leftp, fmt, ap);
+	va_end(ap);
+	if (n < 0)
+		return (util_err(session, EINVAL, NULL));
+	*bufp += n;
+	*leftp -= (size_t)n;
+	return (0);
+}
+
+/*
+ * dump_projection --
+ *	Create a new config containing projection information.
+ */
+static int
+dump_projection(WT_SESSION *session, const char *config, WT_CURSOR *cursor,
+    char **newconfigp)
+{
+	WT_DECL_RET;
+	WT_CONFIG_ITEM key, value;
+	WT_CONFIG_PARSER *parser;
+	WT_EXTENSION_API *wt_api;
+	size_t len, vallen;
+	int nkeys;
+	char *newconfig;
+	const char *keyformat, *p;
+
+	len = strlen(config) + strlen(cursor->value_format) +
+	    strlen(cursor->uri) + 20;
+	if ((newconfig = malloc(len)) == NULL)
+		return util_err(session, errno, NULL);
+	*newconfigp = newconfig;
+	wt_api = session->connection->get_extension_api(session->connection);
+	if ((ret = wt_api->config_parser_open(wt_api, session, config,
+	    strlen(config), &parser)) != 0)
+		return (util_err(
+		    session, ret, "WT_EXTENSION_API.config_parser_open"));
+	keyformat = cursor->key_format;
+	for (nkeys = 0; *keyformat; keyformat++)
+		if (!__wt_isdigit((u_char)*keyformat))
+			nkeys++;
+
+	/*
+	 * Copy the configuration, replacing some fields to match the
+	 * projection.
+	 */
+	while ((ret = parser->next(parser, &key, &value)) == 0) {
+		WT_RET(dump_add_config(session, &newconfig, &len,
+		    "%.*s=", (int)key.len, key.str));
+		if (STRING_MATCH_CONFIG("value_format", key))
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "%s", cursor->value_format));
+		else if (STRING_MATCH_CONFIG("columns", key)) {
+			/* copy names of keys */
+			p = value.str;
+			vallen = value.len;
+			while (vallen > 0) {
+				if ((*p == ',' || *p == ')') && --nkeys == 0)
+					break;
+				p++;
+				vallen--;
+			}
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "%.*s", (int)(p - value.str), value.str));
+
+			/* copy names of projected values */
+			p = strchr(cursor->uri, '(');
+			assert(p != NULL);
+			assert(p[strlen(p) - 1] == ')');
+			p++;
+			if (*p != ')')
+				WT_RET(dump_add_config(session, &newconfig,
+				    &len, "%s", ","));
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "%.*s),", (int)(strlen(p) - 1), p));
+		} else if (value.type == WT_CONFIG_ITEM_STRING &&
+		    value.len != 0)
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "\"%.*s\",", (int)value.len, value.str));
+		else
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "%.*s,", (int)value.len, value.str));
+	}
+	if (ret != WT_NOTFOUND)
+		return (util_err(session, ret, "WT_CONFIG_PARSER.next"));
+
+	assert(len > 0);
+	if ((ret = parser->close(parser)) != 0)
+		return (util_err(
+		    session, ret, "WT_CONFIG_PARSER.close"));
+
+	return (0);
+}
+
 /*
  * dump_table_config --
  *	Dump the config for a table.
  */
 static int
 dump_table_config(
-    WT_SESSION *session, WT_CURSOR *cursor, const char *uri, bool json)
+    WT_SESSION *session, WT_CURSOR *mcursor, WT_CURSOR *cursor,
+    const char *uri, bool json)
 {
 	WT_DECL_RET;
+	char *proj_config;
 	const char *name, *v;
 
+	proj_config = NULL;
 	/* Get the table name. */
 	if ((name = strchr(uri, ':')) == NULL) {
 		fprintf(stderr, "%s: %s: corrupted uri\n", progname, uri);
@@ -246,20 +381,25 @@ dump_table_config(
 	 * Dump out the config information: first, dump the uri entry itself,
 	 * it overrides all subsequent configurations.
 	 */
-	cursor->set_key(cursor, uri);
-	if ((ret = cursor->search(cursor)) != 0)
-		return (util_cerr(cursor, "search", ret));
-	if ((ret = cursor->get_value(cursor, &v)) != 0)
-		return (util_cerr(cursor, "get_value", ret));
+	mcursor->set_key(mcursor, uri);
+	if ((ret = mcursor->search(mcursor)) != 0)
+		return (util_cerr(mcursor, "search", ret));
+	if ((ret = mcursor->get_value(mcursor, &v)) != 0)
+		return (util_cerr(mcursor, "get_value", ret));
 
-	WT_RET(print_config(session, uri, v, json, true));
+	if (strchr(cursor->uri, '(') != NULL) {
+		WT_ERR(dump_projection(session, v, cursor, &proj_config));
+		v = proj_config;
+	}
+	WT_ERR(print_config(session, uri, v, json, true));
 
-	WT_RET(dump_table_parts_config(
-	    session, cursor, name, "colgroup:", json));
-	WT_RET(dump_table_parts_config(
-	    session, cursor, name, "index:", json));
+	WT_ERR(dump_table_parts_config(
+	    session, mcursor, name, "colgroup:", json));
+	WT_ERR(dump_table_parts_config(
+	    session, mcursor, name, "index:", json));
 
-	return (0);
+err:	free(proj_config);
+	return (ret);
 }
 
 /*

test/suite/test_backup03.py

@@ -74,7 +74,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess):
         ('backup_9', dict(big=3,list=[])),         # Backup everything
     ]
 
-    scenarios = make_scenarios(list)
+    scenarios = make_scenarios(list, prune=3, prunelong=1000)
     # Create a large cache, otherwise this test runs quite slowly.
     conn_config = 'cache_size=1G'
 

test/suite/test_config03.py

@@ -71,7 +71,7 @@ class test_config03(test_base03.test_base03):
         cache_size_scenarios, create_scenarios, error_prefix_scenarios,
         eviction_target_scenarios, eviction_trigger_scenarios,
         multiprocess_scenarios, session_max_scenarios,
-        transactional_scenarios, verbose_scenarios, prune=1000)
+        transactional_scenarios, verbose_scenarios, prune=100, prunelong=1000)
 
     #wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \
     #                      str(len(scenarios)) + ' of ' + \

test/suite/test_dump.py

@@ -32,7 +32,7 @@ import wiredtiger, wttest
 from suite_subprocess import suite_subprocess
 from wtscenario import make_scenarios
 from wtdataset import SimpleDataSet, SimpleIndexDataSet, SimpleLSMDataSet, \
-    ComplexDataSet, ComplexLSMDataSet
+    ComplexDataSet, ComplexLSMDataSet, ProjectionDataSet, ProjectionIndexDataSet
 
 # test_dump.py
 #    Utilities: wt dump
@@ -62,6 +62,10 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
         ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)),
         ('table-complex', dict(uri='table:', dataset=ComplexDataSet)),
         ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)),
+        ('table-simple-proj', dict(uri='table:',
+            dataset=ProjectionDataSet, projection=True)),
+        ('table-index-proj', dict(uri='table:',
+            dataset=ProjectionIndexDataSet, projection=True)),
     ]
     scenarios = make_scenarios(types, keyfmt, dumpfmt)
 
@@ -158,5 +162,53 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
         pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt)
         pop.check()
 
+# test_dump_projection
+#    Utilities: wt dump
+# Test the dump utility with projections
+class test_dump_projection(wttest.WiredTigerTestCase, suite_subprocess):
+    dir = 'dump.dir'            # Backup directory name
+
+    name = 'test_dump'
+    nentries = 2500
+    uri = 'table:'
+
+    # Dump, re-load and do a content comparison.
+    def test_dump(self):
+
+        # Create the object.
+        uri = self.uri + self.name
+        pop = ProjectionDataSet(self, uri, self.nentries, key_format='S')
+        pop.populate()
+
+        # Check some cases with invalid projections.
+        self.runWt(['dump', '-x', uri + '('], \
+            outfilename='bad1.out', errfilename='err1.out', failure=True)
+        self.check_non_empty_file('err1.out')
+        self.runWt(['dump', '-x', uri + '(xx)'], \
+            outfilename='bad2.out', errfilename='err2.out', failure=True)
+        self.check_non_empty_file('err2.out')
+        self.runWt(['dump', '-x', uri + pop.projection[:-1]], \
+            outfilename='bad3.out', errfilename='err3.out', failure=True)
+        self.check_non_empty_file('err3.out')
+
+        # Dump the object with a valid projection.
+        self.runWt(['dump', '-x', uri + pop.projection], outfilename='dump.out')
+
+        # Re-load the object in a new home.
+        os.mkdir(self.dir)
+        self.runWt(['-h', self.dir, 'load', '-f', 'dump.out'])
+
+        # Check the database contents.
+        self.runWt(['list'], outfilename='list.out')
+        self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new')
+        s1 = set(open('list.out').read().split())
+        s2 = set(open('list.out.new').read().split())
+        self.assertEqual(not s1.symmetric_difference(s2), True)
+
+        # Check the object's contents.
+        self.reopen_conn(self.dir)
+        pop_reload = ProjectionDataSet(self, uri, self.nentries, key_format='S')
+        pop_reload.check()
+
 if __name__ == '__main__':
     wttest.run()

test/suite/test_intpack.py

@@ -126,8 +126,8 @@ class PackTester:
 class test_intpack(wttest.WiredTigerTestCase):
     name = 'test_intpack'
 
-    # We have to be a bit verbose here with naming, as there can be problems with
-    # case insensitive test names:w
+    # We have to be a bit verbose here with naming, scenario names are
+    # case insensitive and must be unique.
 
     scenarios = make_scenarios([
         ('int8_t_b', dict(formatcode='b', low=-128, high=127, nbits=8)),

test/suite/test_join01.py

@@ -69,7 +69,7 @@ class test_join01(wttest.WiredTigerTestCase):
     ]
     scenarios = make_scenarios(type_scen, bloom0_scen, bloom1_scen,
                                projection_scen, nested_scen, stats_scen,
-                               order_scen)
+                               order_scen, prune=50, prunelong=1000)
 
     # We need statistics for these tests.
     conn_config = 'statistics=(all)'

test/suite/test_jsondump02.py

@@ -234,6 +234,24 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess):
                 ('"ikey" : 4,\n"Skey" : "key4"',
                  '"S1" : "val16",\n"i2" : 16,\n"S3" : "val64",\n"i4" : 64'))
         self.check_json(self.table_uri4, table4_json)
+        # This projection has 3 value fields reversed with a key at the end.
+        table4_json_projection = (
+                ('"ikey" : 1,\n"Skey" : "key1"',
+                 '"i4" : 1,\n"S3" : "val1",\n"i2" : 1,\n"ikey" : 1'),
+                ('"ikey" : 2,\n"Skey" : "key2"',
+                 '"i4" : 8,\n"S3" : "val8",\n"i2" : 4,\n"ikey" : 2'),
+                ('"ikey" : 3,\n"Skey" : "key3"',
+                 '"i4" : 27,\n"S3" : "val27",\n"i2" : 9,\n"ikey" : 3'),
+                ('"ikey" : 4,\n"Skey" : "key4"',
+                 '"i4" : 64,\n"S3" : "val64",\n"i2" : 16,\n"ikey" : 4'))
+        # bad projection URI
+        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+                 lambda: self.check_json(self.table_uri4 + '(i4,S3,i2,ikey',
+                        table4_json_projection),
+            '/Unbalanced brackets/')
+        # This projection should work.
+        self.check_json(self.table_uri4 + '(i4,S3,i2,ikey)',
+                        table4_json_projection)
         # The dump config currently is not supported for the index type.
         self.check_json(uri4index1, (
                 ('"Skey" : "key1"',

test/suite/test_lsm01.py

@@ -57,7 +57,7 @@ class test_lsm01(wttest.WiredTigerTestCase):
     scenarios = wtscenario.make_scenarios(
         chunk_size_scenarios, merge_max_scenarios, bloom_scenarios,
         bloom_bit_scenarios, bloom_hash_scenarios, record_count_scenarios,
-        prune=500)
+        prune=100, prunelong=500)
 
     # Test drop of an object.
     def test_lsm(self):

test/suite/test_perf001.py

@@ -40,7 +40,8 @@ class test_perf001(wttest.WiredTigerTestCase):
 
     scenarios = make_scenarios([
         #('file-file', dict(tabletype='file',indextype='file')),
-        ('file-lsm', dict(tabletype='file',indextype='lsm')),
+        ('file-lsm', dict(tabletype='file',indextype='lsm', cfg='',
+            conn_config="statistics=(fast),statistics_log=(wait=1)")),
         #('lsm-file', dict(tabletype='lsm',indextype='file')),
         #('lsm-lsm', dict(tabletype='lsm',indextype='lsm')),
     ])

test/suite/test_truncate01.py

@@ -183,11 +183,11 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
     # those tests to file objects.
     types = [
         ('file', dict(type='file:', valuefmt='S',
-            config='allocation_size=512,leaf_page_max=512')),
+            config='allocation_size=512,leaf_page_max=512', P=0.25)),
         ('file8t', dict(type='file:', valuefmt='8t',
-            config='allocation_size=512,leaf_page_max=512')),
+            config='allocation_size=512,leaf_page_max=512', P=0.25)),
         ('table', dict(type='table:', valuefmt='S',
-            config='allocation_size=512,leaf_page_max=512')),
+            config='allocation_size=512,leaf_page_max=512', P=0.5)),
     ]
     keyfmt = [
         ('integer', dict(keyfmt='i')),
@@ -203,7 +203,8 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
         ('big', dict(nentries=1000,skip=37)),
     ]
 
-    scenarios = make_scenarios(types, keyfmt, size, reopen)
+    scenarios = make_scenarios(types, keyfmt, size, reopen,
+        prune=10, prunelong=1000)
 
     # Set a cursor key.
     def cursorKey(self, ds, uri, key):

test/suite/test_truncate02.py

@@ -85,7 +85,8 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase):
         ('txn2', dict(commit=False)),
         ]
 
-    scenarios = make_scenarios(types, keyfmt, overflow, reads, writes, txn)
+    scenarios = make_scenarios(types, keyfmt, overflow, reads, writes, txn,
+                               prune=20, prunelong=1000)
 
     # Return the number of records visible to the cursor; test both forward
     # and backward iteration, they are different code paths in this case.

test/suite/test_txn07.py

@@ -70,7 +70,8 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess):
         ('none', dict(compress='')),
     ]
 
-    scenarios = make_scenarios(types, op1s, txn1s, compress)
+    scenarios = make_scenarios(types, op1s, txn1s, compress,
+                               prune=30, prunelong=1000)
     # Overrides WiredTigerTestCase
     def setUpConnectionOpen(self, dir):
         self.home = dir

test/suite/wtdataset.py

@@ -41,6 +41,7 @@ class BaseDataSet(object):
         self.key_format = kwargs.get('key_format', 'S')
         self.value_format = kwargs.get('value_format', 'S')
         self.config = kwargs.get('config', '')
+        self.projection = kwargs.get('projection', '')
 
     def create(self):
         self.testcase.session.create(self.uri, 'key_format=' + self.key_format
@@ -103,7 +104,8 @@ class BaseDataSet(object):
 
     def check(self):
         self.testcase.pr('check: ' + self.uri)
-        cursor = self.testcase.session.open_cursor(self.uri, None)
+        cursor = self.testcase.session.open_cursor(
+            self.uri + self.projection, None, None)
         self.check_cursor(cursor)
         cursor.close()
 
@@ -289,6 +291,94 @@ class ComplexLSMDataSet(ComplexDataSet):
     def is_lsm(cls):
         return True
 
+class ProjectionDataSet(SimpleDataSet):
+    """
+    ProjectionDataSet creates a table with predefined data identical to
+    SimpleDataSet (single key and value), but when checking it, uses
+    a cursor with a projection.
+    """
+    def __init__(self, testcase, uri, rows, **kwargs):
+        kwargs['config'] = kwargs.get('config', '') + ',columns=(k,v0)'
+        kwargs['projection'] = '(v0,v0,v0)'
+        super(ProjectionDataSet, self).__init__(testcase, uri, rows, **kwargs)
+
+    # A value suitable for checking the value returned by a cursor.
+    def comparable_value(self, i):
+        v0 = self.value(i)
+        return [v0, v0, v0]
+
+    def check_cursor(self, cursor):
+        i = 0
+        for key, got0, got1, got2 in cursor:
+            i += 1
+            self.testcase.assertEqual(key, self.key(i))
+            if cursor.value_format == '8t' and got0 == 0:    # deleted
+                continue
+            self.testcase.assertEqual([got0, got1, got2],
+                self.comparable_value(i))
+        self.testcase.assertEqual(i, self.rows)
+
+class ProjectionIndexDataSet(BaseDataSet):
+    """
+    ProjectionIndexDataSet creates a table with three values and
+    an index.  Checks are made against a projection of the main table
+    and a projection of the index.
+    """
+    def __init__(self, testcase, uri, rows, **kwargs):
+        self.origconfig = kwargs.get('config', '')
+        self.indexname = 'index:' + uri.split(":")[1] +  ':index0'
+        kwargs['config'] = self.origconfig + ',columns=(k,v0,v1,v2)'
+        kwargs['value_format'] = kwargs.get('value_format', 'SiS')
+        kwargs['projection'] = '(v1,v2,v0)'
+        super(ProjectionIndexDataSet, self).__init__(
+            testcase, uri, rows, **kwargs)
+
+    def value(self, i):
+        return ('v0:' + str(i), i*i, 'v2:' + str(i))
+
+    # Suitable for checking the value returned by a cursor using a projection.
+    def comparable_value(self, i):
+        return [i*i, 'v2:' + str(i), 'v0:' + str(i)]
+
+    def create(self):
+        super(ProjectionIndexDataSet, self).create()
+        self.testcase.session.create(self.indexname, 'columns=(v2,v1),' +
+            self.origconfig)
+
+    def check_cursor(self, cursor):
+        i = 0
+        for key, got0, got1, got2 in cursor:
+            i += 1
+            self.testcase.assertEqual(key, self.key(i))
+            if cursor.value_format == '8t' and got0 == 0:    # deleted
+                continue
+            self.testcase.assertEqual([got0, got1, got2],
+                self.comparable_value(i))
+        self.testcase.assertEqual(i, self.rows)
+
+    def check_index_cursor(self, cursor):
+        for i in xrange(1, self.rows + 1):
+            k = self.key(i)
+            v = self.value(i)
+            ik = (v[2], v[1])  # The index key is (v2,v2)
+            expect = [v[1],k,v[2],v[0]]
+            self.testcase.assertEqual(expect, cursor[ik])
+
+    def check(self):
+        BaseDataSet.check(self)
+
+        # Check values in the index.
+        idxcursor = self.testcase.session.open_cursor(
+            self.indexname + '(v1,k,v2,v0)')
+        self.check_index_cursor(idxcursor)
+        idxcursor.close()
+
+    def index_count(self):
+        return 1
+
+    def index_name(self, i):
+        return self.indexname
+
 # create a key based on a cursor as a shortcut to creating a SimpleDataSet
 def simple_key(cursor, i):
     return BaseDataSet.key_by_format(i, cursor.key_format)

test/suite/wttest.py

@@ -287,6 +287,7 @@ class WiredTigerTestCase(unittest.TestCase):
             self.testsubdir = self.className() + '.' + str(self.__class__.wt_ntests)
         self.testdir = os.path.join(WiredTigerTestCase._parentTestdir, self.testsubdir)
         self.__class__.wt_ntests += 1
+        self.starttime = time.time()
         if WiredTigerTestCase._verbose > 2:
             self.prhead('started in ' + self.testdir, True)
         # tearDown needs connections list, set it here in case the open fails.
@@ -355,6 +356,9 @@ class WiredTigerTestCase(unittest.TestCase):
         else:
             self.pr('preserving directory ' + self.testdir)
 
+        elapsed = time.time() - self.starttime
+        if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2:
+            print "%s: %.2f seconds" % (str(self), elapsed)
         if not passed and not skipped:
             print "ERROR in " + str(self)
             self.pr('FAIL')

tools/wtstats/stat_data.py

@@ -94,6 +94,7 @@ no_scale_per_second_list = [
     'btree: row-store leaf pages',
     'cache: bytes currently in the cache',
     'cache: overflow values cached in memory',
+    'cache: tracked dirty bytes in the cache',
     'cache_walk: Average difference between current eviction generation when the page was last considered',
     'cache_walk: Average on-disk page image size seen',
     'cache_walk: Clean pages currently in cache',
@@ -186,6 +187,7 @@ no_clear_list = [
     'transaction: transaction range of IDs currently pinned by named snapshots',
     'btree: btree checkpoint generation',
     'cache: bytes currently in the cache',
+    'cache: tracked dirty bytes in the cache',
     'cache_walk: Average difference between current eviction generation when the page was last considered',
     'cache_walk: Average on-disk page image size seen',
     'cache_walk: Clean pages currently in cache',