Update build system files for jemalloc

8 files changed, 364 insertions, 2 deletions

diff --git a/memory/jemalloc/0001-Dont-overwrite-VERSION-on-a-git-repository.patch b/memory/jemalloc/0001-Dont-overwrite-VERSION-on-a-git-repository.patch
new file mode 100644
index 000000000..34f445327
--- /dev/null
+++ b/memory/jemalloc/0001-Dont-overwrite-VERSION-on-a-git-repository.patch
@@ -0,0 +1,68 @@
+diff --git a/configure b/configure
+--- a/configure
++++ b/configure
+@@ -6662,28 +6662,6 @@ fi
+ 
+ 
+ 
+-if test "x`git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then
+-        rm -f "${srcroot}VERSION"
+-  for pattern in '[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \
+-                 '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \
+-                 '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \
+-                 '[0-9][0-9].[0-9][0-9].[0-9]' \
+-                 '[0-9][0-9].[0-9][0-9].[0-9][0-9]'; do
+-    if test ! -e "${srcroot}VERSION" ; then
+-      git describe --long --abbrev=40 --match="${pattern}" > "${srcroot}VERSION.tmp" 2>/dev/null
+-      if test $? -eq 0 ; then
+-        mv "${srcroot}VERSION.tmp" "${srcroot}VERSION"
+-        break
+-      fi
+-    fi
+-  done
+-fi
+-rm -f "${srcroot}VERSION.tmp"
+-if test ! -e "${srcroot}VERSION" ; then
+-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: Missing VERSION file, and unable to generate it; creating bogus VERSION" >&5
+-$as_echo "Missing VERSION file, and unable to generate it; creating bogus VERSION" >&6; }
+-  echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${srcroot}VERSION"
+-fi
+ jemalloc_version=`cat "${srcroot}VERSION"`
+ jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $1}'`
+ jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $2}'`
+diff --git a/configure.ac b/configure.ac
+--- a/configure.ac
++++ b/configure.ac
+@@ -1055,32 +1055,6 @@ dnl ============================================================================
+ dnl jemalloc configuration.
+ dnl 
+ 
+-dnl Set VERSION if source directory is inside a git repository.
+-if test "x`git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then
+-  dnl Pattern globs aren't powerful enough to match both single- and
+-  dnl double-digit version numbers, so iterate over patterns to support up to
+-  dnl version 99.99.99 without any accidental matches.
+-  rm -f "${srcroot}VERSION"
+-  for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \
+-                 '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \
+-                 '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \
+-                 '[0-9][0-9].[0-9][0-9].[0-9]' \
+-                 '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do
+-    if test ! -e "${srcroot}VERSION" ; then
+-      git describe --long --abbrev=40 --match="${pattern}" > "${srcroot}VERSION.tmp" 2>/dev/null
+-      if test $? -eq 0 ; then
+-        mv "${srcroot}VERSION.tmp" "${srcroot}VERSION"
+-        break
+-      fi
+-    fi
+-  done
+-fi
+-rm -f "${srcroot}VERSION.tmp"
+-if test ! -e "${srcroot}VERSION" ; then
+-  AC_MSG_RESULT(
+-    [Missing VERSION file, and unable to generate it; creating bogus VERSION])
+-  echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${srcroot}VERSION"
+-fi
+ jemalloc_version=`cat "${srcroot}VERSION"`
+ jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'`
+ jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'`
diff --git a/memory/jemalloc/0002-Move-variable-declaration-to-the-top-its-block-for-M.patch b/memory/jemalloc/0002-Move-variable-declaration-to-the-top-its-block-for-M.patch
new file mode 100644
index 000000000..83a75ac69
--- /dev/null
+++ b/memory/jemalloc/0002-Move-variable-declaration-to-the-top-its-block-for-M.patch
@@ -0,0 +1,35 @@
+From 9c6a8d3b0cc14fd26b119ad08f190e537771464f Mon Sep 17 00:00:00 2001
+From: Guilherme Goncalves <guilherme.p.gonc@gmail.com>
+Date: Wed, 17 Dec 2014 14:46:35 -0200
+Subject: [PATCH] Move variable declaration to the top its block for MSVC
+ compatibility.
+
+---
+ src/arena.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/arena.c b/src/arena.c
+index bf78995..1eb4000 100644
+--- a/src/arena.c
++++ b/src/arena.c
+@@ -2022,6 +2022,7 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
+ 		 * following run, then merge the first part with the existing
+ 		 * allocation.
+ 		 */
++		arena_run_t *run;
+ 		size_t flag_dirty, splitsize, usize;
+ 
+ 		usize = s2u(size + extra);
+@@ -2030,8 +2031,7 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
+ 		assert(usize >= usize_min);
+ 		splitsize = usize - oldsize;
+ 
+-		arena_run_t *run = &arena_miscelm_get(chunk,
+-		    pageind+npages)->run;
++		run = &arena_miscelm_get(chunk, pageind+npages)->run;
+ 		arena_run_split_large(arena, run, splitsize, zero);
+ 
+ 		size = oldsize + splitsize;
+-- 
+2.1.3
+
diff --git a/memory/jemalloc/0003-Add-a-isblank-definition-for-MSVC-2013.patch b/memory/jemalloc/0003-Add-a-isblank-definition-for-MSVC-2013.patch
new file mode 100644
index 000000000..6b75dd367
--- /dev/null
+++ b/memory/jemalloc/0003-Add-a-isblank-definition-for-MSVC-2013.patch
@@ -0,0 +1,30 @@
+From 4dd4193c59ff3f77e4ab36214ec63425ca834323 Mon Sep 17 00:00:00 2001
+From: Guilherme Goncalves <guilherme.p.gonc@gmail.com>
+Date: Thu, 18 Dec 2014 15:01:21 +0900
+Subject: [PATCH] Add a isblank definition for MSVC < 2013
+
+---
+ include/jemalloc/internal/jemalloc_internal_decls.h | 7 +++++++
+ 1 file changed, 7 insertions(+)
+
+diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h
+index fb2effb..65f2e4b 100644
+--- a/include/jemalloc/internal/jemalloc_internal_decls.h
++++ b/include/jemalloc/internal/jemalloc_internal_decls.h
+@@ -52,6 +52,13 @@ typedef intptr_t ssize_t;
+ #  define __func__ __FUNCTION__
+ /* Disable warnings about deprecated system functions. */
+ #  pragma warning(disable: 4996)
++#if _MSC_VER < 1800
++static int
++isblank(int c)
++{
++	return (c == '\t' || c == ' ');
++}
++#endif
+ #else
+ #  include <unistd.h>
+ #endif
+-- 
+2.1.3
+
diff --git a/memory/jemalloc/0004-Implement-stats.bookkeeping.patch b/memory/jemalloc/0004-Implement-stats.bookkeeping.patch
new file mode 100644
index 000000000..d3b3a8361
--- /dev/null
+++ b/memory/jemalloc/0004-Implement-stats.bookkeeping.patch
@@ -0,0 +1,158 @@
+commit a8a2e2b34e21870733c75121f431530d42d4d3cd
+Author: Guilherme Goncalves <guilherme.p.gonc@gmail.com>
+Date:   Tue Dec 23 15:21:04 2014 -0200
+
+    Implement stats.bookkeeping.
+    
+    This measures the number of bytes used for arena and chunk headers and all base
+    allocations (note that arena headers are already allocated through the base
+    allocator). It does not account for internal allocations made through arenas.
+
+diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h
+index 3fb80b9..9dc431c 100644
+--- a/include/jemalloc/internal/base.h
++++ b/include/jemalloc/internal/base.h
+@@ -9,6 +9,10 @@
+ /******************************************************************************/
+ #ifdef JEMALLOC_H_EXTERNS
+ 
++/* base_mtx synchronizes base allocations and protects base_allocated */
++extern malloc_mutex_t	base_mtx;
++extern size_t base_allocated;
++
+ void	*base_alloc(size_t size);
+ void	*base_calloc(size_t number, size_t size);
+ extent_node_t *base_node_alloc(void);
+diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h
+index a3e899e..cbb0923 100644
+--- a/include/jemalloc/internal/ctl.h
++++ b/include/jemalloc/internal/ctl.h
+@@ -52,6 +52,7 @@ struct ctl_arena_stats_s {
+ struct ctl_stats_s {
+ 	size_t			allocated;
+ 	size_t			active;
++	size_t			bookkeeping;
+ 	size_t			mapped;
+ 	struct {
+ 		size_t		current;	/* stats_chunks.curchunks */
+diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
+index ee973c9..93271ce 100644
+--- a/include/jemalloc/internal/private_symbols.txt
++++ b/include/jemalloc/internal/private_symbols.txt
+@@ -91,8 +91,10 @@ atomic_sub_uint32
+ atomic_sub_uint64
+ atomic_sub_z
+ base_alloc
++base_allocated
+ base_boot
+ base_calloc
++base_mtx
+ base_node_alloc
+ base_node_dalloc
+ base_postfork_child
+diff --git a/src/base.c b/src/base.c
+index 409c7bb..3350ca3 100644
+--- a/src/base.c
++++ b/src/base.c
+@@ -4,7 +4,7 @@
+ /******************************************************************************/
+ /* Data. */
+ 
+-static malloc_mutex_t	base_mtx;
++malloc_mutex_t	base_mtx;
+ 
+ /*
+  * Current pages that are being used for internal memory allocations.  These
+@@ -16,6 +16,8 @@ static void		*base_next_addr;
+ static void		*base_past_addr; /* Addr immediately past base_pages. */
+ static extent_node_t	*base_nodes;
+ 
++size_t base_allocated;
++
+ /******************************************************************************/
+ 
+ static bool
+@@ -54,6 +56,8 @@ base_alloc(size_t size)
+ 	/* Allocate. */
+ 	ret = base_next_addr;
+ 	base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
++	if (config_stats)
++		base_allocated += csize;
+ 	malloc_mutex_unlock(&base_mtx);
+ 	JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize);
+ 
+diff --git a/src/ctl.c b/src/ctl.c
+index b367c9f..1999145 100644
+--- a/src/ctl.c
++++ b/src/ctl.c
+@@ -187,6 +187,7 @@ INDEX_PROTO(stats_arenas_i)
+ CTL_PROTO(stats_cactive)
+ CTL_PROTO(stats_allocated)
+ CTL_PROTO(stats_active)
++CTL_PROTO(stats_bookkeeping)
+ CTL_PROTO(stats_mapped)
+ 
+ /******************************************************************************/
+@@ -448,12 +449,13 @@ static const ctl_indexed_node_t stats_arenas_node[] = {
+ };
+ 
+ static const ctl_named_node_t stats_node[] = {
+-	{NAME("cactive"),	CTL(stats_cactive)},
+-	{NAME("allocated"),	CTL(stats_allocated)},
+-	{NAME("active"),	CTL(stats_active)},
+-	{NAME("mapped"),	CTL(stats_mapped)},
+-	{NAME("chunks"),	CHILD(named, stats_chunks)},
+-	{NAME("arenas"),	CHILD(indexed, stats_arenas)}
++	{NAME("cactive"),		CTL(stats_cactive)},
++	{NAME("allocated"),		CTL(stats_allocated)},
++	{NAME("active"),		CTL(stats_active)},
++	{NAME("bookkeeping"),	CTL(stats_bookkeeping)},
++	{NAME("mapped"),		CTL(stats_mapped)},
++	{NAME("chunks"),		CHILD(named, stats_chunks)},
++	{NAME("arenas"),		CHILD(indexed, stats_arenas)}
+ };
+ 
+ static const ctl_named_node_t	root_node[] = {
+@@ -705,6 +707,12 @@ ctl_refresh(void)
+ 		ctl_stats.active =
+ 		    (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE);
+ 		ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk);
++
++		/* add chunk headers to bookkeeping */
++		ctl_stats.bookkeeping = ctl_stats.chunks.current * (map_bias << LG_PAGE);
++		malloc_mutex_lock(&base_mtx);
++		ctl_stats.bookkeeping += base_allocated;
++		malloc_mutex_unlock(&base_mtx);
+ 	}
+ 
+ 	ctl_epoch++;
+@@ -1806,6 +1814,7 @@ CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t)
+ CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *)
+ CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t)
+ CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t)
++CTL_RO_CGEN(config_stats, stats_bookkeeping, ctl_stats.bookkeeping, size_t)
+ CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t)
+ 
+ CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current,
+diff --git a/test/unit/stats.c b/test/unit/stats.c
+index 946e737..e84cbf7 100644
+--- a/test/unit/stats.c
++++ b/test/unit/stats.c
+@@ -3,7 +3,7 @@
+ TEST_BEGIN(test_stats_summary)
+ {
+ 	size_t *cactive;
+-	size_t sz, allocated, active, mapped;
++	size_t sz, allocated, active, mapped, bookkeeping;
+ 	int expected = config_stats ? 0 : ENOENT;
+ 
+ 	sz = sizeof(cactive);
+@@ -17,6 +17,8 @@ TEST_BEGIN(test_stats_summary)
+ 	    "Unexpected mallctl() result");
+ 	assert_d_eq(mallctl("stats.mapped", &mapped, &sz, NULL, 0), expected,
+ 	    "Unexpected mallctl() result");
++	assert_d_eq(mallctl("stats.bookkeeping", &bookkeeping, &sz, NULL, 0),
++	    expected, "Unexpected mallctl() result");
+ 
+ 	if (config_stats) {
+ 		assert_zu_le(active, *cactive,
diff --git a/memory/jemalloc/0005-Bug-1121314-Avoid-needing-the-arena-in-chunk_alloc_d.patch b/memory/jemalloc/0005-Bug-1121314-Avoid-needing-the-arena-in-chunk_alloc_d.patch
new file mode 100644
index 000000000..89b84dcbb
--- /dev/null
+++ b/memory/jemalloc/0005-Bug-1121314-Avoid-needing-the-arena-in-chunk_alloc_d.patch
@@ -0,0 +1,58 @@
+From 722400adde3b4cb307fdc310e6296e10e6ace7ae Mon Sep 17 00:00:00 2001
+From: Mike Hommey <mh+mozilla@glandium.org>
+Date: Wed, 14 Jan 2015 12:49:24 +0900
+Subject: [PATCH] Bug 1121314 - Avoid needing the arena in chunk_alloc_default
+ to avoid possible infinite loops involving a0malloc
+
+---
+ src/chunk.c | 21 +++++++++++++--------
+ 1 file changed, 13 insertions(+), 8 deletions(-)
+
+diff --git a/src/chunk.c b/src/chunk.c
+index 7926452..32e659c 100644
+--- a/src/chunk.c
++++ b/src/chunk.c
+@@ -247,27 +247,32 @@ chunk_alloc_arena(chunk_alloc_t *chunk_alloc, chunk_dalloc_t *chunk_dalloc,
+ 	return (ret);
+ }
+ 
+ /* Default arena chunk allocation routine in the absence of user override. */
+ void *
+ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
+     unsigned arena_ind)
+ {
+-	arena_t *arena;
++	dss_prec_t dss_prec = dss_prec_disabled;
+ 
+-	arena = arena_get(tsd_fetch(), arena_ind, false, true);
+-	/*
+-	 * The arena we're allocating on behalf of must have been initialized
+-	 * already.
+-	 */
+-	assert(arena != NULL);
++	if (have_dss) {
++		arena_t *arena;
++
++		arena = arena_get(tsd_fetch(), arena_ind, false, true);
++		/*
++		 * The arena we're allocating on behalf of must have been
++		 * initialized already.
++		 */
++		assert(arena != NULL);
++		dss_prec = arena->dss_prec;
++	}
+ 
+ 	return (chunk_alloc_core(new_addr, size, alignment, false, zero,
+-	    arena->dss_prec));
++	    dss_prec));
+ }
+ 
+ static void
+ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
+     size_t size)
+ {
+ 	bool unzeroed;
+ 	extent_node_t *xnode, *node, *prev, *xprev, key;
+-- 
+2.2.1.dirty
+
diff --git a/memory/jemalloc/Makefile.in b/memory/jemalloc/Makefile.in
index 42b083e34..6a0894eec 100644
--- a/memory/jemalloc/Makefile.in
+++ b/memory/jemalloc/Makefile.in
@@ -33,7 +33,11 @@ LOCAL_INCLUDES += \
 
 ifdef _MSC_VER
 LOCAL_INCLUDES += -I$(srcdir)/src/include/msvc_compat
-
+ifndef HAVE_INTTYPES_H
+LOCAL_INCLUDES += \
+  -I$(srcdir)/src/msvc_compat/C99 \
+  $(NULL)
+endif
 DEFINES += -DDLLEXPORT
 endif
 
diff --git a/memory/jemalloc/moz.build b/memory/jemalloc/moz.build
index 94f119943..91e3d5a1c 100644
--- a/memory/jemalloc/moz.build
+++ b/memory/jemalloc/moz.build
@@ -29,6 +29,8 @@ CSRCS += [
     'tcache.c',
     'tsd.c',
     'util.c',
+    # Do we ever want valgrind?
+    # 'valgrind.c',
 ]
 
 # Only OSX needs the zone allocation implementation,
diff --git a/memory/jemalloc/update.sh b/memory/jemalloc/update.sh
index 3803b2df1..0f7631545 100755
--- a/memory/jemalloc/update.sh
+++ b/memory/jemalloc/update.sh
@@ -12,7 +12,14 @@ cd src
 git checkout "$UPSTREAM_COMMIT"
 autoconf
 git describe --long --abbrev=40 > VERSION
-rm -rf .git .gitignore autom4te.cache
+rm -rf .git .gitignore .gitattributes autom4te.cache .autom4te.cfg
+
+patch -p1 < ../0001-Dont-overwrite-VERSION-on-a-git-repository.patch
+patch -p1 < ../0002-Move-variable-declaration-to-the-top-its-block-for-M.patch
+patch -p1 < ../0003-Add-a-isblank-definition-for-MSVC-2013.patch
+patch -p1 < ../0004-Implement-stats.bookkeeping.patch
+patch -p1 < ../0005-Bug-1121314-Avoid-needing-the-arena-in-chunk_alloc_d.patch
+
 cd ..
 hg addremove -q src