LCOV - code coverage report
Current view: directory - memory/jemalloc - jemalloc.c (source / functions) Found Hit Coverage
Test: app.info Lines: 1540 1073 69.7 %
Date: 2012-06-02 Functions: 139 89 64.0 %

       1                 : /* -*- Mode: C; tab-width: 8; c-basic-offset: 8; indent-tabs-mode: t -*- */
       2                 : /* vim:set softtabstop=8 shiftwidth=8 noet: */
       3                 : /*-
       4                 :  * Copyright (C) 2006-2008 Jason Evans <jasone@FreeBSD.org>.
       5                 :  * All rights reserved.
       6                 :  *
       7                 :  * Redistribution and use in source and binary forms, with or without
       8                 :  * modification, are permitted provided that the following conditions
       9                 :  * are met:
      10                 :  * 1. Redistributions of source code must retain the above copyright
      11                 :  *    notice(s), this list of conditions and the following disclaimer as
      12                 :  *    the first lines of this file unmodified other than the possible
      13                 :  *    addition of one or more copyright notices.
      14                 :  * 2. Redistributions in binary form must reproduce the above copyright
      15                 :  *    notice(s), this list of conditions and the following disclaimer in
      16                 :  *    the documentation and/or other materials provided with the
      17                 :  *    distribution.
      18                 :  *
      19                 :  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
      20                 :  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
      21                 :  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
      22                 :  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
      23                 :  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
      24                 :  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
      25                 :  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
      26                 :  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
      27                 :  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
      28                 :  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
      29                 :  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
      30                 :  *
      31                 :  *******************************************************************************
      32                 :  *
      33                 :  * This allocator implementation is designed to provide scalable performance
      34                 :  * for multi-threaded programs on multi-processor systems.  The following
      35                 :  * features are included for this purpose:
      36                 :  *
      37                 :  *   + Multiple arenas are used if there are multiple CPUs, which reduces lock
      38                 :  *     contention and cache sloshing.
      39                 :  *
      40                 :  *   + Cache line sharing between arenas is avoided for internal data
      41                 :  *     structures.
      42                 :  *
      43                 :  *   + Memory is managed in chunks and runs (chunks can be split into runs),
      44                 :  *     rather than as individual pages.  This provides a constant-time
      45                 :  *     mechanism for associating allocations with particular arenas.
      46                 :  *
      47                 :  * Allocation requests are rounded up to the nearest size class, and no record
      48                 :  * of the original request size is maintained.  Allocations are broken into
      49                 :  * categories according to size class.  Assuming runtime defaults, 4 kB pages
      50                 :  * and a 16 byte quantum on a 32-bit system, the size classes in each category
      51                 :  * are as follows:
      52                 :  *
      53                 :  *   |=====================================|
      54                 :  *   | Category | Subcategory    |    Size |
      55                 :  *   |=====================================|
      56                 :  *   | Small    | Tiny           |       2 |
      57                 :  *   |          |                |       4 |
      58                 :  *   |          |                |       8 |
      59                 :  *   |          |----------------+---------|
      60                 :  *   |          | Quantum-spaced |      16 |
      61                 :  *   |          |                |      32 |
      62                 :  *   |          |                |      48 |
      63                 :  *   |          |                |     ... |
      64                 :  *   |          |                |     480 |
      65                 :  *   |          |                |     496 |
      66                 :  *   |          |                |     512 |
      67                 :  *   |          |----------------+---------|
      68                 :  *   |          | Sub-page       |    1 kB |
      69                 :  *   |          |                |    2 kB |
      70                 :  *   |=====================================|
      71                 :  *   | Large                     |    4 kB |
      72                 :  *   |                           |    8 kB |
      73                 :  *   |                           |   12 kB |
      74                 :  *   |                           |     ... |
      75                 :  *   |                           | 1012 kB |
      76                 :  *   |                           | 1016 kB |
      77                 :  *   |                           | 1020 kB |
      78                 :  *   |=====================================|
      79                 :  *   | Huge                      |    1 MB |
      80                 :  *   |                           |    2 MB |
      81                 :  *   |                           |    3 MB |
      82                 :  *   |                           |     ... |
      83                 :  *   |=====================================|
      84                 :  *
      85                 :  * NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an
      86                 :  * allocation on Linux or Mac.  So on 32-bit *nix, the smallest bucket size is
      87                 :  * 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes.
      88                 :  *
      89                 :  * A different mechanism is used for each category:
      90                 :  *
      91                 :  *   Small : Each size class is segregated into its own set of runs.  Each run
      92                 :  *           maintains a bitmap of which regions are free/allocated.
      93                 :  *
      94                 :  *   Large : Each allocation is backed by a dedicated run.  Metadata are stored
      95                 :  *           in the associated arena chunk header maps.
      96                 :  *
      97                 :  *   Huge : Each allocation is backed by a dedicated contiguous set of chunks.
      98                 :  *          Metadata are stored in a separate red-black tree.
      99                 :  *
     100                 :  *******************************************************************************
     101                 :  */
     102                 : 
     103                 : #ifdef MOZ_MEMORY_ANDROID
     104                 : #define NO_TLS
     105                 : #define _pthread_self() pthread_self()
     106                 : #endif
     107                 : 
     108                 : /*
     109                 :  * On Linux, we use madvise(MADV_DONTNEED) to release memory back to the
     110                 :  * operating system.  If we release 1MB of live pages with MADV_DONTNEED, our
     111                 :  * RSS will decrease by 1MB (almost) immediately.
     112                 :  *
     113                 :  * On Mac, we use madvise(MADV_FREE).  Unlike MADV_DONTNEED on Linux, MADV_FREE
     114                 :  * on Mac doesn't cause the OS to release the specified pages immediately; the
     115                 :  * OS keeps them in our process until the machine comes under memory pressure.
     116                 :  *
     117                 :  * It's therefore difficult to measure the process's RSS on Mac, since, in the
     118                 :  * absence of memory pressure, the contribution from the heap to RSS will not
     119                 :  * decrease due to our madvise calls.
     120                 :  *
     121                 :  * We therefore define MALLOC_DOUBLE_PURGE on Mac.  This causes jemalloc to
     122                 :  * track which pages have been MADV_FREE'd.  You can then call
     123                 :  * jemalloc_purge_freed_pages(), which will force the OS to release those
     124                 :  * MADV_FREE'd pages, making the process's RSS reflect its true memory usage.
     125                 :  *
     126                 :  */
     127                 : #ifdef MOZ_MEMORY_DARWIN
     128                 : #define MALLOC_DOUBLE_PURGE
     129                 : #endif
     130                 : 
     131                 : /*
     132                 :  * MALLOC_PRODUCTION disables assertions and statistics gathering.  It also
     133                 :  * defaults the A and J runtime options to off.  These settings are appropriate
     134                 :  * for production systems.
     135                 :  */
     136                 : #ifndef MOZ_MEMORY_DEBUG
     137                 : #  define       MALLOC_PRODUCTION
     138                 : #endif
     139                 : 
     140                 : /*
     141                 :  * Use only one arena by default.  Mozilla does not currently make extensive
     142                 :  * use of concurrent allocation, so the increased fragmentation associated with
     143                 :  * multiple arenas is not warranted.
     144                 :  */
     145                 : #define MOZ_MEMORY_NARENAS_DEFAULT_ONE
     146                 : 
     147                 : /*
     148                 :  * MALLOC_STATS enables statistics calculation, and is required for
     149                 :  * jemalloc_stats().
     150                 :  */
     151                 : #define MALLOC_STATS
     152                 : 
     153                 : #ifndef MALLOC_PRODUCTION
     154                 :    /*
     155                 :     * MALLOC_DEBUG enables assertions and other sanity checks, and disables
     156                 :     * inline functions.
     157                 :     */
     158                 : #  define MALLOC_DEBUG
     159                 : 
     160                 :    /* Memory filling (junk/zero). */
     161                 : #  define MALLOC_FILL
     162                 : 
     163                 :    /* Allocation tracing. */
     164                 : #  ifndef MOZ_MEMORY_WINDOWS
     165                 : #    define MALLOC_UTRACE
     166                 : #  endif
     167                 : 
     168                 :    /* Support optional abort() on OOM. */
     169                 : #  define MALLOC_XMALLOC
     170                 : 
     171                 :    /* Support SYSV semantics. */
     172                 : #  define MALLOC_SYSV
     173                 : #endif
     174                 : 
     175                 : /*
     176                 :  * MALLOC_VALIDATE causes malloc_usable_size() to perform some pointer
     177                 :  * validation.  There are many possible errors that validation does not even
     178                 :  * attempt to detect.
     179                 :  */
     180                 : #define MALLOC_VALIDATE
     181                 : 
     182                 : /* Embed no-op macros that support memory allocation tracking via valgrind. */
     183                 : #ifdef MOZ_VALGRIND
     184                 : #  define MALLOC_VALGRIND
     185                 : #endif
     186                 : #ifdef MALLOC_VALGRIND
     187                 : #  include <valgrind/valgrind.h>
     188                 : #else
     189                 : #  define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
     190                 : #  define VALGRIND_FREELIKE_BLOCK(addr, rzB)
     191                 : #endif
     192                 : 
     193                 : /*
     194                 :  * MALLOC_BALANCE enables monitoring of arena lock contention and dynamically
     195                 :  * re-balances arena load if exponentially averaged contention exceeds a
     196                 :  * certain threshold.
     197                 :  */
     198                 : /* #define      MALLOC_BALANCE */
     199                 : 
     200                 : /*
     201                 :  * MALLOC_PAGEFILE causes all mmap()ed memory to be backed by temporary
     202                 :  * files, so that if a chunk is mapped, it is guaranteed to be swappable.
     203                 :  * This avoids asynchronous OOM failures that are due to VM over-commit.
     204                 :  */
     205                 : /* #define MALLOC_PAGEFILE */
     206                 : 
     207                 : #ifdef MALLOC_PAGEFILE
     208                 : /* Write size when initializing a page file. */
     209                 : #  define MALLOC_PAGEFILE_WRITE_SIZE 512
     210                 : #endif
     211                 : 
     212                 : #if defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
     213                 : #define _GNU_SOURCE /* For mremap(2). */
     214                 : #define issetugid() 0
     215                 : #if 0 /* Enable in order to test decommit code on Linux. */
     216                 : #  define MALLOC_DECOMMIT
     217                 : #endif
     218                 : #endif
     219                 : 
     220                 : #include <sys/types.h>
     221                 : 
     222                 : #include <errno.h>
     223                 : #include <stdlib.h>
     224                 : #include <limits.h>
     225                 : #include <stdarg.h>
     226                 : #include <stdio.h>
     227                 : #include <string.h>
     228                 : 
     229                 : #ifdef MOZ_MEMORY_WINDOWS
     230                 : 
     231                 : /* Some defines from the CRT internal headers that we need here. */
     232                 : #define _CRT_SPINCOUNT 5000
     233                 : #define __crtInitCritSecAndSpinCount InitializeCriticalSectionAndSpinCount
     234                 : #include <io.h>
     235                 : #include <windows.h>
     236                 : 
     237                 : #pragma warning( disable: 4267 4996 4146 )
     238                 : 
     239                 : #define bool BOOL
     240                 : #define false FALSE
     241                 : #define true TRUE
     242                 : #define inline __inline
     243                 : #define SIZE_T_MAX SIZE_MAX
     244                 : #define STDERR_FILENO 2
     245                 : #define PATH_MAX MAX_PATH
     246                 : #define vsnprintf _vsnprintf
     247                 : 
     248                 : #ifndef NO_TLS
     249                 : static unsigned long tlsIndex = 0xffffffff;
     250                 : #endif 
     251                 : 
     252                 : #define __thread
     253                 : #define _pthread_self() __threadid()
     254                 : #define issetugid() 0
     255                 : 
     256                 : /* use MSVC intrinsics */
     257                 : #pragma intrinsic(_BitScanForward)
     258                 : static __forceinline int
     259                 : ffs(int x)
     260                 : {
     261                 :         unsigned long i;
     262                 : 
     263                 :         if (_BitScanForward(&i, x) != 0)
     264                 :                 return (i + 1);
     265                 : 
     266                 :         return (0);
     267                 : }
     268                 : 
     269                 : /* Implement getenv without using malloc */
     270                 : static char mozillaMallocOptionsBuf[64];
     271                 : 
     272                 : #define getenv xgetenv
     273                 : static char *
     274                 : getenv(const char *name)
     275                 : {
     276                 : 
     277                 :         if (GetEnvironmentVariableA(name, (LPSTR)&mozillaMallocOptionsBuf,
     278                 :                     sizeof(mozillaMallocOptionsBuf)) > 0)
     279                 :                 return (mozillaMallocOptionsBuf);
     280                 : 
     281                 :         return (NULL);
     282                 : }
     283                 : 
     284                 : typedef unsigned char uint8_t;
     285                 : typedef unsigned uint32_t;
     286                 : typedef unsigned long long uint64_t;
     287                 : typedef unsigned long long uintmax_t;
     288                 : #if defined(MOZ_MEMORY_SIZEOF_PTR_2POW) && (MOZ_MEMORY_SIZEOF_PTR_2POW == 3)
     289                 : typedef long long ssize_t;
     290                 : #else
     291                 : typedef long ssize_t;
     292                 : #endif
     293                 : 
     294                 : #define MALLOC_DECOMMIT
     295                 : #endif
     296                 : 
     297                 : #ifndef MOZ_MEMORY_WINDOWS
     298                 : #ifndef MOZ_MEMORY_SOLARIS
     299                 : #include <sys/cdefs.h>
     300                 : #endif
     301                 : #ifndef __DECONST
     302                 : #  define __DECONST(type, var)  ((type)(uintptr_t)(const void *)(var))
     303                 : #endif
     304                 : #ifndef MOZ_MEMORY
     305                 : __FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z jasone $");
     306                 : #include "libc_private.h"
     307                 : #ifdef MALLOC_DEBUG
     308                 : #  define _LOCK_DEBUG
     309                 : #endif
     310                 : #include "spinlock.h"
     311                 : #include "namespace.h"
     312                 : #endif
     313                 : #include <sys/mman.h>
     314                 : #ifndef MADV_FREE
     315                 : #  define MADV_FREE     MADV_DONTNEED
     316                 : #endif
     317                 : #ifndef MAP_NOSYNC
     318                 : #  define MAP_NOSYNC    0
     319                 : #endif
     320                 : #include <sys/param.h>
     321                 : #ifndef MOZ_MEMORY
     322                 : #include <sys/stddef.h>
     323                 : #endif
     324                 : #include <sys/time.h>
     325                 : #include <sys/types.h>
     326                 : #if !defined(MOZ_MEMORY_SOLARIS) && !defined(MOZ_MEMORY_ANDROID)
     327                 : #include <sys/sysctl.h>
     328                 : #endif
     329                 : #include <sys/uio.h>
     330                 : #ifndef MOZ_MEMORY
     331                 : #include <sys/ktrace.h> /* Must come after several other sys/ includes. */
     332                 : 
     333                 : #include <machine/atomic.h>
     334                 : #include <machine/cpufunc.h>
     335                 : #include <machine/vmparam.h>
     336                 : #endif
     337                 : 
     338                 : #include <errno.h>
     339                 : #include <limits.h>
     340                 : #ifndef SIZE_T_MAX
     341                 : #  define SIZE_T_MAX    SIZE_MAX
     342                 : #endif
     343                 : #include <pthread.h>
     344                 : #ifdef MOZ_MEMORY_DARWIN
     345                 : #define _pthread_self pthread_self
     346                 : #define _pthread_mutex_init pthread_mutex_init
     347                 : #define _pthread_mutex_trylock pthread_mutex_trylock
     348                 : #define _pthread_mutex_lock pthread_mutex_lock
     349                 : #define _pthread_mutex_unlock pthread_mutex_unlock
     350                 : #endif
     351                 : #include <sched.h>
     352                 : #include <stdarg.h>
     353                 : #include <stdio.h>
     354                 : #include <stdbool.h>
     355                 : #include <stdint.h>
     356                 : #include <stdlib.h>
     357                 : #include <string.h>
     358                 : #ifndef MOZ_MEMORY_DARWIN
     359                 : #include <strings.h>
     360                 : #endif
     361                 : #include <unistd.h>
     362                 : 
     363                 : #ifdef MOZ_MEMORY_DARWIN
     364                 : #include <libkern/OSAtomic.h>
     365                 : #include <mach/mach_error.h>
     366                 : #include <mach/mach_init.h>
     367                 : #include <mach/vm_map.h>
     368                 : #include <malloc/malloc.h>
     369                 : #endif
     370                 : 
     371                 : #ifndef MOZ_MEMORY
     372                 : #include "un-namespace.h"
     373                 : #endif
     374                 : 
     375                 : #endif
     376                 : 
     377                 : #include "jemalloc.h"
     378                 : #include "linkedlist.h"
     379                 : 
     380                 : /* Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that
     381                 :  * happen to override mmap() and call dlsym() from their overridden
     382                 :  * mmap(). The problem is that dlsym() calls malloc(), and this ends
     383                 :  * up in a dead lock in jemalloc.
     384                 :  * On these systems, we prefer to directly use the system call.
     385                 :  * We do that for Linux systems and kfreebsd with GNU userland.
     386                 :  * Note sanity checks are not done (alignment of offset, ...) because
     387                 :  * the uses of mmap are pretty limited, in jemalloc.
     388                 :  *
     389                 :  * On Alpha, glibc has a bug that prevents syscall() to work for system
     390                 :  * calls with 6 arguments
     391                 :  */
     392                 : #if (defined(MOZ_MEMORY_LINUX) && !defined(__alpha__)) || \
     393                 :     (defined(MOZ_MEMORY_BSD) && defined(__GLIBC__))
     394                 : #include <sys/syscall.h>
     395                 : #if defined(SYS_mmap) || defined(SYS_mmap2)
     396                 : static inline
     397           82582 : void *_mmap(void *addr, size_t length, int prot, int flags,
     398                 :             int fd, off_t offset)
     399                 : {
     400                 : /* S390 only passes one argument to the mmap system call, which is a
     401                 :  * pointer to a structure containing the arguments */
     402                 : #ifdef __s390__
     403                 :         struct {
     404                 :                 void *addr;
     405                 :                 size_t length;
     406                 :                 int prot;
     407                 :                 int flags;
     408                 :                 int fd;
     409                 :                 off_t offset;
     410                 :         } args = { addr, length, prot, flags, fd, offset };
     411                 :         return (void *) syscall(SYS_mmap, &args);
     412                 : #else
     413                 : #ifdef SYS_mmap2
     414           82582 :         return (void *) syscall(SYS_mmap2, addr, length, prot, flags,
     415                 :                                fd, offset >> 12);
     416                 : #else
     417                 :         return (void *) syscall(SYS_mmap, addr, length, prot, flags,
     418                 :                                fd, offset);
     419                 : #endif
     420                 : #endif
     421                 : }
     422                 : #define mmap _mmap
     423                 : #define munmap(a, l) syscall(SYS_munmap, a, l)
     424                 : #endif
     425                 : #endif
     426                 : 
     427                 : #ifdef MOZ_MEMORY_DARWIN
     428                 : static const bool __isthreaded = true;
     429                 : #endif
     430                 : 
     431                 : #if defined(MOZ_MEMORY_SOLARIS) && defined(MAP_ALIGN) && !defined(JEMALLOC_NEVER_USES_MAP_ALIGN)
     432                 : #define JEMALLOC_USES_MAP_ALIGN  /* Required on Solaris 10. Might improve performance elsewhere. */
     433                 : #endif
     434                 : 
     435                 : #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
     436                 : 
     437                 : #ifdef MOZ_MEMORY_WINDOWS
     438                 :    /* MSVC++ does not support C99 variable-length arrays. */
     439                 : #  define RB_NO_C99_VARARRAYS
     440                 : #endif
     441                 : #include "rb.h"
     442                 : 
     443                 : #ifdef MALLOC_DEBUG
     444                 :    /* Disable inlining to make debugging easier. */
     445                 : #ifdef inline
     446                 : #undef inline
     447                 : #endif
     448                 : 
     449                 : #  define inline
     450                 : #endif
     451                 : 
     452                 : /* Size of stack-allocated buffer passed to strerror_r(). */
     453                 : #define STRERROR_BUF            64
     454                 : 
     455                 : /* Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. */
     456                 : #  define QUANTUM_2POW_MIN      4
     457                 : #ifdef MOZ_MEMORY_SIZEOF_PTR_2POW
     458                 : #  define SIZEOF_PTR_2POW               MOZ_MEMORY_SIZEOF_PTR_2POW
     459                 : #else
     460                 : #  define SIZEOF_PTR_2POW       2
     461                 : #endif
     462                 : #define PIC
     463                 : #ifndef MOZ_MEMORY_DARWIN
     464                 : static const bool __isthreaded = true;
     465                 : #else
     466                 : #  define NO_TLS
     467                 : #endif
     468                 : #if 0
     469                 : #ifdef __i386__
     470                 : #  define QUANTUM_2POW_MIN      4
     471                 : #  define SIZEOF_PTR_2POW       2
     472                 : #  define CPU_SPINWAIT          __asm__ volatile("pause")
     473                 : #endif
     474                 : #ifdef __ia64__
     475                 : #  define QUANTUM_2POW_MIN      4
     476                 : #  define SIZEOF_PTR_2POW       3
     477                 : #endif
     478                 : #ifdef __alpha__
     479                 : #  define QUANTUM_2POW_MIN      4
     480                 : #  define SIZEOF_PTR_2POW       3
     481                 : #  define NO_TLS
     482                 : #endif
     483                 : #ifdef __sparc64__
     484                 : #  define QUANTUM_2POW_MIN      4
     485                 : #  define SIZEOF_PTR_2POW       3
     486                 : #  define NO_TLS
     487                 : #endif
     488                 : #ifdef __amd64__
     489                 : #  define QUANTUM_2POW_MIN      4
     490                 : #  define SIZEOF_PTR_2POW       3
     491                 : #  define CPU_SPINWAIT          __asm__ volatile("pause")
     492                 : #endif
     493                 : #ifdef __arm__
     494                 : #  define QUANTUM_2POW_MIN      3
     495                 : #  define SIZEOF_PTR_2POW       2
     496                 : #  define NO_TLS
     497                 : #endif
     498                 : #ifdef __mips__
     499                 : #  define QUANTUM_2POW_MIN      3
     500                 : #  define SIZEOF_PTR_2POW       2
     501                 : #  define NO_TLS
     502                 : #endif
     503                 : #ifdef __powerpc__
     504                 : #  define QUANTUM_2POW_MIN      4
     505                 : #  define SIZEOF_PTR_2POW       2
     506                 : #endif
     507                 : #endif
     508                 : 
     509                 : #define SIZEOF_PTR              (1U << SIZEOF_PTR_2POW)
     510                 : 
     511                 : /* sizeof(int) == (1U << SIZEOF_INT_2POW). */
     512                 : #ifndef SIZEOF_INT_2POW
     513                 : #  define SIZEOF_INT_2POW       2
     514                 : #endif
     515                 : 
     516                 : /* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
     517                 : #if (!defined(PIC) && !defined(NO_TLS))
     518                 : #  define NO_TLS
     519                 : #endif
     520                 : 
     521                 : #ifdef NO_TLS
     522                 :    /* MALLOC_BALANCE requires TLS. */
     523                 : #  ifdef MALLOC_BALANCE
     524                 : #    undef MALLOC_BALANCE
     525                 : #  endif
     526                 : #endif
     527                 : 
     528                 : /*
     529                 :  * Size and alignment of memory chunks that are allocated by the OS's virtual
     530                 :  * memory system.
     531                 :  */
     532                 : #define CHUNK_2POW_DEFAULT      20
     533                 : /* Maximum number of dirty pages per arena. */
     534                 : #define DIRTY_MAX_DEFAULT       (1U << 10)
     535                 : 
     536                 : /*
     537                 :  * Maximum size of L1 cache line.  This is used to avoid cache line aliasing,
     538                 :  * so over-estimates are okay (up to a point), but under-estimates will
     539                 :  * negatively affect performance.
     540                 :  */
     541                 : #define CACHELINE_2POW          6
     542                 : #define CACHELINE               ((size_t)(1U << CACHELINE_2POW))
     543                 : 
     544                 : /*
     545                 :  * Smallest size class to support.  On Linux and Mac, even malloc(1) must
     546                 :  * reserve a word's worth of memory (see Mozilla bug 691003).
     547                 :  */
     548                 : #ifdef MOZ_MEMORY_WINDOWS
     549                 : #define TINY_MIN_2POW           1
     550                 : #else
     551                 : #define TINY_MIN_2POW           (sizeof(void*) == 8 ? 3 : 2)
     552                 : #endif
     553                 : 
     554                 : /*
     555                 :  * Maximum size class that is a multiple of the quantum, but not (necessarily)
     556                 :  * a power of 2.  Above this size, allocations are rounded up to the nearest
     557                 :  * power of 2.
     558                 :  */
     559                 : #define SMALL_MAX_2POW_DEFAULT  9
     560                 : #define SMALL_MAX_DEFAULT       (1U << SMALL_MAX_2POW_DEFAULT)
     561                 : 
     562                 : /*
     563                 :  * RUN_MAX_OVRHD indicates maximum desired run header overhead.  Runs are sized
     564                 :  * as small as possible such that this setting is still honored, without
     565                 :  * violating other constraints.  The goal is to make runs as small as possible
     566                 :  * without exceeding a per run external fragmentation threshold.
     567                 :  *
     568                 :  * We use binary fixed point math for overhead computations, where the binary
     569                 :  * point is implicitly RUN_BFP bits to the left.
     570                 :  *
     571                 :  * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
     572                 :  * honored for some/all object sizes, since there is one bit of header overhead
     573                 :  * per object (plus a constant).  This constraint is relaxed (ignored) for runs
     574                 :  * that are so small that the per-region overhead is greater than:
     575                 :  *
     576                 :  *   (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
     577                 :  */
     578                 : #define RUN_BFP                 12
     579                 : /*                                    \/   Implicit binary fixed point. */
     580                 : #define RUN_MAX_OVRHD           0x0000003dU
     581                 : #define RUN_MAX_OVRHD_RELAX     0x00001800U
     582                 : 
     583                 : /* Put a cap on small object run size.  This overrides RUN_MAX_OVRHD. */
     584                 : #define RUN_MAX_SMALL_2POW      15
     585                 : #define RUN_MAX_SMALL           (1U << RUN_MAX_SMALL_2POW)
     586                 : 
     587                 : /*
     588                 :  * Hyper-threaded CPUs may need a special instruction inside spin loops in
     589                 :  * order to yield to another virtual CPU.  If no such instruction is defined
     590                 :  * above, make CPU_SPINWAIT a no-op.
     591                 :  */
     592                 : #ifndef CPU_SPINWAIT
     593                 : #  define CPU_SPINWAIT
     594                 : #endif
     595                 : 
     596                 : /*
     597                 :  * Adaptive spinning must eventually switch to blocking, in order to avoid the
     598                 :  * potential for priority inversion deadlock.  Backing off past a certain point
     599                 :  * can actually waste time.
     600                 :  */
     601                 : #define SPIN_LIMIT_2POW         11
     602                 : 
     603                 : /*
     604                 :  * Conversion from spinning to blocking is expensive; we use (1U <<
     605                 :  * BLOCK_COST_2POW) to estimate how many more times costly blocking is than
     606                 :  * worst-case spinning.
     607                 :  */
     608                 : #define BLOCK_COST_2POW         4
     609                 : 
     610                 : #ifdef MALLOC_BALANCE
     611                 :    /*
     612                 :     * We use an exponential moving average to track recent lock contention,
     613                 :     * where the size of the history window is N, and alpha=2/(N+1).
     614                 :     *
     615                 :     * Due to integer math rounding, very small values here can cause
     616                 :     * substantial degradation in accuracy, thus making the moving average decay
     617                 :     * faster than it would with precise calculation.
     618                 :     */
     619                 : #  define BALANCE_ALPHA_INV_2POW        9
     620                 : 
     621                 :    /*
     622                 :     * Threshold value for the exponential moving contention average at which to
     623                 :     * re-assign a thread.
     624                 :     */
     625                 : #  define BALANCE_THRESHOLD_DEFAULT     (1U << (SPIN_LIMIT_2POW-4))
     626                 : #endif
     627                 : 
     628                 : /******************************************************************************/
     629                 : 
     630                 : /* MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. */
     631                 : #if defined(MALLOC_DECOMMIT) && defined(MALLOC_DOUBLE_PURGE)
     632                 : #error MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive.
     633                 : #endif
     634                 : 
     635                 : /*
     636                 :  * Mutexes based on spinlocks.  We can't use normal pthread spinlocks in all
     637                 :  * places, because they require malloc()ed memory, which causes bootstrapping
     638                 :  * issues in some cases.
     639                 :  */
     640                 : #if defined(MOZ_MEMORY_WINDOWS)
     641                 : #define malloc_mutex_t CRITICAL_SECTION
     642                 : #define malloc_spinlock_t CRITICAL_SECTION
     643                 : #elif defined(MOZ_MEMORY_DARWIN)
     644                 : typedef struct {
     645                 :         OSSpinLock      lock;
     646                 : } malloc_mutex_t;
     647                 : typedef struct {
     648                 :         OSSpinLock      lock;
     649                 : } malloc_spinlock_t;
     650                 : #elif defined(MOZ_MEMORY)
     651                 : typedef pthread_mutex_t malloc_mutex_t;
     652                 : typedef pthread_mutex_t malloc_spinlock_t;
     653                 : #else
     654                 : /* XXX these should #ifdef these for freebsd (and linux?) only */
     655                 : typedef struct {
     656                 :         spinlock_t      lock;
     657                 : } malloc_mutex_t;
     658                 : typedef malloc_spinlock_t malloc_mutex_t;
     659                 : #endif
     660                 : 
     661                 : /* Set to true once the allocator has been initialized. */
     662                 : static bool malloc_initialized = false;
     663                 : 
     664                 : #if defined(MOZ_MEMORY_WINDOWS)
     665                 : /* No init lock for Windows. */
     666                 : #elif defined(MOZ_MEMORY_DARWIN)
     667                 : static malloc_mutex_t init_lock = {OS_SPINLOCK_INIT};
     668                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
     669                 : static malloc_mutex_t init_lock = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
     670                 : #elif defined(MOZ_MEMORY)
     671                 : static malloc_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
     672                 : #else
     673                 : static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
     674                 : #endif
     675                 : 
     676                 : /******************************************************************************/
     677                 : /*
     678                 :  * Statistics data structures.
     679                 :  */
     680                 : 
     681                 : #ifdef MALLOC_STATS
     682                 : 
     683                 : typedef struct malloc_bin_stats_s malloc_bin_stats_t;
     684                 : struct malloc_bin_stats_s {
     685                 :         /*
     686                 :          * Number of allocation requests that corresponded to the size of this
     687                 :          * bin.
     688                 :          */
     689                 :         uint64_t        nrequests;
     690                 : 
     691                 :         /* Total number of runs created for this bin's size class. */
     692                 :         uint64_t        nruns;
     693                 : 
     694                 :         /*
     695                 :          * Total number of runs reused by extracting them from the runs tree for
     696                 :          * this bin's size class.
     697                 :          */
     698                 :         uint64_t        reruns;
     699                 : 
     700                 :         /* High-water mark for this bin. */
     701                 :         unsigned long   highruns;
     702                 : 
     703                 :         /* Current number of runs in this bin. */
     704                 :         unsigned long   curruns;
     705                 : };
     706                 : 
     707                 : typedef struct arena_stats_s arena_stats_t;
     708                 : struct arena_stats_s {
     709                 :         /* Number of bytes currently mapped. */
     710                 :         size_t          mapped;
     711                 : 
     712                 :         /*
     713                 :          * Total number of purge sweeps, total number of madvise calls made,
     714                 :          * and total pages purged in order to keep dirty unused memory under
     715                 :          * control.
     716                 :          */
     717                 :         uint64_t        npurge;
     718                 :         uint64_t        nmadvise;
     719                 :         uint64_t        purged;
     720                 : #ifdef MALLOC_DECOMMIT
     721                 :         /*
     722                 :          * Total number of decommit/commit operations, and total number of
     723                 :          * pages decommitted.
     724                 :          */
     725                 :         uint64_t        ndecommit;
     726                 :         uint64_t        ncommit;
     727                 :         uint64_t        decommitted;
     728                 : #endif
     729                 : 
     730                 :         /* Current number of committed pages. */
     731                 :         size_t          committed;
     732                 : 
     733                 :         /* Per-size-category statistics. */
     734                 :         size_t          allocated_small;
     735                 :         uint64_t        nmalloc_small;
     736                 :         uint64_t        ndalloc_small;
     737                 : 
     738                 :         size_t          allocated_large;
     739                 :         uint64_t        nmalloc_large;
     740                 :         uint64_t        ndalloc_large;
     741                 : 
     742                 : #ifdef MALLOC_BALANCE
     743                 :         /* Number of times this arena reassigned a thread due to contention. */
     744                 :         uint64_t        nbalance;
     745                 : #endif
     746                 : };
     747                 : 
     748                 : typedef struct chunk_stats_s chunk_stats_t;
     749                 : struct chunk_stats_s {
     750                 :         /* Number of chunks that were allocated. */
     751                 :         uint64_t        nchunks;
     752                 : 
     753                 :         /* High-water mark for number of chunks allocated. */
     754                 :         unsigned long   highchunks;
     755                 : 
     756                 :         /*
     757                 :          * Current number of chunks allocated.  This value isn't maintained for
     758                 :          * any other purpose, so keep track of it in order to be able to set
     759                 :          * highchunks.
     760                 :          */
     761                 :         unsigned long   curchunks;
     762                 : };
     763                 : 
     764                 : #endif /* #ifdef MALLOC_STATS */
     765                 : 
     766                 : /******************************************************************************/
     767                 : /*
     768                 :  * Extent data structures.
     769                 :  */
     770                 : 
     771                 : /* Tree of extents. */
     772                 : typedef struct extent_node_s extent_node_t;
     773                 : struct extent_node_s {
     774                 :         /* Linkage for the size/address-ordered tree. */
     775                 :         rb_node(extent_node_t) link_szad;
     776                 : 
     777                 :         /* Linkage for the address-ordered tree. */
     778                 :         rb_node(extent_node_t) link_ad;
     779                 : 
     780                 :         /* Pointer to the extent that this tree node is responsible for. */
     781                 :         void    *addr;
     782                 : 
     783                 :         /* Total region size. */
     784                 :         size_t  size;
     785                 : };
     786                 : typedef rb_tree(extent_node_t) extent_tree_t;
     787                 : 
     788                 : /******************************************************************************/
     789                 : /*
     790                 :  * Radix tree data structures.
     791                 :  */
     792                 : 
     793                 : #ifdef MALLOC_VALIDATE
     794                 :    /*
     795                 :     * Size of each radix tree node (must be a power of 2).  This impacts tree
     796                 :     * depth.
     797                 :     */
     798                 : #  if (SIZEOF_PTR == 4)
     799                 : #    define MALLOC_RTREE_NODESIZE (1U << 14)
     800                 : #  else
     801                 : #    define MALLOC_RTREE_NODESIZE CACHELINE
     802                 : #  endif
     803                 : 
     804                 : typedef struct malloc_rtree_s malloc_rtree_t;
     805                 : struct malloc_rtree_s {
     806                 :         malloc_spinlock_t       lock;
     807                 :         void                    **root;
     808                 :         unsigned                height;
     809                 :         unsigned                level2bits[1]; /* Dynamically sized. */
     810                 : };
     811                 : #endif
     812                 : 
     813                 : /******************************************************************************/
     814                 : /*
     815                 :  * Arena data structures.
     816                 :  */
     817                 : 
     818                 : typedef struct arena_s arena_t;
     819                 : typedef struct arena_bin_s arena_bin_t;
     820                 : 
     821                 : /* Each element of the chunk map corresponds to one page within the chunk. */
     822                 : typedef struct arena_chunk_map_s arena_chunk_map_t;
     823                 : struct arena_chunk_map_s {
     824                 :         /*
     825                 :          * Linkage for run trees.  There are two disjoint uses:
     826                 :          *
     827                 :          * 1) arena_t's runs_avail tree.
     828                 :          * 2) arena_run_t conceptually uses this linkage for in-use non-full
     829                 :          *    runs, rather than directly embedding linkage.
     830                 :          */
     831                 :         rb_node(arena_chunk_map_t)      link;
     832                 : 
     833                 :         /*
     834                 :          * Run address (or size) and various flags are stored together.  The bit
     835                 :          * layout looks like (assuming 32-bit system):
     836                 :          *
     837                 :          *   ???????? ???????? ????---- -mckdzla
     838                 :          *
     839                 :          * ? : Unallocated: Run address for first/last pages, unset for internal
     840                 :          *                  pages.
     841                 :          *     Small: Run address.
     842                 :          *     Large: Run size for first page, unset for trailing pages.
     843                 :          * - : Unused.
     844                 :          * m : MADV_FREE/MADV_DONTNEED'ed?
     845                 :          * c : decommitted?
     846                 :          * k : key?
     847                 :          * d : dirty?
     848                 :          * z : zeroed?
     849                 :          * l : large?
     850                 :          * a : allocated?
     851                 :          *
     852                 :          * Following are example bit patterns for the three types of runs.
     853                 :          *
     854                 :          * r : run address
     855                 :          * s : run size
     856                 :          * x : don't care
     857                 :          * - : 0
     858                 :          * [cdzla] : bit set
     859                 :          *
     860                 :          *   Unallocated:
     861                 :          *     ssssssss ssssssss ssss---- --c-----
     862                 :          *     xxxxxxxx xxxxxxxx xxxx---- ----d---
     863                 :          *     ssssssss ssssssss ssss---- -----z--
     864                 :          *
     865                 :          *   Small:
     866                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
     867                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
     868                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
     869                 :          *
     870                 :          *   Large:
     871                 :          *     ssssssss ssssssss ssss---- ------la
     872                 :          *     -------- -------- -------- ------la
     873                 :          *     -------- -------- -------- ------la
     874                 :          */
     875                 :         size_t                          bits;
     876                 : 
     877                 : /* Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether
     878                 :  * MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined.
     879                 :  *
     880                 :  * If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be
     881                 :  * re-committed with pages_commit() before it may be touched.  If
     882                 :  * MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined.
     883                 :  *
     884                 :  * If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which
     885                 :  * are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with
     886                 :  * CHUNK_MAP_MADVISED.
     887                 :  *
     888                 :  * Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is
     889                 :  * defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED.
     890                 :  * When it's finally freed with jemalloc_purge_freed_pages, the page is marked
     891                 :  * as CHUNK_MAP_DECOMMITTED.
     892                 :  */
     893                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
     894                 : #define CHUNK_MAP_MADVISED      ((size_t)0x40U)
     895                 : #define CHUNK_MAP_DECOMMITTED   ((size_t)0x20U)
     896                 : #define CHUNK_MAP_MADVISED_OR_DECOMMITTED (CHUNK_MAP_MADVISED | CHUNK_MAP_DECOMMITTED)
     897                 : #endif
     898                 : #define CHUNK_MAP_KEY           ((size_t)0x10U)
     899                 : #define CHUNK_MAP_DIRTY         ((size_t)0x08U)
     900                 : #define CHUNK_MAP_ZEROED        ((size_t)0x04U)
     901                 : #define CHUNK_MAP_LARGE         ((size_t)0x02U)
     902                 : #define CHUNK_MAP_ALLOCATED     ((size_t)0x01U)
     903                 : };
     904                 : typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
     905                 : typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
     906                 : 
     907                 : /* Arena chunk header. */
     908                 : typedef struct arena_chunk_s arena_chunk_t;
     909                 : struct arena_chunk_s {
     910                 :         /* Arena that owns the chunk. */
     911                 :         arena_t         *arena;
     912                 : 
     913                 :         /* Linkage for the arena's chunks_dirty tree. */
     914                 :         rb_node(arena_chunk_t) link_dirty;
     915                 : 
     916                 : #ifdef MALLOC_DOUBLE_PURGE
     917                 :         /* If we're double-purging, we maintain a linked list of chunks which
     918                 :          * have pages which have been madvise(MADV_FREE)'d but not explicitly
     919                 :          * purged.
     920                 :          *
     921                 :          * We're currently lazy and don't remove a chunk from this list when
     922                 :          * all its madvised pages are recommitted. */
     923                 :         LinkedList      chunks_madvised_elem;
     924                 : #endif
     925                 : 
     926                 :         /* Number of dirty pages. */
     927                 :         size_t          ndirty;
     928                 : 
     929                 :         /* Map of pages within chunk that keeps track of free/large/small. */
     930                 :         arena_chunk_map_t map[1]; /* Dynamically sized. */
     931                 : };
     932                 : typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
     933                 : 
     934                 : typedef struct arena_run_s arena_run_t;
     935                 : struct arena_run_s {
     936                 : #ifdef MALLOC_DEBUG
     937                 :         uint32_t        magic;
     938                 : #  define ARENA_RUN_MAGIC 0x384adf93
     939                 : #endif
     940                 : 
     941                 :         /* Bin this run is associated with. */
     942                 :         arena_bin_t     *bin;
     943                 : 
     944                 :         /* Index of first element that might have a free region. */
     945                 :         unsigned        regs_minelm;
     946                 : 
     947                 :         /* Number of free regions in run. */
     948                 :         unsigned        nfree;
     949                 : 
     950                 :         /* Bitmask of in-use regions (0: in use, 1: free). */
     951                 :         unsigned        regs_mask[1]; /* Dynamically sized. */
     952                 : };
     953                 : 
     954                 : struct arena_bin_s {
     955                 :         /*
     956                 :          * Current run being used to service allocations of this bin's size
     957                 :          * class.
     958                 :          */
     959                 :         arena_run_t     *runcur;
     960                 : 
     961                 :         /*
     962                 :          * Tree of non-full runs.  This tree is used when looking for an
     963                 :          * existing run when runcur is no longer usable.  We choose the
     964                 :          * non-full run that is lowest in memory; this policy tends to keep
     965                 :          * objects packed well, and it can also help reduce the number of
     966                 :          * almost-empty chunks.
     967                 :          */
     968                 :         arena_run_tree_t runs;
     969                 : 
     970                 :         /* Size of regions in a run for this bin's size class. */
     971                 :         size_t          reg_size;
     972                 : 
     973                 :         /* Total size of a run for this bin's size class. */
     974                 :         size_t          run_size;
     975                 : 
     976                 :         /* Total number of regions in a run for this bin's size class. */
     977                 :         uint32_t        nregs;
     978                 : 
     979                 :         /* Number of elements in a run's regs_mask for this bin's size class. */
     980                 :         uint32_t        regs_mask_nelms;
     981                 : 
     982                 :         /* Offset of first region in a run for this bin's size class. */
     983                 :         uint32_t        reg0_offset;
     984                 : 
     985                 : #ifdef MALLOC_STATS
     986                 :         /* Bin statistics. */
     987                 :         malloc_bin_stats_t stats;
     988                 : #endif
     989                 : };
     990                 : 
     991                 : struct arena_s {
     992                 : #ifdef MALLOC_DEBUG
     993                 :         uint32_t                magic;
     994                 : #  define ARENA_MAGIC 0x947d3d24
     995                 : #endif
     996                 : 
     997                 :         /* All operations on this arena require that lock be locked. */
     998                 : #ifdef MOZ_MEMORY
     999                 :         malloc_spinlock_t       lock;
    1000                 : #else
    1001                 :         pthread_mutex_t         lock;
    1002                 : #endif
    1003                 : 
    1004                 : #ifdef MALLOC_STATS
    1005                 :         arena_stats_t           stats;
    1006                 : #endif
    1007                 : 
    1008                 :         /* Tree of dirty-page-containing chunks this arena manages. */
    1009                 :         arena_chunk_tree_t      chunks_dirty;
    1010                 : 
    1011                 : #ifdef MALLOC_DOUBLE_PURGE
    1012                 :         /* Head of a linked list of MADV_FREE'd-page-containing chunks this
    1013                 :          * arena manages. */
    1014                 :         LinkedList              chunks_madvised;
    1015                 : #endif
    1016                 : 
    1017                 :         /*
    1018                 :          * In order to avoid rapid chunk allocation/deallocation when an arena
    1019                 :          * oscillates right on the cusp of needing a new chunk, cache the most
    1020                 :          * recently freed chunk.  The spare is left in the arena's chunk trees
    1021                 :          * until it is deleted.
    1022                 :          *
    1023                 :          * There is one spare chunk per arena, rather than one spare total, in
    1024                 :          * order to avoid interactions between multiple threads that could make
    1025                 :          * a single spare inadequate.
    1026                 :          */
    1027                 :         arena_chunk_t           *spare;
    1028                 : 
    1029                 :         /*
    1030                 :          * Current count of pages within unused runs that are potentially
    1031                 :          * dirty, and for which madvise(... MADV_FREE) has not been called.  By
    1032                 :          * tracking this, we can institute a limit on how much dirty unused
    1033                 :          * memory is mapped for each arena.
    1034                 :          */
    1035                 :         size_t                  ndirty;
    1036                 : 
    1037                 :         /*
    1038                 :          * Size/address-ordered tree of this arena's available runs.  This tree
    1039                 :          * is used for first-best-fit run allocation.
    1040                 :          */
    1041                 :         arena_avail_tree_t      runs_avail;
    1042                 : 
    1043                 : #ifdef MALLOC_BALANCE
    1044                 :         /*
    1045                 :          * The arena load balancing machinery needs to keep track of how much
    1046                 :          * lock contention there is.  This value is exponentially averaged.
    1047                 :          */
    1048                 :         uint32_t                contention;
    1049                 : #endif
    1050                 : 
    1051                 :         /*
    1052                 :          * bins is used to store rings of free regions of the following sizes,
    1053                 :          * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS.
    1054                 :          *
    1055                 :          *   bins[i] | size |
    1056                 :          *   --------+------+
    1057                 :          *        0  |    2 |
    1058                 :          *        1  |    4 |
    1059                 :          *        2  |    8 |
    1060                 :          *   --------+------+
    1061                 :          *        3  |   16 |
    1062                 :          *        4  |   32 |
    1063                 :          *        5  |   48 |
    1064                 :          *        6  |   64 |
    1065                 :          *           :      :
    1066                 :          *           :      :
    1067                 :          *       33  |  496 |
    1068                 :          *       34  |  512 |
    1069                 :          *   --------+------+
    1070                 :          *       35  | 1024 |
    1071                 :          *       36  | 2048 |
    1072                 :          *   --------+------+
    1073                 :          */
    1074                 :         arena_bin_t             bins[1]; /* Dynamically sized. */
    1075                 : };
    1076                 : 
    1077                 : /******************************************************************************/
    1078                 : /*
    1079                 :  * Data.
    1080                 :  */
    1081                 : 
    1082                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    1083                 : /* Number of CPUs. */
    1084                 : static unsigned         ncpus;
    1085                 : #endif
    1086                 : 
    1087                 : /*
    1088                 :  * When MALLOC_STATIC_SIZES is defined most of the parameters
    1089                 :  * controlling the malloc behavior are defined as compile-time constants
    1090                 :  * for best performance and cannot be altered at runtime.
    1091                 :  */
    1092                 : #define MALLOC_STATIC_SIZES 1
    1093                 : 
    1094                 : #ifdef MALLOC_STATIC_SIZES
    1095                 : 
    1096                 : /*
    1097                 :  * VM page size. It must divide the runtime CPU page size or the code
    1098                 :  * will abort.
    1099                 :  */
    1100                 : #define pagesize_2pow                   ((size_t) 12)
    1101                 : #define pagesize                        ((size_t) 1 << pagesize_2pow)
    1102                 : #define pagesize_mask                   (pagesize - 1)
    1103                 : 
    1104                 : /* Various quantum-related settings. */
    1105                 : 
    1106                 : #define QUANTUM_DEFAULT                 ((size_t) 1 << QUANTUM_2POW_MIN)
    1107                 : static const size_t     quantum =       QUANTUM_DEFAULT;
    1108                 : static const size_t     quantum_mask =  QUANTUM_DEFAULT - 1;
    1109                 : 
    1110                 : /* Various bin-related settings. */
    1111                 : 
    1112                 : static const size_t     small_min =     (QUANTUM_DEFAULT >> 1) + 1;
    1113                 : static const size_t     small_max =     (size_t) SMALL_MAX_DEFAULT;
    1114                 : 
    1115                 : /* Max size class for bins. */
    1116                 : static const size_t     bin_maxclass =  pagesize >> 1;
    1117                 : 
    1118                 :  /* Number of (2^n)-spaced tiny bins. */
    1119                 : static const unsigned   ntbins =        (unsigned)
    1120                 :                                         (QUANTUM_2POW_MIN - TINY_MIN_2POW);
    1121                 : 
    1122                 :  /* Number of quantum-spaced bins. */
    1123                 : static const unsigned   nqbins =        (unsigned)
    1124                 :                                         (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
    1125                 : 
    1126                 : /* Number of (2^n)-spaced sub-page bins. */
    1127                 : static const unsigned   nsbins =        (unsigned)
    1128                 :                                         (pagesize_2pow -
    1129                 :                                          SMALL_MAX_2POW_DEFAULT - 1);
    1130                 : 
    1131                 : #else /* !MALLOC_STATIC_SIZES */
    1132                 : 
    1133                 : /* VM page size. */
    1134                 : static size_t           pagesize;
    1135                 : static size_t           pagesize_mask;
    1136                 : static size_t           pagesize_2pow;
    1137                 : 
    1138                 : /* Various bin-related settings. */
    1139                 : static size_t           bin_maxclass; /* Max size class for bins. */
    1140                 : static unsigned         ntbins; /* Number of (2^n)-spaced tiny bins. */
    1141                 : static unsigned         nqbins; /* Number of quantum-spaced bins. */
    1142                 : static unsigned         nsbins; /* Number of (2^n)-spaced sub-page bins. */
    1143                 : static size_t           small_min;
    1144                 : static size_t           small_max;
    1145                 : 
    1146                 : /* Various quantum-related settings. */
    1147                 : static size_t           quantum;
    1148                 : static size_t           quantum_mask; /* (quantum - 1). */
    1149                 : 
    1150                 : #endif
    1151                 : 
    1152                 : /* Various chunk-related settings. */
    1153                 : 
    1154                 : /*
    1155                 :  * Compute the header size such that it is large enough to contain the page map
    1156                 :  * and enough nodes for the worst case: one node per non-header page plus one
    1157                 :  * extra for situations where we briefly have one more node allocated than we
    1158                 :  * will need.
    1159                 :  */
    1160                 : #define calculate_arena_header_size()                                   \
    1161                 :         (sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1))
    1162                 : 
    1163                 : #define calculate_arena_header_pages()                                  \
    1164                 :         ((calculate_arena_header_size() >> pagesize_2pow) +               \
    1165                 :          ((calculate_arena_header_size() & pagesize_mask) ? 1 : 0))
    1166                 : 
    1167                 : /* Max size class for arenas. */
    1168                 : #define calculate_arena_maxclass()                                      \
    1169                 :         (chunksize - (arena_chunk_header_npages << pagesize_2pow))
    1170                 : 
    1171                 : #ifdef MALLOC_STATIC_SIZES
    1172                 : #define CHUNKSIZE_DEFAULT               ((size_t) 1 << CHUNK_2POW_DEFAULT)
    1173                 : static const size_t     chunksize =     CHUNKSIZE_DEFAULT;
    1174                 : static const size_t     chunksize_mask =CHUNKSIZE_DEFAULT - 1;
    1175                 : static const size_t     chunk_npages =  CHUNKSIZE_DEFAULT >> pagesize_2pow;
    1176                 : #define arena_chunk_header_npages       calculate_arena_header_pages()
    1177                 : #define arena_maxclass                  calculate_arena_maxclass()
    1178                 : #else
    1179                 : static size_t           chunksize;
    1180                 : static size_t           chunksize_mask; /* (chunksize - 1). */
    1181                 : static size_t           chunk_npages;
    1182                 : static size_t           arena_chunk_header_npages;
    1183                 : static size_t           arena_maxclass; /* Max size class for arenas. */
    1184                 : #endif
    1185                 : 
    1186                 : /********/
    1187                 : /*
    1188                 :  * Chunks.
    1189                 :  */
    1190                 : 
    1191                 : #ifdef MALLOC_VALIDATE
    1192                 : static malloc_rtree_t *chunk_rtree;
    1193                 : #endif
    1194                 : 
    1195                 : /* Protects chunk-related data structures. */
    1196                 : static malloc_mutex_t   huge_mtx;
    1197                 : 
    1198                 : /* Tree of chunks that are stand-alone huge allocations. */
    1199                 : static extent_tree_t    huge;
    1200                 : 
    1201                 : #ifdef MALLOC_STATS
    1202                 : /* Huge allocation statistics. */
    1203                 : static uint64_t         huge_nmalloc;
    1204                 : static uint64_t         huge_ndalloc;
    1205                 : static size_t           huge_allocated;
    1206                 : #endif
    1207                 : 
    1208                 : #ifdef MALLOC_PAGEFILE
    1209                 : static char             pagefile_templ[PATH_MAX];
    1210                 : #endif
    1211                 : 
    1212                 : /****************************/
    1213                 : /*
    1214                 :  * base (internal allocation).
    1215                 :  */
    1216                 : 
    1217                 : /*
    1218                 :  * Current pages that are being used for internal memory allocations.  These
    1219                 :  * pages are carved up in cacheline-size quanta, so that there is no chance of
    1220                 :  * false cache line sharing.
    1221                 :  */
    1222                 : static void             *base_pages;
    1223                 : static void             *base_next_addr;
    1224                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
    1225                 : static void             *base_next_decommitted;
    1226                 : #endif
    1227                 : static void             *base_past_addr; /* Addr immediately past base_pages. */
    1228                 : static extent_node_t    *base_nodes;
    1229                 : static malloc_mutex_t   base_mtx;
    1230                 : #ifdef MALLOC_STATS
    1231                 : static size_t           base_mapped;
    1232                 : static size_t           base_committed;
    1233                 : #endif
    1234                 : 
    1235                 : /********/
    1236                 : /*
    1237                 :  * Arenas.
    1238                 :  */
    1239                 : 
    1240                 : /*
    1241                 :  * Arenas that are used to service external requests.  Not all elements of the
    1242                 :  * arenas array are necessarily used; arenas are created lazily as needed.
    1243                 :  */
    1244                 : static arena_t          **arenas;
    1245                 : static unsigned         narenas;
    1246                 : #ifndef NO_TLS
    1247                 : #  ifdef MALLOC_BALANCE
    1248                 : static unsigned         narenas_2pow;
    1249                 : #  else
    1250                 : static unsigned         next_arena;
    1251                 : #  endif
    1252                 : #endif
    1253                 : #ifdef MOZ_MEMORY
    1254                 : static malloc_spinlock_t arenas_lock; /* Protects arenas initialization. */
    1255                 : #else
    1256                 : static pthread_mutex_t arenas_lock; /* Protects arenas initialization. */
    1257                 : #endif
    1258                 : 
    1259                 : #ifndef NO_TLS
    1260                 : /*
    1261                 :  * Map of pthread_self() --> arenas[???], used for selecting an arena to use
    1262                 :  * for allocations.
    1263                 :  */
    1264                 : #ifndef MOZ_MEMORY_WINDOWS
    1265                 : static __thread arena_t *arenas_map;
    1266                 : #endif
    1267                 : #endif
    1268                 : 
    1269                 : #ifdef MALLOC_STATS
    1270                 : /* Chunk statistics. */
    1271                 : static chunk_stats_t    stats_chunks;
    1272                 : #endif
    1273                 : 
    1274                 : /*******************************/
    1275                 : /*
    1276                 :  * Runtime configuration options.
    1277                 :  */
    1278                 : const char      *_malloc_options;
    1279                 : 
    1280                 : #ifndef MALLOC_PRODUCTION
    1281                 : static bool     opt_abort = true;
    1282                 : #ifdef MALLOC_FILL
    1283                 : static bool     opt_junk = true;
    1284                 : #endif
    1285                 : #else
    1286                 : static bool     opt_abort = false;
    1287                 : #ifdef MALLOC_FILL
    1288                 : static bool     opt_junk = false;
    1289                 : #endif
    1290                 : #endif
    1291                 : static size_t   opt_dirty_max = DIRTY_MAX_DEFAULT;
    1292                 : #ifdef MALLOC_BALANCE
    1293                 : static uint64_t opt_balance_threshold = BALANCE_THRESHOLD_DEFAULT;
    1294                 : #endif
    1295                 : static bool     opt_print_stats = false;
    1296                 : #ifdef MALLOC_STATIC_SIZES
    1297                 : #define opt_quantum_2pow        QUANTUM_2POW_MIN
    1298                 : #define opt_small_max_2pow      SMALL_MAX_2POW_DEFAULT
    1299                 : #define opt_chunk_2pow          CHUNK_2POW_DEFAULT
    1300                 : #else
    1301                 : static size_t   opt_quantum_2pow = QUANTUM_2POW_MIN;
    1302                 : static size_t   opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
    1303                 : static size_t   opt_chunk_2pow = CHUNK_2POW_DEFAULT;
    1304                 : #endif
    1305                 : #ifdef MALLOC_PAGEFILE
    1306                 : static bool     opt_pagefile = false;
    1307                 : #endif
    1308                 : #ifdef MALLOC_UTRACE
    1309                 : static bool     opt_utrace = false;
    1310                 : #endif
    1311                 : #ifdef MALLOC_SYSV
    1312                 : static bool     opt_sysv = false;
    1313                 : #endif
    1314                 : #ifdef MALLOC_XMALLOC
    1315                 : static bool     opt_xmalloc = false;
    1316                 : #endif
    1317                 : #ifdef MALLOC_FILL
    1318                 : static bool     opt_zero = false;
    1319                 : #endif
    1320                 : static int      opt_narenas_lshift = 0;
    1321                 : 
    1322                 : #ifdef MALLOC_UTRACE
    1323                 : typedef struct {
    1324                 :         void    *p;
    1325                 :         size_t  s;
    1326                 :         void    *r;
    1327                 : } malloc_utrace_t;
    1328                 : 
    1329                 : #define UTRACE(a, b, c)                                                 \
    1330                 :         if (opt_utrace) {                                               \
    1331                 :                 malloc_utrace_t ut;                                     \
    1332                 :                 ut.p = (a);                                             \
    1333                 :                 ut.s = (b);                                             \
    1334                 :                 ut.r = (c);                                             \
    1335                 :                 utrace(&ut, sizeof(ut));                            \
    1336                 :         }
    1337                 : #else
    1338                 : #define UTRACE(a, b, c)
    1339                 : #endif
    1340                 : 
    1341                 : /******************************************************************************/
    1342                 : /*
    1343                 :  * Begin function prototypes for non-inline static functions.
    1344                 :  */
    1345                 : 
    1346                 : static char     *umax2s(uintmax_t x, unsigned base, char *s);
    1347                 : static bool     malloc_mutex_init(malloc_mutex_t *mutex);
    1348                 : static bool     malloc_spin_init(malloc_spinlock_t *lock);
    1349                 : static void     wrtmessage(const char *p1, const char *p2, const char *p3,
    1350                 :                 const char *p4);
    1351                 : #ifdef MALLOC_STATS
    1352                 : #ifdef MOZ_MEMORY_DARWIN
    1353                 : /* Avoid namespace collision with OS X's malloc APIs. */
    1354                 : #define malloc_printf moz_malloc_printf
    1355                 : #endif
    1356                 : static void     malloc_printf(const char *format, ...);
    1357                 : #endif
    1358                 : static bool     base_pages_alloc_mmap(size_t minsize);
    1359                 : static bool     base_pages_alloc(size_t minsize);
    1360                 : static void     *base_alloc(size_t size);
    1361                 : static void     *base_calloc(size_t number, size_t size);
    1362                 : static extent_node_t *base_node_alloc(void);
    1363                 : static void     base_node_dealloc(extent_node_t *node);
    1364                 : #ifdef MALLOC_STATS
    1365                 : static void     stats_print(arena_t *arena);
    1366                 : #endif
    1367                 : static void     *pages_map(void *addr, size_t size, int pfd);
    1368                 : static void     pages_unmap(void *addr, size_t size);
    1369                 : static void     *chunk_alloc_mmap(size_t size, bool pagefile);
    1370                 : #ifdef MALLOC_PAGEFILE
    1371                 : static int      pagefile_init(size_t size);
    1372                 : static void     pagefile_close(int pfd);
    1373                 : #endif
    1374                 : static void     *chunk_alloc(size_t size, bool zero, bool pagefile);
    1375                 : static void     chunk_dealloc_mmap(void *chunk, size_t size);
    1376                 : static void     chunk_dealloc(void *chunk, size_t size);
    1377                 : #ifndef NO_TLS
    1378                 : static arena_t  *choose_arena_hard(void);
    1379                 : #endif
    1380                 : static void     arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
    1381                 :     bool large, bool zero);
    1382                 : static void arena_chunk_init(arena_t *arena, arena_chunk_t *chunk);
    1383                 : static void     arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
    1384                 : static arena_run_t *arena_run_alloc(arena_t *arena, arena_bin_t *bin,
    1385                 :     size_t size, bool large, bool zero);
    1386                 : static void     arena_purge(arena_t *arena);
    1387                 : static void     arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
    1388                 : static void     arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
    1389                 :     arena_run_t *run, size_t oldsize, size_t newsize);
    1390                 : static void     arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
    1391                 :     arena_run_t *run, size_t oldsize, size_t newsize, bool dirty);
    1392                 : static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
    1393                 : static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
    1394                 : static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
    1395                 : #ifdef MALLOC_BALANCE
    1396                 : static void     arena_lock_balance_hard(arena_t *arena);
    1397                 : #endif
    1398                 : static void     *arena_malloc_large(arena_t *arena, size_t size, bool zero);
    1399                 : static void     *arena_palloc(arena_t *arena, size_t alignment, size_t size,
    1400                 :     size_t alloc_size);
    1401                 : static size_t   arena_salloc(const void *ptr);
    1402                 : static void     arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk,
    1403                 :     void *ptr);
    1404                 : static void     arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
    1405                 :     void *ptr, size_t size, size_t oldsize);
    1406                 : static bool     arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk,
    1407                 :     void *ptr, size_t size, size_t oldsize);
    1408                 : static bool     arena_ralloc_large(void *ptr, size_t size, size_t oldsize);
    1409                 : static void     *arena_ralloc(void *ptr, size_t size, size_t oldsize);
    1410                 : static bool     arena_new(arena_t *arena);
    1411                 : static arena_t  *arenas_extend(unsigned ind);
    1412                 : static void     *huge_malloc(size_t size, bool zero);
    1413                 : static void     *huge_palloc(size_t alignment, size_t size);
    1414                 : static void     *huge_ralloc(void *ptr, size_t size, size_t oldsize);
    1415                 : static void     huge_dalloc(void *ptr);
    1416                 : static void     malloc_print_stats(void);
    1417                 : #ifndef MOZ_MEMORY_WINDOWS
    1418                 : static
    1419                 : #endif
    1420                 : bool            malloc_init_hard(void);
    1421                 : 
    1422                 : #ifdef MOZ_MEMORY_ANDROID
    1423                 : void    _malloc_prefork(void);
    1424                 : void    _malloc_postfork(void);
    1425                 : #else
    1426                 : static void     _malloc_prefork(void);
    1427                 : static void     _malloc_postfork(void);
    1428                 : #endif
    1429                 : 
    1430                 : #ifdef MOZ_MEMORY_DARWIN
    1431                 : /*
    1432                 :  * MALLOC_ZONE_T_NOTE
    1433                 :  *
    1434                 :  * On Darwin, we hook into the memory allocator using a malloc_zone_t struct.
    1435                 :  * We must be very careful around this struct because of different behaviour on
    1436                 :  * different versions of OSX.
    1437                 :  *
    1438                 :  * Each of OSX 10.5, 10.6 and 10.7 use different versions of the struct
    1439                 :  * (with version numbers 3, 6 and 8 respectively). The binary we use on each of
    1440                 :  * these platforms will not necessarily be built using the correct SDK [1].
    1441                 :  * This means we need to statically know the correct struct size to use on all
    1442                 :  * OSX releases, and have a fallback for unknown future versions. The struct
    1443                 :  * sizes defined in osx_zone_types.h.
    1444                 :  *
    1445                 :  * For OSX 10.8 and later, we may expect the malloc_zone_t struct to change
    1446                 :  * again, and need to dynamically account for this. By simply leaving
    1447                 :  * malloc_zone_t alone, we don't quite deal with the problem, because there
    1448                 :  * remain calls to jemalloc through the mozalloc interface. We check this
    1449                 :  * dynamically on each allocation, using the CHECK_DARWIN macro and
    1450                 :  * osx_use_jemalloc.
    1451                 :  *
    1452                 :  *
    1453                 :  * [1] Mozilla is built as a universal binary on Mac, supporting i386 and
    1454                 :  *     x86_64. The i386 target is built using the 10.5 SDK, even if it runs on
    1455                 :  *     10.6. The x86_64 target is built using the 10.6 SDK, even if it runs on
    1456                 :  *     10.7 or later, or 10.5.
    1457                 :  *
    1458                 :  * FIXME:
    1459                 :  *   When later versions of OSX come out (10.8 and up), we need to check their
    1460                 :  *   malloc_zone_t versions. If they're greater than 8, we need a new version
    1461                 :  *   of malloc_zone_t adapted into osx_zone_types.h.
    1462                 :  */
    1463                 : 
    1464                 : #include "osx_zone_types.h"
    1465                 : 
    1466                 : #define LEOPARD_MALLOC_ZONE_T_VERSION 3
    1467                 : #define SNOW_LEOPARD_MALLOC_ZONE_T_VERSION 6
    1468                 : #define LION_MALLOC_ZONE_T_VERSION 8
    1469                 : 
    1470                 : static bool osx_use_jemalloc = false;
    1471                 : 
    1472                 : 
    1473                 : static lion_malloc_zone l_szone;
    1474                 : static malloc_zone_t * szone = (malloc_zone_t*)(&l_szone);
    1475                 : 
    1476                 : static lion_malloc_introspection l_ozone_introspect;
    1477                 : static malloc_introspection_t * const ozone_introspect =
    1478                 :         (malloc_introspection_t*)(&l_ozone_introspect);
    1479                 : static void szone2ozone(malloc_zone_t *zone, size_t size);
    1480                 : static size_t zone_version_size(int version);
    1481                 : #endif
    1482                 : 
    1483                 : /*
    1484                 :  * End function prototypes.
    1485                 :  */
    1486                 : /******************************************************************************/
    1487                 : 
    1488                 : /*
    1489                 :  * umax2s() provides minimal integer printing functionality, which is
    1490                 :  * especially useful for situations where allocation in vsnprintf() calls would
    1491                 :  * potentially cause deadlock.
    1492                 :  */
    1493                 : #define UMAX2S_BUFSIZE  65
    1494                 : char *
    1495               0 : umax2s(uintmax_t x, unsigned base, char *s)
    1496                 : {
    1497                 :         unsigned i;
    1498                 : 
    1499               0 :         i = UMAX2S_BUFSIZE - 1;
    1500               0 :         s[i] = '\0';
    1501               0 :         switch (base) {
    1502                 :         case 10:
    1503                 :                 do {
    1504               0 :                         i--;
    1505               0 :                         s[i] = "0123456789"[x % 10];
    1506               0 :                         x /= 10;
    1507               0 :                 } while (x > 0);
    1508               0 :                 break;
    1509                 :         case 16:
    1510                 :                 do {
    1511               0 :                         i--;
    1512               0 :                         s[i] = "0123456789abcdef"[x & 0xf];
    1513               0 :                         x >>= 4;
    1514               0 :                 } while (x > 0);
    1515               0 :                 break;
    1516                 :         default:
    1517                 :                 do {
    1518               0 :                         i--;
    1519               0 :                         s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % base];
    1520               0 :                         x /= base;
    1521               0 :                 } while (x > 0);
    1522                 :         }
    1523                 : 
    1524               0 :         return (&s[i]);
    1525                 : }
    1526                 : 
    1527                 : static void
    1528               0 : wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
    1529                 : {
    1530                 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_WINDOWS)
    1531                 : #define _write  write
    1532                 : #endif
    1533               0 :         _write(STDERR_FILENO, p1, (unsigned int) strlen(p1));
    1534               0 :         _write(STDERR_FILENO, p2, (unsigned int) strlen(p2));
    1535               0 :         _write(STDERR_FILENO, p3, (unsigned int) strlen(p3));
    1536               0 :         _write(STDERR_FILENO, p4, (unsigned int) strlen(p4));
    1537               0 : }
    1538                 : 
    1539                 : #define _malloc_message malloc_message
    1540                 : 
    1541                 : void    (*_malloc_message)(const char *p1, const char *p2, const char *p3,
    1542                 :             const char *p4) = wrtmessage;
    1543                 : 
    1544                 : #ifdef MALLOC_DEBUG
    1545                 : #  define assert(e) do {                                                \
    1546                 :         if (!(e)) {                                                     \
    1547                 :                 char line_buf[UMAX2S_BUFSIZE];                          \
    1548                 :                 _malloc_message(__FILE__, ":", umax2s(__LINE__, 10,   \
    1549                 :                     line_buf), ": Failed assertion: ");                       \
    1550                 :                 _malloc_message("\"", #e, "\"\n", "");                  \
    1551                 :                 abort();                                                \
    1552                 :         }                                                               \
    1553                 : } while (0)
    1554                 : #else
    1555                 : #define assert(e)
    1556                 : #endif
    1557                 : 
    1558                 : /******************************************************************************/
    1559                 : /*
    1560                 :  * Begin mutex.  We can't use normal pthread mutexes in all places, because
    1561                 :  * they require malloc()ed memory, which causes bootstrapping issues in some
    1562                 :  * cases.
    1563                 :  */
    1564                 : 
    1565                 : static bool
    1566           99505 : malloc_mutex_init(malloc_mutex_t *mutex)
    1567                 : {
    1568                 : #if defined(MOZ_MEMORY_WINDOWS)
    1569                 :         if (__isthreaded)
    1570                 :                 if (! __crtInitCritSecAndSpinCount(mutex, _CRT_SPINCOUNT))
    1571                 :                         return (true);
    1572                 : #elif defined(MOZ_MEMORY_DARWIN)
    1573                 :         mutex->lock = OS_SPINLOCK_INIT;
    1574                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
    1575                 :         pthread_mutexattr_t attr;
    1576           99505 :         if (pthread_mutexattr_init(&attr) != 0)
    1577               0 :                 return (true);
    1578           99505 :         pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
    1579           99505 :         if (pthread_mutex_init(mutex, &attr) != 0) {
    1580               0 :                 pthread_mutexattr_destroy(&attr);
    1581               0 :                 return (true);
    1582                 :         }
    1583           99505 :         pthread_mutexattr_destroy(&attr);
    1584                 : #elif defined(MOZ_MEMORY)
    1585                 :         if (pthread_mutex_init(mutex, NULL) != 0)
    1586                 :                 return (true);
    1587                 : #else
    1588                 :         static const spinlock_t lock = _SPINLOCK_INITIALIZER;
    1589                 : 
    1590                 :         mutex->lock = lock;
    1591                 : #endif
    1592           99505 :         return (false);
    1593                 : }
    1594                 : 
    1595                 : static inline void
    1596       261618339 : malloc_mutex_lock(malloc_mutex_t *mutex)
    1597                 : {
    1598                 : 
    1599                 : #if defined(MOZ_MEMORY_WINDOWS)
    1600                 :         EnterCriticalSection(mutex);
    1601                 : #elif defined(MOZ_MEMORY_DARWIN)
    1602                 :         OSSpinLockLock(&mutex->lock);
    1603                 : #elif defined(MOZ_MEMORY)
    1604       261618339 :         pthread_mutex_lock(mutex);
    1605                 : #else
    1606                 :         if (__isthreaded)
    1607                 :                 _SPINLOCK(&mutex->lock);
    1608                 : #endif
    1609       261646910 : }
    1610                 : 
    1611                 : static inline void
    1612       261645882 : malloc_mutex_unlock(malloc_mutex_t *mutex)
    1613                 : {
    1614                 : 
    1615                 : #if defined(MOZ_MEMORY_WINDOWS)
    1616                 :         LeaveCriticalSection(mutex);
    1617                 : #elif defined(MOZ_MEMORY_DARWIN)
    1618                 :         OSSpinLockUnlock(&mutex->lock);
    1619                 : #elif defined(MOZ_MEMORY)
    1620       261645882 :         pthread_mutex_unlock(mutex);
    1621                 : #else
    1622                 :         if (__isthreaded)
    1623                 :                 _SPINUNLOCK(&mutex->lock);
    1624                 : #endif
    1625       261645682 : }
    1626                 : 
    1627                 : static bool
    1628               0 : malloc_spin_init(malloc_spinlock_t *lock)
    1629                 : {
    1630                 : #if defined(MOZ_MEMORY_WINDOWS)
    1631                 :         if (__isthreaded)
    1632                 :                 if (! __crtInitCritSecAndSpinCount(lock, _CRT_SPINCOUNT))
    1633                 :                         return (true);
    1634                 : #elif defined(MOZ_MEMORY_DARWIN)
    1635                 :         lock->lock = OS_SPINLOCK_INIT;
    1636                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
    1637                 :         pthread_mutexattr_t attr;
    1638               0 :         if (pthread_mutexattr_init(&attr) != 0)
    1639               0 :                 return (true);
    1640               0 :         pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
    1641               0 :         if (pthread_mutex_init(lock, &attr) != 0) {
    1642               0 :                 pthread_mutexattr_destroy(&attr);
    1643               0 :                 return (true);
    1644                 :         }
    1645               0 :         pthread_mutexattr_destroy(&attr);
    1646                 : #elif defined(MOZ_MEMORY)
    1647                 :         if (pthread_mutex_init(lock, NULL) != 0)
    1648                 :                 return (true);
    1649                 : #else
    1650                 :         lock->lock = _SPINLOCK_INITIALIZER;
    1651                 : #endif
    1652               0 :         return (false);
    1653                 : }
    1654                 : 
    1655                 : static inline void
    1656               0 : malloc_spin_lock(malloc_spinlock_t *lock)
    1657                 : {
    1658                 : 
    1659                 : #if defined(MOZ_MEMORY_WINDOWS)
    1660                 :         EnterCriticalSection(lock);
    1661                 : #elif defined(MOZ_MEMORY_DARWIN)
    1662                 :         OSSpinLockLock(&lock->lock);
    1663                 : #elif defined(MOZ_MEMORY)
    1664               0 :         pthread_mutex_lock(lock);
    1665                 : #else
    1666                 :         if (__isthreaded)
    1667                 :                 _SPINLOCK(&lock->lock);
    1668                 : #endif
    1669               0 : }
    1670                 : 
    1671                 : static inline void
    1672               0 : malloc_spin_unlock(malloc_spinlock_t *lock)
    1673                 : {
    1674                 : #if defined(MOZ_MEMORY_WINDOWS)
    1675                 :         LeaveCriticalSection(lock);
    1676                 : #elif defined(MOZ_MEMORY_DARWIN)
    1677                 :         OSSpinLockUnlock(&lock->lock);
    1678                 : #elif defined(MOZ_MEMORY)
    1679               0 :         pthread_mutex_unlock(lock);
    1680                 : #else
    1681                 :         if (__isthreaded)
    1682                 :                 _SPINUNLOCK(&lock->lock);
    1683                 : #endif
    1684               0 : }
    1685                 : 
    1686                 : /*
    1687                 :  * End mutex.
    1688                 :  */
    1689                 : /******************************************************************************/
    1690                 : /*
    1691                 :  * Begin spin lock.  Spin locks here are actually adaptive mutexes that block
    1692                 :  * after a period of spinning, because unbounded spinning would allow for
    1693                 :  * priority inversion.
    1694                 :  */
    1695                 : 
    1696                 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_DARWIN)
    1697                 : #  define       malloc_spin_init        malloc_mutex_init
    1698                 : #  define       malloc_spin_lock        malloc_mutex_lock
    1699                 : #  define       malloc_spin_unlock      malloc_mutex_unlock
    1700                 : #endif
    1701                 : 
    1702                 : #ifndef MOZ_MEMORY
    1703                 : /*
    1704                 :  * We use an unpublished interface to initialize pthread mutexes with an
    1705                 :  * allocation callback, in order to avoid infinite recursion.
    1706                 :  */
    1707                 : int     _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
    1708                 :     void *(calloc_cb)(size_t, size_t));
    1709                 : 
    1710                 : __weak_reference(_pthread_mutex_init_calloc_cb_stub,
    1711                 :     _pthread_mutex_init_calloc_cb);
    1712                 : 
    1713                 : int
    1714                 : _pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
    1715                 :     void *(calloc_cb)(size_t, size_t))
    1716                 : {
    1717                 : 
    1718                 :         return (0);
    1719                 : }
    1720                 : 
    1721                 : static bool
    1722                 : malloc_spin_init(pthread_mutex_t *lock)
    1723                 : {
    1724                 : 
    1725                 :         if (_pthread_mutex_init_calloc_cb(lock, base_calloc) != 0)
    1726                 :                 return (true);
    1727                 : 
    1728                 :         return (false);
    1729                 : }
    1730                 : 
    1731                 : static inline unsigned
    1732                 : malloc_spin_lock(pthread_mutex_t *lock)
    1733                 : {
    1734                 :         unsigned ret = 0;
    1735                 : 
    1736                 :         if (__isthreaded) {
    1737                 :                 if (_pthread_mutex_trylock(lock) != 0) {
    1738                 :                         unsigned i;
    1739                 :                         volatile unsigned j;
    1740                 : 
    1741                 :                         /* Exponentially back off. */
    1742                 :                         for (i = 1; i <= SPIN_LIMIT_2POW; i++) {
    1743                 :                                 for (j = 0; j < (1U << i); j++)
    1744                 :                                         ret++;
    1745                 : 
    1746                 :                                 CPU_SPINWAIT;
    1747                 :                                 if (_pthread_mutex_trylock(lock) == 0)
    1748                 :                                         return (ret);
    1749                 :                         }
    1750                 : 
    1751                 :                         /*
    1752                 :                          * Spinning failed.  Block until the lock becomes
    1753                 :                          * available, in order to avoid indefinite priority
    1754                 :                          * inversion.
    1755                 :                          */
    1756                 :                         _pthread_mutex_lock(lock);
    1757                 :                         assert((ret << BLOCK_COST_2POW) != 0);
    1758                 :                         return (ret << BLOCK_COST_2POW);
    1759                 :                 }
    1760                 :         }
    1761                 : 
    1762                 :         return (ret);
    1763                 : }
    1764                 : 
    1765                 : static inline void
    1766                 : malloc_spin_unlock(pthread_mutex_t *lock)
    1767                 : {
    1768                 : 
    1769                 :         if (__isthreaded)
    1770                 :                 _pthread_mutex_unlock(lock);
    1771                 : }
    1772                 : #endif
    1773                 : 
    1774                 : /*
    1775                 :  * End spin lock.
    1776                 :  */
    1777                 : /******************************************************************************/
    1778                 : /*
    1779                 :  * Begin Utility functions/macros.
    1780                 :  */
    1781                 : 
    1782                 : /* Return the chunk address for allocation address a. */
    1783                 : #define CHUNK_ADDR2BASE(a)                                              \
    1784                 :         ((void *)((uintptr_t)(a) & ~chunksize_mask))
    1785                 : 
    1786                 : /* Return the chunk offset of address a. */
    1787                 : #define CHUNK_ADDR2OFFSET(a)                                            \
    1788                 :         ((size_t)((uintptr_t)(a) & chunksize_mask))
    1789                 : 
    1790                 : /* Return the smallest chunk multiple that is >= s. */
    1791                 : #define CHUNK_CEILING(s)                                                \
    1792                 :         (((s) + chunksize_mask) & ~chunksize_mask)
    1793                 : 
    1794                 : /* Return the smallest cacheline multiple that is >= s. */
    1795                 : #define CACHELINE_CEILING(s)                                            \
    1796                 :         (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1))
    1797                 : 
    1798                 : /* Return the smallest quantum multiple that is >= a. */
    1799                 : #define QUANTUM_CEILING(a)                                              \
    1800                 :         (((a) + quantum_mask) & ~quantum_mask)
    1801                 : 
    1802                 : /* Return the smallest pagesize multiple that is >= s. */
    1803                 : #define PAGE_CEILING(s)                                                 \
    1804                 :         (((s) + pagesize_mask) & ~pagesize_mask)
    1805                 : 
    1806                 : /* Compute the smallest power of 2 that is >= x. */
    1807                 : static inline size_t
    1808        32082902 : pow2_ceil(size_t x)
    1809                 : {
    1810                 : 
    1811        32082902 :         x--;
    1812        32082902 :         x |= x >> 1;
    1813        32082902 :         x |= x >> 2;
    1814        32082902 :         x |= x >> 4;
    1815        32082902 :         x |= x >> 8;
    1816        32082902 :         x |= x >> 16;
    1817                 : #if (SIZEOF_PTR == 8)
    1818                 :         x |= x >> 32;
    1819                 : #endif
    1820        32082902 :         x++;
    1821        32082902 :         return (x);
    1822                 : }
    1823                 : 
    1824                 : #ifdef MALLOC_BALANCE
    1825                 : /*
    1826                 :  * Use a simple linear congruential pseudo-random number generator:
    1827                 :  *
    1828                 :  *   prn(y) = (a*x + c) % m
    1829                 :  *
    1830                 :  * where the following constants ensure maximal period:
    1831                 :  *
    1832                 :  *   a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4.
    1833                 :  *   c == Odd number (relatively prime to 2^n).
    1834                 :  *   m == 2^32
    1835                 :  *
    1836                 :  * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
    1837                 :  *
    1838                 :  * This choice of m has the disadvantage that the quality of the bits is
    1839                 :  * proportional to bit position.  For example. the lowest bit has a cycle of 2,
    1840                 :  * the next has a cycle of 4, etc.  For this reason, we prefer to use the upper
    1841                 :  * bits.
    1842                 :  */
    1843                 : #  define PRN_DEFINE(suffix, var, a, c)                                 \
    1844                 : static inline void                                                      \
    1845                 : sprn_##suffix(uint32_t seed)                                            \
    1846                 : {                                                                       \
    1847                 :         var = seed;                                                     \
    1848                 : }                                                                       \
    1849                 :                                                                         \
    1850                 : static inline uint32_t                                                  \
    1851                 : prn_##suffix(uint32_t lg_range)                                         \
    1852                 : {                                                                       \
    1853                 :         uint32_t ret, x;                                                \
    1854                 :                                                                         \
    1855                 :         assert(lg_range > 0);                                                \
    1856                 :         assert(lg_range <= 32);                                              \
    1857                 :                                                                         \
    1858                 :         x = (var * (a)) + (c);                                          \
    1859                 :         var = x;                                                        \
    1860                 :         ret = x >> (32 - lg_range);                                       \
    1861                 :                                                                         \
    1862                 :         return (ret);                                                   \
    1863                 : }
    1864                 : #  define SPRN(suffix, seed)    sprn_##suffix(seed)
    1865                 : #  define PRN(suffix, lg_range) prn_##suffix(lg_range)
    1866                 : #endif
    1867                 : 
    1868                 : #ifdef MALLOC_BALANCE
    1869                 : /* Define the PRNG used for arena assignment. */
    1870                 : static __thread uint32_t balance_x;
    1871                 : PRN_DEFINE(balance, balance_x, 1297, 1301)
    1872                 : #endif
    1873                 : 
    1874                 : #ifdef MALLOC_UTRACE
    1875                 : static int
    1876               0 : utrace(const void *addr, size_t len)
    1877                 : {
    1878               0 :         malloc_utrace_t *ut = (malloc_utrace_t *)addr;
    1879                 :         char buf_a[UMAX2S_BUFSIZE];
    1880                 :         char buf_b[UMAX2S_BUFSIZE];
    1881                 : 
    1882               0 :         assert(len == sizeof(malloc_utrace_t));
    1883                 : 
    1884               0 :         if (ut->p == NULL && ut->s == 0 && ut->r == NULL) {
    1885               0 :                 _malloc_message(
    1886               0 :                     umax2s(getpid(), 10, buf_a),
    1887                 :                     " x USER malloc_init()\n", "", "");
    1888               0 :         } else if (ut->p == NULL && ut->r != NULL) {
    1889               0 :                 _malloc_message(
    1890               0 :                     umax2s(getpid(), 10, buf_a),
    1891                 :                     " x USER 0x",
    1892               0 :                     umax2s((uintptr_t)ut->r, 16, buf_b),
    1893                 :                     " = malloc(");
    1894               0 :                 _malloc_message(
    1895               0 :                     umax2s(ut->s, 10, buf_a),
    1896                 :                     ")\n", "", "");
    1897               0 :         } else if (ut->p != NULL && ut->r != NULL) {
    1898               0 :                 _malloc_message(
    1899               0 :                     umax2s(getpid(), 10, buf_a),
    1900                 :                     " x USER 0x",
    1901               0 :                     umax2s((uintptr_t)ut->r, 16, buf_b),
    1902                 :                     " = realloc(0x");
    1903               0 :                 _malloc_message(
    1904               0 :                     umax2s((uintptr_t)ut->p, 16, buf_a),
    1905                 :                     ", ",
    1906               0 :                     umax2s(ut->s, 10, buf_b),
    1907                 :                     ")\n");
    1908                 :         } else {
    1909               0 :                 _malloc_message(
    1910               0 :                     umax2s(getpid(), 10, buf_a),
    1911                 :                     " x USER free(0x",
    1912               0 :                     umax2s((uintptr_t)ut->p, 16, buf_b),
    1913                 :                     ")\n");
    1914                 :         }
    1915                 : 
    1916               0 :         return (0);
    1917                 : }
    1918                 : #endif
    1919                 : 
    1920                 : static inline const char *
    1921               0 : _getprogname(void)
    1922                 : {
    1923                 : 
    1924               0 :         return ("<jemalloc>");
    1925                 : }
    1926                 : 
    1927                 : #ifdef MALLOC_STATS
    1928                 : /*
    1929                 :  * Print to stderr in such a way as to (hopefully) avoid memory allocation.
    1930                 :  */
    1931                 : static void
    1932               0 : malloc_printf(const char *format, ...)
    1933                 : {
    1934                 :         char buf[4096];
    1935                 :         va_list ap;
    1936                 : 
    1937               0 :         va_start(ap, format);
    1938               0 :         vsnprintf(buf, sizeof(buf), format, ap);
    1939               0 :         va_end(ap);
    1940               0 :         _malloc_message(buf, "", "", "");
    1941               0 : }
    1942                 : #endif
    1943                 : 
    1944                 : /******************************************************************************/
    1945                 : 
    1946                 : static inline void
    1947               0 : pages_decommit(void *addr, size_t size)
    1948                 : {
    1949                 : 
    1950                 : #ifdef MOZ_MEMORY_WINDOWS
    1951                 :         VirtualFree(addr, size, MEM_DECOMMIT);
    1952                 : #else
    1953               0 :         if (mmap(addr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1,
    1954                 :             0) == MAP_FAILED)
    1955               0 :                 abort();
    1956                 : #endif
    1957               0 : }
    1958                 : 
    1959                 : static inline void
    1960               0 : pages_commit(void *addr, size_t size)
    1961                 : {
    1962                 : 
    1963                 : #  ifdef MOZ_MEMORY_WINDOWS
    1964                 :         VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
    1965                 : #  else
    1966               0 :         if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE |
    1967                 :             MAP_ANON, -1, 0) == MAP_FAILED)
    1968               0 :                 abort();
    1969                 : #  endif
    1970               0 : }
    1971                 : 
    1972                 : static bool
    1973           19901 : base_pages_alloc_mmap(size_t minsize)
    1974                 : {
    1975                 :         bool ret;
    1976                 :         size_t csize;
    1977                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
    1978                 :         size_t pminsize;
    1979                 : #endif
    1980                 :         int pfd;
    1981                 : 
    1982           19901 :         assert(minsize != 0);
    1983           19901 :         csize = CHUNK_CEILING(minsize);
    1984                 : #ifdef MALLOC_PAGEFILE
    1985                 :         if (opt_pagefile) {
    1986                 :                 pfd = pagefile_init(csize);
    1987                 :                 if (pfd == -1)
    1988                 :                         return (true);
    1989                 :         } else
    1990                 : #endif
    1991           19901 :                 pfd = -1;
    1992           19901 :         base_pages = pages_map(NULL, csize, pfd);
    1993           19901 :         if (base_pages == NULL) {
    1994               0 :                 ret = true;
    1995               0 :                 goto RETURN;
    1996                 :         }
    1997           19901 :         base_next_addr = base_pages;
    1998           19901 :         base_past_addr = (void *)((uintptr_t)base_pages + csize);
    1999                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
    2000                 :         /*
    2001                 :          * Leave enough pages for minsize committed, since otherwise they would
    2002                 :          * have to be immediately recommitted.
    2003                 :          */
    2004           19901 :         pminsize = PAGE_CEILING(minsize);
    2005           19901 :         base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize);
    2006                 : #  if defined(MALLOC_DECOMMIT)
    2007                 :         if (pminsize < csize)
    2008                 :                 pages_decommit(base_next_decommitted, csize - pminsize);
    2009                 : #  endif
    2010                 : #  ifdef MALLOC_STATS
    2011           19901 :         base_mapped += csize;
    2012           19901 :         base_committed += pminsize;
    2013                 : #  endif
    2014                 : #endif
    2015                 : 
    2016           19901 :         ret = false;
    2017                 : RETURN:
    2018                 : #ifdef MALLOC_PAGEFILE
    2019                 :         if (pfd != -1)
    2020                 :                 pagefile_close(pfd);
    2021                 : #endif
    2022           19901 :         return (false);
    2023                 : }
    2024                 : 
    2025                 : static bool
    2026           19901 : base_pages_alloc(size_t minsize)
    2027                 : {
    2028                 : 
    2029           19901 :         if (base_pages_alloc_mmap(minsize) == false)
    2030           19901 :                 return (false);
    2031                 : 
    2032               0 :         return (true);
    2033                 : }
    2034                 : 
    2035                 : static void *
    2036           80680 : base_alloc(size_t size)
    2037                 : {
    2038                 :         void *ret;
    2039                 :         size_t csize;
    2040                 : 
    2041                 :         /* Round size up to nearest multiple of the cacheline size. */
    2042           80680 :         csize = CACHELINE_CEILING(size);
    2043                 : 
    2044           80680 :         malloc_mutex_lock(&base_mtx);
    2045                 :         /* Make sure there's enough space for the allocation. */
    2046           80680 :         if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
    2047           19901 :                 if (base_pages_alloc(csize)) {
    2048               0 :                         malloc_mutex_unlock(&base_mtx);
    2049               0 :                         return (NULL);
    2050                 :                 }
    2051                 :         }
    2052                 :         /* Allocate. */
    2053           80680 :         ret = base_next_addr;
    2054           80680 :         base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
    2055                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
    2056                 :         /* Make sure enough pages are committed for the new allocation. */
    2057           80680 :         if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) {
    2058           19901 :                 void *pbase_next_addr =
    2059           19901 :                     (void *)(PAGE_CEILING((uintptr_t)base_next_addr));
    2060                 : 
    2061                 : #  ifdef MALLOC_DECOMMIT
    2062                 :                 pages_commit(base_next_decommitted, (uintptr_t)pbase_next_addr -
    2063                 :                     (uintptr_t)base_next_decommitted);
    2064                 : #  endif
    2065           19901 :                 base_next_decommitted = pbase_next_addr;
    2066                 : #  ifdef MALLOC_STATS
    2067           39802 :                 base_committed += (uintptr_t)pbase_next_addr -
    2068           19901 :                     (uintptr_t)base_next_decommitted;
    2069                 : #  endif
    2070                 :         }
    2071                 : #endif
    2072           80680 :         malloc_mutex_unlock(&base_mtx);
    2073                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
    2074                 : 
    2075           80680 :         return (ret);
    2076                 : }
    2077                 : 
    2078                 : static void *
    2079           39802 : base_calloc(size_t number, size_t size)
    2080                 : {
    2081                 :         void *ret;
    2082                 : 
    2083           39802 :         ret = base_alloc(number * size);
    2084                 : #ifdef MALLOC_VALGRIND
    2085                 :         if (ret != NULL) {
    2086                 :                 VALGRIND_FREELIKE_BLOCK(ret, 0);
    2087                 :                 VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, true);
    2088                 :         }
    2089                 : #endif
    2090           39802 :         memset(ret, 0, number * size);
    2091                 : 
    2092           39802 :         return (ret);
    2093                 : }
    2094                 : 
    2095                 : static extent_node_t *
    2096            5538 : base_node_alloc(void)
    2097                 : {
    2098                 :         extent_node_t *ret;
    2099                 : 
    2100            5538 :         malloc_mutex_lock(&base_mtx);
    2101            5538 :         if (base_nodes != NULL) {
    2102            4462 :                 ret = base_nodes;
    2103            4462 :                 base_nodes = *(extent_node_t **)ret;
    2104                 :                 VALGRIND_FREELIKE_BLOCK(ret, 0);
    2105                 :                 VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(extent_node_t), 0, false);
    2106            4462 :                 malloc_mutex_unlock(&base_mtx);
    2107                 :         } else {
    2108            1076 :                 malloc_mutex_unlock(&base_mtx);
    2109            1076 :                 ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
    2110                 :         }
    2111                 : 
    2112            5538 :         return (ret);
    2113                 : }
    2114                 : 
    2115                 : static void
    2116            5538 : base_node_dealloc(extent_node_t *node)
    2117                 : {
    2118                 : 
    2119            5538 :         malloc_mutex_lock(&base_mtx);
    2120                 :         VALGRIND_FREELIKE_BLOCK(node, 0);
    2121                 :         VALGRIND_MALLOCLIKE_BLOCK(node, sizeof(extent_node_t *), 0, false);
    2122            5538 :         *(extent_node_t **)node = base_nodes;
    2123            5538 :         base_nodes = node;
    2124            5538 :         malloc_mutex_unlock(&base_mtx);
    2125            5538 : }
    2126                 : 
    2127                 : /******************************************************************************/
    2128                 : 
    2129                 : #ifdef MALLOC_STATS
    2130                 : static void
    2131               0 : stats_print(arena_t *arena)
    2132                 : {
    2133                 :         unsigned i, gap_start;
    2134                 : 
    2135                 : #ifdef MOZ_MEMORY_WINDOWS
    2136                 :         malloc_printf("dirty: %Iu page%s dirty, %I64u sweep%s,"
    2137                 :             " %I64u madvise%s, %I64u page%s purged\n",
    2138                 :             arena->ndirty, arena->ndirty == 1 ? "" : "s",
    2139                 :             arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
    2140                 :             arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
    2141                 :             arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
    2142                 : #  ifdef MALLOC_DECOMMIT
    2143                 :         malloc_printf("decommit: %I64u decommit%s, %I64u commit%s,"
    2144                 :             " %I64u page%s decommitted\n",
    2145                 :             arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
    2146                 :             arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
    2147                 :             arena->stats.decommitted,
    2148                 :             (arena->stats.decommitted == 1) ? "" : "s");
    2149                 : #  endif
    2150                 : 
    2151                 :         malloc_printf("            allocated      nmalloc      ndalloc\n");
    2152                 :         malloc_printf("small:   %12Iu %12I64u %12I64u\n",
    2153                 :             arena->stats.allocated_small, arena->stats.nmalloc_small,
    2154                 :             arena->stats.ndalloc_small);
    2155                 :         malloc_printf("large:   %12Iu %12I64u %12I64u\n",
    2156                 :             arena->stats.allocated_large, arena->stats.nmalloc_large,
    2157                 :             arena->stats.ndalloc_large);
    2158                 :         malloc_printf("total:   %12Iu %12I64u %12I64u\n",
    2159                 :             arena->stats.allocated_small + arena->stats.allocated_large,
    2160                 :             arena->stats.nmalloc_small + arena->stats.nmalloc_large,
    2161                 :             arena->stats.ndalloc_small + arena->stats.ndalloc_large);
    2162                 :         malloc_printf("mapped:  %12Iu\n", arena->stats.mapped);
    2163                 : #else
    2164               0 :         malloc_printf("dirty: %zu page%s dirty, %llu sweep%s,"
    2165                 :             " %llu madvise%s, %llu page%s purged\n",
    2166               0 :             arena->ndirty, arena->ndirty == 1 ? "" : "s",
    2167               0 :             arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
    2168               0 :             arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
    2169               0 :             arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
    2170                 : #  ifdef MALLOC_DECOMMIT
    2171                 :         malloc_printf("decommit: %llu decommit%s, %llu commit%s,"
    2172                 :             " %llu page%s decommitted\n",
    2173                 :             arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
    2174                 :             arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
    2175                 :             arena->stats.decommitted,
    2176                 :             (arena->stats.decommitted == 1) ? "" : "s");
    2177                 : #  endif
    2178                 : 
    2179               0 :         malloc_printf("            allocated      nmalloc      ndalloc\n");
    2180               0 :         malloc_printf("small:   %12zu %12llu %12llu\n",
    2181                 :             arena->stats.allocated_small, arena->stats.nmalloc_small,
    2182                 :             arena->stats.ndalloc_small);
    2183               0 :         malloc_printf("large:   %12zu %12llu %12llu\n",
    2184                 :             arena->stats.allocated_large, arena->stats.nmalloc_large,
    2185                 :             arena->stats.ndalloc_large);
    2186               0 :         malloc_printf("total:   %12zu %12llu %12llu\n",
    2187               0 :             arena->stats.allocated_small + arena->stats.allocated_large,
    2188               0 :             arena->stats.nmalloc_small + arena->stats.nmalloc_large,
    2189               0 :             arena->stats.ndalloc_small + arena->stats.ndalloc_large);
    2190               0 :         malloc_printf("mapped:  %12zu\n", arena->stats.mapped);
    2191                 : #endif
    2192               0 :         malloc_printf("bins:     bin   size regs pgs  requests   newruns"
    2193                 :             "    reruns maxruns curruns\n");
    2194               0 :         for (i = 0, gap_start = UINT_MAX; i < ntbins + nqbins + nsbins; i++) {
    2195               0 :                 if (arena->bins[i].stats.nrequests == 0) {
    2196               0 :                         if (gap_start == UINT_MAX)
    2197               0 :                                 gap_start = i;
    2198                 :                 } else {
    2199               0 :                         if (gap_start != UINT_MAX) {
    2200               0 :                                 if (i > gap_start + 1) {
    2201                 :                                         /* Gap of more than one size class. */
    2202               0 :                                         malloc_printf("[%u..%u]\n",
    2203                 :                                             gap_start, i - 1);
    2204                 :                                 } else {
    2205                 :                                         /* Gap of one size class. */
    2206               0 :                                         malloc_printf("[%u]\n", gap_start);
    2207                 :                                 }
    2208               0 :                                 gap_start = UINT_MAX;
    2209                 :                         }
    2210               0 :                         malloc_printf(
    2211                 : #if defined(MOZ_MEMORY_WINDOWS)
    2212                 :                             "%13u %1s %4u %4u %3u %9I64u %9I64u"
    2213                 :                             " %9I64u %7u %7u\n",
    2214                 : #else
    2215                 :                             "%13u %1s %4u %4u %3u %9llu %9llu"
    2216                 :                             " %9llu %7lu %7lu\n",
    2217                 : #endif
    2218                 :                             i,
    2219               0 :                             i < ntbins ? "T" : i < ntbins + nqbins ? "Q" : "S",
    2220                 :                             arena->bins[i].reg_size,
    2221                 :                             arena->bins[i].nregs,
    2222               0 :                             arena->bins[i].run_size >> pagesize_2pow,
    2223                 :                             arena->bins[i].stats.nrequests,
    2224                 :                             arena->bins[i].stats.nruns,
    2225                 :                             arena->bins[i].stats.reruns,
    2226                 :                             arena->bins[i].stats.highruns,
    2227                 :                             arena->bins[i].stats.curruns);
    2228                 :                 }
    2229                 :         }
    2230               0 :         if (gap_start != UINT_MAX) {
    2231               0 :                 if (i > gap_start + 1) {
    2232                 :                         /* Gap of more than one size class. */
    2233               0 :                         malloc_printf("[%u..%u]\n", gap_start, i - 1);
    2234                 :                 } else {
    2235                 :                         /* Gap of one size class. */
    2236               0 :                         malloc_printf("[%u]\n", gap_start);
    2237                 :                 }
    2238                 :         }
    2239               0 : }
    2240                 : #endif
    2241                 : 
    2242                 : /*
    2243                 :  * End Utility functions/macros.
    2244                 :  */
    2245                 : /******************************************************************************/
    2246                 : /*
    2247                 :  * Begin extent tree code.
    2248                 :  */
    2249                 : 
    2250                 : static inline int
    2251               0 : extent_szad_comp(extent_node_t *a, extent_node_t *b)
    2252                 : {
    2253                 :         int ret;
    2254               0 :         size_t a_size = a->size;
    2255               0 :         size_t b_size = b->size;
    2256                 : 
    2257               0 :         ret = (a_size > b_size) - (a_size < b_size);
    2258               0 :         if (ret == 0) {
    2259               0 :                 uintptr_t a_addr = (uintptr_t)a->addr;
    2260               0 :                 uintptr_t b_addr = (uintptr_t)b->addr;
    2261                 : 
    2262               0 :                 ret = (a_addr > b_addr) - (a_addr < b_addr);
    2263                 :         }
    2264                 : 
    2265               0 :         return (ret);
    2266                 : }
    2267                 : 
    2268                 : /* Wrap red-black tree macros in functions. */
    2269               0 : rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
    2270               0 :     link_szad, extent_szad_comp)
    2271                 : 
    2272                 : static inline int
    2273           33113 : extent_ad_comp(extent_node_t *a, extent_node_t *b)
    2274                 : {
    2275           33113 :         uintptr_t a_addr = (uintptr_t)a->addr;
    2276           33113 :         uintptr_t b_addr = (uintptr_t)b->addr;
    2277                 : 
    2278           33113 :         return ((a_addr > b_addr) - (a_addr < b_addr));
    2279                 : }
    2280                 : 
    2281                 : /* Wrap red-black tree macros in functions. */
    2282           39791 : rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
    2283           39791 :     extent_ad_comp)
    2284                 : 
    2285                 : /*
    2286                 :  * End extent tree code.
    2287                 :  */
    2288                 : /******************************************************************************/
    2289                 : /*
    2290                 :  * Begin chunk management functions.
    2291                 :  */
    2292                 : 
    2293                 : #ifdef MOZ_MEMORY_WINDOWS
    2294                 : 
    2295                 : static void *
    2296                 : pages_map(void *addr, size_t size, int pfd)
    2297                 : {
    2298                 :         void *ret = NULL;
    2299                 :         ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
    2300                 :             PAGE_READWRITE);
    2301                 :         return (ret);
    2302                 : }
    2303                 : 
    2304                 : static void
    2305                 : pages_unmap(void *addr, size_t size)
    2306                 : {
    2307                 :         if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
    2308                 :                 _malloc_message(_getprogname(),
    2309                 :                     ": (malloc) Error in VirtualFree()\n", "", "");
    2310                 :                 if (opt_abort)
    2311                 :                         abort();
    2312                 :         }
    2313                 : }
    2314                 : #else
    2315                 : #ifdef JEMALLOC_USES_MAP_ALIGN
    2316                 : static void *
    2317                 : pages_map_align(size_t size, int pfd, size_t alignment)
    2318                 : {
    2319                 :         void *ret;
    2320                 : 
    2321                 :         /*
    2322                 :          * We don't use MAP_FIXED here, because it can cause the *replacement*
    2323                 :          * of existing mappings, and we only want to create new mappings.
    2324                 :          */
    2325                 : #ifdef MALLOC_PAGEFILE
    2326                 :         if (pfd != -1) {
    2327                 :                 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
    2328                 :                     MAP_NOSYNC | MAP_ALIGN, pfd, 0);
    2329                 :         } else
    2330                 : #endif
    2331                 :                {
    2332                 :                 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
    2333                 :                     MAP_NOSYNC | MAP_ALIGN | MAP_ANON, -1, 0);
    2334                 :         }
    2335                 :         assert(ret != NULL);
    2336                 : 
    2337                 :         if (ret == MAP_FAILED)
    2338                 :                 ret = NULL;
    2339                 :         return (ret);
    2340                 : }
    2341                 : #endif
    2342                 : 
    2343                 : static void *
    2344           82582 : pages_map(void *addr, size_t size, int pfd)
    2345                 : {
    2346                 :         void *ret;
    2347                 : #if defined(__ia64__)
    2348                 :         /*
    2349                 :          * The JS engine assumes that all allocated pointers have their high 17 bits clear,
    2350                 :          * which ia64's mmap doesn't support directly. However, we can emulate it by passing
    2351                 :          * mmap an "addr" parameter with those bits clear. The mmap will return that address,
    2352                 :          * or the nearest available memory above that address, providing a near-guarantee
    2353                 :          * that those bits are clear. If they are not, we return NULL below to indicate
    2354                 :          * out-of-memory.
    2355                 :          * 
    2356                 :          * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual 
    2357                 :          * address space.
    2358                 :          * 
    2359                 :          * See Bug 589735 for more information.
    2360                 :          */
    2361                 :         bool check_placement = true;
    2362                 :         if (addr == NULL) {
    2363                 :                 addr = (void*)0x0000070000000000;
    2364                 :                 check_placement = false;
    2365                 :         }
    2366                 : #endif
    2367                 : 
    2368                 :         /*
    2369                 :          * We don't use MAP_FIXED here, because it can cause the *replacement*
    2370                 :          * of existing mappings, and we only want to create new mappings.
    2371                 :          */
    2372                 : #ifdef MALLOC_PAGEFILE
    2373                 :         if (pfd != -1) {
    2374                 :                 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
    2375                 :                     MAP_NOSYNC, pfd, 0);
    2376                 :         } else
    2377                 : #endif
    2378                 :                {
    2379           82582 :                 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
    2380                 :                     MAP_ANON, -1, 0);
    2381                 :         }
    2382           82582 :         assert(ret != NULL);
    2383                 : 
    2384           82582 :         if (ret == MAP_FAILED) {
    2385              19 :                 ret = NULL;
    2386                 :         }
    2387                 : #if defined(__ia64__)
    2388                 :         /* 
    2389                 :          * If the allocated memory doesn't have its upper 17 bits clear, consider it 
    2390                 :          * as out of memory.
    2391                 :         */
    2392                 :         else if ((long long)ret & 0xffff800000000000) {
    2393                 :                 munmap(ret, size);
    2394                 :                 ret = NULL;
    2395                 :         }
    2396                 :         /* If the caller requested a specific memory location, verify that's what mmap returned. */
    2397                 :         else if (check_placement && ret != addr) {
    2398                 : #else
    2399           82563 :         else if (addr != NULL && ret != addr) {
    2400                 : #endif
    2401                 :                 /*
    2402                 :                  * We succeeded in mapping memory, but not in the right place.
    2403                 :                  */
    2404               0 :                 if (munmap(ret, size) == -1) {
    2405                 :                         char buf[STRERROR_BUF];
    2406                 : 
    2407               0 :                         strerror_r(errno, buf, sizeof(buf));
    2408               0 :                         _malloc_message(_getprogname(),
    2409                 :                             ": (malloc) Error in munmap(): ", buf, "\n");
    2410               0 :                         if (opt_abort)
    2411               0 :                                 abort();
    2412                 :                 }
    2413               0 :                 ret = NULL;
    2414                 :         }
    2415                 : 
    2416                 : #if defined(__ia64__)
    2417                 :         assert(ret == NULL || (!check_placement && ret != NULL)
    2418                 :             || (check_placement && ret == addr));
    2419                 : #else
    2420           82582 :         assert(ret == NULL || (addr == NULL && ret != addr)
    2421                 :             || (addr != NULL && ret == addr));
    2422                 : #endif
    2423           82582 :         return (ret);
    2424                 : }
    2425                 : 
    2426                 : static void
    2427           40007 : pages_unmap(void *addr, size_t size)
    2428                 : {
    2429                 : 
    2430           40007 :         if (munmap(addr, size) == -1) {
    2431                 :                 char buf[STRERROR_BUF];
    2432                 : 
    2433               0 :                 strerror_r(errno, buf, sizeof(buf));
    2434               0 :                 _malloc_message(_getprogname(),
    2435                 :                     ": (malloc) Error in munmap(): ", buf, "\n");
    2436               0 :                 if (opt_abort)
    2437               0 :                         abort();
    2438                 :         }
    2439           40007 : }
    2440                 : #endif
    2441                 : 
    2442                 : #ifdef MOZ_MEMORY_DARWIN
    2443                 : #define VM_COPY_MIN (pagesize << 5)
    2444                 : static inline void
    2445                 : pages_copy(void *dest, const void *src, size_t n)
    2446                 : {
    2447                 : 
    2448                 :         assert((void *)((uintptr_t)dest & ~pagesize_mask) == dest);
    2449                 :         assert(n >= VM_COPY_MIN);
    2450                 :         assert((void *)((uintptr_t)src & ~pagesize_mask) == src);
    2451                 : 
    2452                 :         vm_copy(mach_task_self(), (vm_address_t)src, (vm_size_t)n,
    2453                 :             (vm_address_t)dest);
    2454                 : }
    2455                 : #endif
    2456                 : 
    2457                 : #ifdef MALLOC_VALIDATE
    2458                 : static inline malloc_rtree_t *
    2459           19901 : malloc_rtree_new(unsigned bits)
    2460                 : {
    2461                 :         malloc_rtree_t *ret;
    2462                 :         unsigned bits_per_level, height, i;
    2463                 : 
    2464           39802 :         bits_per_level = ffs(pow2_ceil((MALLOC_RTREE_NODESIZE /
    2465           19901 :             sizeof(void *)))) - 1;
    2466           19901 :         height = bits / bits_per_level;
    2467           19901 :         if (height * bits_per_level != bits)
    2468               0 :                 height++;
    2469           19901 :         assert(height * bits_per_level >= bits);
    2470                 : 
    2471           19901 :         ret = (malloc_rtree_t*)base_calloc(1, sizeof(malloc_rtree_t) +
    2472                 :             (sizeof(unsigned) * (height - 1)));
    2473           19901 :         if (ret == NULL)
    2474               0 :                 return (NULL);
    2475                 : 
    2476           19901 :         malloc_spin_init(&ret->lock);
    2477           19901 :         ret->height = height;
    2478           19901 :         if (bits_per_level * height > bits)
    2479               0 :                 ret->level2bits[0] = bits % bits_per_level;
    2480                 :         else
    2481           19901 :                 ret->level2bits[0] = bits_per_level;
    2482           19901 :         for (i = 1; i < height; i++)
    2483               0 :                 ret->level2bits[i] = bits_per_level;
    2484                 : 
    2485           19901 :         ret->root = (void**)base_calloc(1, sizeof(void *) << ret->level2bits[0]);
    2486           19901 :         if (ret->root == NULL) {
    2487                 :                 /*
    2488                 :                  * We leak the rtree here, since there's no generic base
    2489                 :                  * deallocation.
    2490                 :                  */
    2491               0 :                 return (NULL);
    2492                 :         }
    2493                 : 
    2494           19901 :         return (ret);
    2495                 : }
    2496                 : 
    2497                 : #define MALLOC_RTREE_GET_GENERATE(f)                                    \
    2498                 : /* The least significant bits of the key are ignored. */                \
    2499                 : static inline void *                                                    \
    2500                 : f(malloc_rtree_t *rtree, uintptr_t key)                                 \
    2501                 : {                                                                       \
    2502                 :         void *ret;                                                      \
    2503                 :         uintptr_t subkey;                                               \
    2504                 :         unsigned i, lshift, height, bits;                               \
    2505                 :         void **node, **child;                                           \
    2506                 :                                                                         \
    2507                 :         MALLOC_RTREE_LOCK(&rtree->lock);                         \
    2508                 :         for (i = lshift = 0, height = rtree->height, node = rtree->root;\
    2509                 :             i < height - 1;                                          \
    2510                 :             i++, lshift += bits, node = child) {                        \
    2511                 :                 bits = rtree->level2bits[i];                         \
    2512                 :                 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);       \
    2513                 :                 child = (void**)node[subkey];                           \
    2514                 :                 if (child == NULL) {                                    \
    2515                 :                         MALLOC_RTREE_UNLOCK(&rtree->lock);               \
    2516                 :                         return (NULL);                                  \
    2517                 :                 }                                                       \
    2518                 :         }                                                               \
    2519                 :                                                                         \
    2520                 :         /*                                                              \
    2521                 :          * node is a leaf, so it contains values rather than node       \
    2522                 :          * pointers.                                                    \
    2523                 :          */                                                             \
    2524                 :         bits = rtree->level2bits[i];                                 \
    2525                 :         subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);               \
    2526                 :         ret = node[subkey];                                             \
    2527                 :         MALLOC_RTREE_UNLOCK(&rtree->lock);                               \
    2528                 :                                                                         \
    2529                 :         MALLOC_RTREE_GET_VALIDATE                                       \
    2530                 :         return (ret);                                                   \
    2531                 : }
    2532                 : 
    2533                 : #ifdef MALLOC_DEBUG
    2534                 : #  define MALLOC_RTREE_LOCK(l)          malloc_spin_lock(l)
    2535                 : #  define MALLOC_RTREE_UNLOCK(l)        malloc_spin_unlock(l)
    2536                 : #  define MALLOC_RTREE_GET_VALIDATE
    2537        14218142 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get_locked)
    2538                 : #  undef MALLOC_RTREE_LOCK
    2539                 : #  undef MALLOC_RTREE_UNLOCK
    2540                 : #  undef MALLOC_RTREE_GET_VALIDATE
    2541                 : #endif
    2542                 : 
    2543                 : #define MALLOC_RTREE_LOCK(l)
    2544                 : #define MALLOC_RTREE_UNLOCK(l)
    2545                 : #ifdef MALLOC_DEBUG
    2546                 :    /*
    2547                 :     * Suppose that it were possible for a jemalloc-allocated chunk to be
    2548                 :     * munmap()ped, followed by a different allocator in another thread re-using
    2549                 :     * overlapping virtual memory, all without invalidating the cached rtree
    2550                 :     * value.  The result would be a false positive (the rtree would claim that
    2551                 :     * jemalloc owns memory that it had actually discarded).  I don't think this
    2552                 :     * scenario is possible, but the following assertion is a prudent sanity
    2553                 :     * check.
    2554                 :     */
    2555                 : #  define MALLOC_RTREE_GET_VALIDATE                                     \
    2556                 :         assert(malloc_rtree_get_locked(rtree, key) == ret);
    2557                 : #else
    2558                 : #  define MALLOC_RTREE_GET_VALIDATE
    2559                 : #endif
    2560        14218181 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get)
    2561                 : #undef MALLOC_RTREE_LOCK
    2562                 : #undef MALLOC_RTREE_UNLOCK
    2563                 : #undef MALLOC_RTREE_GET_VALIDATE
    2564                 : 
    2565                 : static inline bool
    2566           48669 : malloc_rtree_set(malloc_rtree_t *rtree, uintptr_t key, void *val)
    2567                 : {
    2568                 :         uintptr_t subkey;
    2569                 :         unsigned i, lshift, height, bits;
    2570                 :         void **node, **child;
    2571                 : 
    2572           48669 :         malloc_spin_lock(&rtree->lock);
    2573           97338 :         for (i = lshift = 0, height = rtree->height, node = rtree->root;
    2574           48669 :             i < height - 1;
    2575               0 :             i++, lshift += bits, node = child) {
    2576               0 :                 bits = rtree->level2bits[i];
    2577               0 :                 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
    2578               0 :                 child = (void**)node[subkey];
    2579               0 :                 if (child == NULL) {
    2580               0 :                         child = (void**)base_calloc(1, sizeof(void *) <<
    2581               0 :                             rtree->level2bits[i+1]);
    2582               0 :                         if (child == NULL) {
    2583               0 :                                 malloc_spin_unlock(&rtree->lock);
    2584               0 :                                 return (true);
    2585                 :                         }
    2586               0 :                         node[subkey] = child;
    2587                 :                 }
    2588                 :         }
    2589                 : 
    2590                 :         /* node is a leaf, so it contains values rather than node pointers. */
    2591           48669 :         bits = rtree->level2bits[i];
    2592           48669 :         subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
    2593           48669 :         node[subkey] = val;
    2594           48669 :         malloc_spin_unlock(&rtree->lock);
    2595                 : 
    2596           48669 :         return (false);
    2597                 : }
    2598                 : #endif
    2599                 : 
    2600                 : #if defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)
    2601                 : 
    2602                 : /* Allocate an aligned chunk while maintaining a 1:1 correspondence between
    2603                 :  * mmap and unmap calls.  This is important on Windows, but not elsewhere. */
    2604                 : static void *
    2605                 : chunk_alloc_mmap(size_t size, bool pagefile)
    2606                 : {
    2607                 :         void *ret;
    2608                 : #ifndef JEMALLOC_USES_MAP_ALIGN
    2609                 :         size_t offset;
    2610                 : #endif
    2611                 :         int pfd;
    2612                 : 
    2613                 : #ifdef MALLOC_PAGEFILE
    2614                 :         if (opt_pagefile && pagefile) {
    2615                 :                 pfd = pagefile_init(size);
    2616                 :                 if (pfd == -1)
    2617                 :                         return (NULL);
    2618                 :         } else
    2619                 : #endif
    2620                 :                 pfd = -1;
    2621                 : 
    2622                 : #ifdef JEMALLOC_USES_MAP_ALIGN
    2623                 :         ret = pages_map_align(size, pfd, chunksize);
    2624                 : #else
    2625                 :         ret = pages_map(NULL, size, pfd);
    2626                 :         if (ret == NULL)
    2627                 :                 goto RETURN;
    2628                 : 
    2629                 :         offset = CHUNK_ADDR2OFFSET(ret);
    2630                 :         if (offset != 0) {
    2631                 :                 /* Deallocate, then try to allocate at (ret + size - offset). */
    2632                 :                 pages_unmap(ret, size);
    2633                 :                 ret = pages_map((void *)((uintptr_t)ret + size - offset), size,
    2634                 :                     pfd);
    2635                 :                 while (ret == NULL) {
    2636                 :                         /*
    2637                 :                          * Over-allocate in order to map a memory region that
    2638                 :                          * is definitely large enough.
    2639                 :                          */
    2640                 :                         ret = pages_map(NULL, size + chunksize, -1);
    2641                 :                         if (ret == NULL)
    2642                 :                                 goto RETURN;
    2643                 :                         /*
    2644                 :                          * Deallocate, then allocate the correct size, within
    2645                 :                          * the over-sized mapping.
    2646                 :                          */
    2647                 :                         offset = CHUNK_ADDR2OFFSET(ret);
    2648                 :                         pages_unmap(ret, size + chunksize);
    2649                 :                         if (offset == 0)
    2650                 :                                 ret = pages_map(ret, size, pfd);
    2651                 :                         else {
    2652                 :                                 ret = pages_map((void *)((uintptr_t)ret +
    2653                 :                                     chunksize - offset), size, pfd);
    2654                 :                         }
    2655                 :                         /*
    2656                 :                          * Failure here indicates a race with another thread, so
    2657                 :                          * try again.
    2658                 :                          */
    2659                 :                 }
    2660                 :         }
    2661                 : RETURN:
    2662                 : #endif
    2663                 : #ifdef MALLOC_PAGEFILE
    2664                 :         if (pfd != -1)
    2665                 :                 pagefile_close(pfd);
    2666                 : #endif
    2667                 : #ifdef MALLOC_STATS
    2668                 :         if (ret != NULL)
    2669                 :                 stats_chunks.nchunks += (size / chunksize);
    2670                 : #endif
    2671                 :         return (ret);
    2672                 : }
    2673                 : 
    2674                 : #else /* ! (defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)) */
    2675                 : 
    2676                 : /*
    2677                 :  * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
    2678                 :  * potentially avoid some system calls.
    2679                 :  */
    2680                 : #ifndef NO_TLS
    2681                 : static __thread bool    mmap_unaligned_tls __attribute__((tls_model("initial-exec")));
    2682                 : #define MMAP_UNALIGNED_GET()    mmap_unaligned_tls
    2683                 : #define MMAP_UNALIGNED_SET(v)   do {                                    \
    2684                 :         mmap_unaligned_tls = (v);                                       \
    2685                 : } while (0)
    2686                 : #else
    2687                 : #define NEEDS_PTHREAD_MMAP_UNALIGNED_TSD
    2688                 : static pthread_key_t    mmap_unaligned_tsd;
    2689                 : #define MMAP_UNALIGNED_GET()    ((bool)pthread_getspecific(mmap_unaligned_tsd))
    2690                 : #define MMAP_UNALIGNED_SET(v)   do {                                    \
    2691                 :         pthread_setspecific(mmap_unaligned_tsd, (void *)(v));           \
    2692                 : } while (0)
    2693                 : #endif
    2694                 : 
    2695                 : /* chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked from upstream
    2696                 :  * jemalloc 2.2.3 to fix Mozilla bug 694896, enable jemalloc on Mac 10.7. */
    2697                 : 
    2698                 : static void *
    2699            7837 : chunk_alloc_mmap_slow(size_t size, bool unaligned)
    2700                 : {
    2701                 :         void *ret;
    2702                 :         size_t offset;
    2703                 : 
    2704                 :         /* Beware size_t wrap-around. */
    2705            7837 :         if (size + chunksize <= size)
    2706               0 :                 return (NULL);
    2707                 : 
    2708            7837 :         ret = pages_map(NULL, size + chunksize, -1);
    2709            7837 :         if (ret == NULL)
    2710               0 :                 return (NULL);
    2711                 : 
    2712                 :         /* Clean up unneeded leading/trailing space. */
    2713            7837 :         offset = CHUNK_ADDR2OFFSET(ret);
    2714            7837 :         if (offset != 0) {
    2715                 :                 /* Note that mmap() returned an unaligned mapping. */
    2716            2263 :                 unaligned = true;
    2717                 : 
    2718                 :                 /* Leading space. */
    2719            2263 :                 pages_unmap(ret, chunksize - offset);
    2720                 : 
    2721            2263 :                 ret = (void *)((uintptr_t)ret +
    2722            2263 :                     (chunksize - offset));
    2723                 : 
    2724                 :                 /* Trailing space. */
    2725            2263 :                 pages_unmap((void *)((uintptr_t)ret + size),
    2726                 :                     offset);
    2727                 :         } else {
    2728                 :                 /* Trailing space only. */
    2729            5574 :                 pages_unmap((void *)((uintptr_t)ret + size),
    2730                 :                     chunksize);
    2731                 :         }
    2732                 : 
    2733                 :         /*
    2734                 :          * If mmap() returned an aligned mapping, reset mmap_unaligned so that
    2735                 :          * the next chunk_alloc_mmap() execution tries the fast allocation
    2736                 :          * method.
    2737                 :          */
    2738            7837 :         if (unaligned == false)
    2739            5574 :                 MMAP_UNALIGNED_SET(false);
    2740                 : 
    2741            7837 :         return (ret);
    2742                 : }
    2743                 : 
    2744                 : static void *
    2745           40731 : chunk_alloc_mmap(size_t size, bool pagefile)
    2746                 : {
    2747                 :         void *ret;
    2748                 : 
    2749                 :         /*
    2750                 :          * Ideally, there would be a way to specify alignment to mmap() (like
    2751                 :          * NetBSD has), but in the absence of such a feature, we have to work
    2752                 :          * hard to efficiently create aligned mappings.  The reliable, but
    2753                 :          * slow method is to create a mapping that is over-sized, then trim the
    2754                 :          * excess.  However, that always results in at least one call to
    2755                 :          * pages_unmap().
    2756                 :          *
    2757                 :          * A more optimistic approach is to try mapping precisely the right
    2758                 :          * amount, then try to append another mapping if alignment is off.  In
    2759                 :          * practice, this works out well as long as the application is not
    2760                 :          * interleaving mappings via direct mmap() calls.  If we do run into a
    2761                 :          * situation where there is an interleaved mapping and we are unable to
    2762                 :          * extend an unaligned mapping, our best option is to switch to the
    2763                 :          * slow method until mmap() returns another aligned mapping.  This will
    2764                 :          * tend to leave a gap in the memory map that is too small to cause
    2765                 :          * later problems for the optimistic method.
    2766                 :          *
    2767                 :          * Another possible confounding factor is address space layout
    2768                 :          * randomization (ASLR), which causes mmap(2) to disregard the
    2769                 :          * requested address.  mmap_unaligned tracks whether the previous
    2770                 :          * chunk_alloc_mmap() execution received any unaligned or relocated
    2771                 :          * mappings, and if so, the current execution will immediately fall
    2772                 :          * back to the slow method.  However, we keep track of whether the fast
    2773                 :          * method would have succeeded, and if so, we make a note to try the
    2774                 :          * fast method next time.
    2775                 :          */
    2776                 : 
    2777           40731 :         if (MMAP_UNALIGNED_GET() == false) {
    2778                 :                 size_t offset;
    2779                 : 
    2780           32894 :                 ret = pages_map(NULL, size, -1);
    2781           32894 :                 if (ret == NULL)
    2782              19 :                         return (NULL);
    2783                 : 
    2784           32875 :                 offset = CHUNK_ADDR2OFFSET(ret);
    2785           32875 :                 if (offset != 0) {
    2786           21950 :                         MMAP_UNALIGNED_SET(true);
    2787                 :                         /* Try to extend chunk boundary. */
    2788           21950 :                         if (pages_map((void *)((uintptr_t)ret + size),
    2789                 :                             chunksize - offset, -1) == NULL) {
    2790                 :                                 /*
    2791                 :                                  * Extension failed.  Clean up, then revert to
    2792                 :                                  * the reliable-but-expensive method.
    2793                 :                                  */
    2794               0 :                                 pages_unmap(ret, size);
    2795               0 :                                 ret = chunk_alloc_mmap_slow(size, true);
    2796                 :                         } else {
    2797                 :                                 /* Clean up unneeded leading space. */
    2798           21950 :                                 pages_unmap(ret, chunksize - offset);
    2799           21950 :                                 ret = (void *)((uintptr_t)ret + (chunksize -
    2800                 :                                     offset));
    2801                 :                         }
    2802                 :                 }
    2803                 :         } else
    2804            7837 :                 ret = chunk_alloc_mmap_slow(size, false);
    2805                 : 
    2806           40712 :         return (ret);
    2807                 : }
    2808                 : 
    2809                 : #endif /* defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE) */
    2810                 : 
    2811                 : #ifdef MALLOC_PAGEFILE
    2812                 : static int
    2813                 : pagefile_init(size_t size)
    2814                 : {
    2815                 :         int ret;
    2816                 :         size_t i;
    2817                 :         char pagefile_path[PATH_MAX];
    2818                 :         char zbuf[MALLOC_PAGEFILE_WRITE_SIZE];
    2819                 : 
    2820                 :         /*
    2821                 :          * Create a temporary file, then immediately unlink it so that it will
    2822                 :          * not persist.
    2823                 :          */
    2824                 :         strcpy(pagefile_path, pagefile_templ);
    2825                 :         ret = mkstemp(pagefile_path);
    2826                 :         if (ret == -1)
    2827                 :                 return (ret);
    2828                 :         if (unlink(pagefile_path)) {
    2829                 :                 char buf[STRERROR_BUF];
    2830                 : 
    2831                 :                 strerror_r(errno, buf, sizeof(buf));
    2832                 :                 _malloc_message(_getprogname(), ": (malloc) Error in unlink(\"",
    2833                 :                     pagefile_path, "\"):");
    2834                 :                 _malloc_message(buf, "\n", "", "");
    2835                 :                 if (opt_abort)
    2836                 :                         abort();
    2837                 :         }
    2838                 : 
    2839                 :         /*
    2840                 :          * Write sequential zeroes to the file in order to assure that disk
    2841                 :          * space is committed, with minimal fragmentation.  It would be
    2842                 :          * sufficient to write one zero per disk block, but that potentially
    2843                 :          * results in more system calls, for no real gain.
    2844                 :          */
    2845                 :         memset(zbuf, 0, sizeof(zbuf));
    2846                 :         for (i = 0; i < size; i += sizeof(zbuf)) {
    2847                 :                 if (write(ret, zbuf, sizeof(zbuf)) != sizeof(zbuf)) {
    2848                 :                         if (errno != ENOSPC) {
    2849                 :                                 char buf[STRERROR_BUF];
    2850                 : 
    2851                 :                                 strerror_r(errno, buf, sizeof(buf));
    2852                 :                                 _malloc_message(_getprogname(),
    2853                 :                                     ": (malloc) Error in write(): ", buf, "\n");
    2854                 :                                 if (opt_abort)
    2855                 :                                         abort();
    2856                 :                         }
    2857                 :                         pagefile_close(ret);
    2858                 :                         return (-1);
    2859                 :                 }
    2860                 :         }
    2861                 : 
    2862                 :         return (ret);
    2863                 : }
    2864                 : 
    2865                 : static void
    2866                 : pagefile_close(int pfd)
    2867                 : {
    2868                 : 
    2869                 :         if (close(pfd)) {
    2870                 :                 char buf[STRERROR_BUF];
    2871                 : 
    2872                 :                 strerror_r(errno, buf, sizeof(buf));
    2873                 :                 _malloc_message(_getprogname(),
    2874                 :                     ": (malloc) Error in close(): ", buf, "\n");
    2875                 :                 if (opt_abort)
    2876                 :                         abort();
    2877                 :         }
    2878                 : }
    2879                 : #endif
    2880                 : 
    2881                 : static void *
    2882           40731 : chunk_alloc(size_t size, bool zero, bool pagefile)
    2883                 : {
    2884                 :         void *ret;
    2885                 : 
    2886           40731 :         assert(size != 0);
    2887           40731 :         assert((size & chunksize_mask) == 0);
    2888                 : 
    2889           40731 :         ret = chunk_alloc_mmap(size, pagefile);
    2890           40731 :         if (ret != NULL) {
    2891           40712 :                 goto RETURN;
    2892                 :         }
    2893                 : 
    2894                 :         /* All strategies for allocation failed. */
    2895              19 :         ret = NULL;
    2896                 : RETURN:
    2897                 : #ifdef MALLOC_STATS
    2898           40731 :         if (ret != NULL)
    2899           40712 :                 stats_chunks.curchunks += (size / chunksize);
    2900           40731 :         if (stats_chunks.curchunks > stats_chunks.highchunks)
    2901           35557 :                 stats_chunks.highchunks = stats_chunks.curchunks;
    2902                 : #endif
    2903                 : 
    2904                 : #ifdef MALLOC_VALIDATE
    2905           40731 :         if (ret != NULL) {
    2906           40712 :                 if (malloc_rtree_set(chunk_rtree, (uintptr_t)ret, ret)) {
    2907               0 :                         chunk_dealloc(ret, size);
    2908               0 :                         return (NULL);
    2909                 :                 }
    2910                 :         }
    2911                 : #endif
    2912                 : 
    2913           40731 :         assert(CHUNK_ADDR2BASE(ret) == ret);
    2914           40731 :         return (ret);
    2915                 : }
    2916                 : 
    2917                 : static void
    2918            7957 : chunk_dealloc_mmap(void *chunk, size_t size)
    2919                 : {
    2920                 : 
    2921            7957 :         pages_unmap(chunk, size);
    2922            7957 : }
    2923                 : 
    2924                 : static void
    2925            7957 : chunk_dealloc(void *chunk, size_t size)
    2926                 : {
    2927                 : 
    2928            7957 :         assert(chunk != NULL);
    2929            7957 :         assert(CHUNK_ADDR2BASE(chunk) == chunk);
    2930            7957 :         assert(size != 0);
    2931            7957 :         assert((size & chunksize_mask) == 0);
    2932                 : 
    2933                 : #ifdef MALLOC_STATS
    2934            7957 :         stats_chunks.curchunks -= (size / chunksize);
    2935                 : #endif
    2936                 : #ifdef MALLOC_VALIDATE
    2937            7957 :         malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL);
    2938                 : #endif
    2939                 : 
    2940            7957 :         chunk_dealloc_mmap(chunk, size);
    2941            7957 : }
    2942                 : 
    2943                 : /*
    2944                 :  * End chunk management functions.
    2945                 :  */
    2946                 : /******************************************************************************/
    2947                 : /*
    2948                 :  * Begin arena.
    2949                 :  */
    2950                 : 
    2951                 : /*
    2952                 :  * Choose an arena based on a per-thread value (fast-path code, calls slow-path
    2953                 :  * code if necessary).
    2954                 :  */
    2955                 : static inline arena_t *
    2956       125762912 : choose_arena(void)
    2957                 : {
    2958                 :         arena_t *ret;
    2959                 : 
    2960                 :         /*
    2961                 :          * We can only use TLS if this is a PIC library, since for the static
    2962                 :          * library version, libc's malloc is used by TLS allocation, which
    2963                 :          * introduces a bootstrapping issue.
    2964                 :          */
    2965                 : #ifndef NO_TLS
    2966       125762912 :         if (__isthreaded == false) {
    2967                 :             /* Avoid the overhead of TLS for single-threaded operation. */
    2968               0 :             return (arenas[0]);
    2969                 :         }
    2970                 : 
    2971                 : #  ifdef MOZ_MEMORY_WINDOWS
    2972                 :         ret = (arena_t*)TlsGetValue(tlsIndex);
    2973                 : #  else
    2974       125762912 :         ret = arenas_map;
    2975                 : #  endif
    2976                 : 
    2977       125762912 :         if (ret == NULL) {
    2978           11993 :                 ret = choose_arena_hard();
    2979           11993 :                 assert(ret != NULL);
    2980                 :         }
    2981                 : #else
    2982                 :         if (__isthreaded && narenas > 1) {
    2983                 :                 unsigned long ind;
    2984                 : 
    2985                 :                 /*
    2986                 :                  * Hash _pthread_self() to one of the arenas.  There is a prime
    2987                 :                  * number of arenas, so this has a reasonable chance of
    2988                 :                  * working.  Even so, the hashing can be easily thwarted by
    2989                 :                  * inconvenient _pthread_self() values.  Without specific
    2990                 :                  * knowledge of how _pthread_self() calculates values, we can't
    2991                 :                  * easily do much better than this.
    2992                 :                  */
    2993                 :                 ind = (unsigned long) _pthread_self() % narenas;
    2994                 : 
    2995                 :                 /*
    2996                 :                  * Optimistially assume that arenas[ind] has been initialized.
    2997                 :                  * At worst, we find out that some other thread has already
    2998                 :                  * done so, after acquiring the lock in preparation.  Note that
    2999                 :                  * this lazy locking also has the effect of lazily forcing
    3000                 :                  * cache coherency; without the lock acquisition, there's no
    3001                 :                  * guarantee that modification of arenas[ind] by another thread
    3002                 :                  * would be seen on this CPU for an arbitrary amount of time.
    3003                 :                  *
    3004                 :                  * In general, this approach to modifying a synchronized value
    3005                 :                  * isn't a good idea, but in this case we only ever modify the
    3006                 :                  * value once, so things work out well.
    3007                 :                  */
    3008                 :                 ret = arenas[ind];
    3009                 :                 if (ret == NULL) {
    3010                 :                         /*
    3011                 :                          * Avoid races with another thread that may have already
    3012                 :                          * initialized arenas[ind].
    3013                 :                          */
    3014                 :                         malloc_spin_lock(&arenas_lock);
    3015                 :                         if (arenas[ind] == NULL)
    3016                 :                                 ret = arenas_extend((unsigned)ind);
    3017                 :                         else
    3018                 :                                 ret = arenas[ind];
    3019                 :                         malloc_spin_unlock(&arenas_lock);
    3020                 :                 }
    3021                 :         } else
    3022                 :                 ret = arenas[0];
    3023                 : #endif
    3024                 : 
    3025       125762912 :         assert(ret != NULL);
    3026       125762912 :         return (ret);
    3027                 : }
    3028                 : 
    3029                 : #ifndef NO_TLS
    3030                 : /*
    3031                 :  * Choose an arena based on a per-thread value (slow-path code only, called
    3032                 :  * only by choose_arena()).
    3033                 :  */
    3034                 : static arena_t *
    3035           11993 : choose_arena_hard(void)
    3036                 : {
    3037                 :         arena_t *ret;
    3038                 : 
    3039           11993 :         assert(__isthreaded);
    3040                 : 
    3041                 : #ifdef MALLOC_BALANCE
    3042                 :         /* Seed the PRNG used for arena load balancing. */
    3043                 :         SPRN(balance, (uint32_t)(uintptr_t)(_pthread_self()));
    3044                 : #endif
    3045                 : 
    3046           11993 :         if (narenas > 1) {
    3047                 : #ifdef MALLOC_BALANCE
    3048                 :                 unsigned ind;
    3049                 : 
    3050                 :                 ind = PRN(balance, narenas_2pow);
    3051                 :                 if ((ret = arenas[ind]) == NULL) {
    3052                 :                         malloc_spin_lock(&arenas_lock);
    3053                 :                         if ((ret = arenas[ind]) == NULL)
    3054                 :                                 ret = arenas_extend(ind);
    3055                 :                         malloc_spin_unlock(&arenas_lock);
    3056                 :                 }
    3057                 : #else
    3058               0 :                 malloc_spin_lock(&arenas_lock);
    3059               0 :                 if ((ret = arenas[next_arena]) == NULL)
    3060               0 :                         ret = arenas_extend(next_arena);
    3061               0 :                 next_arena = (next_arena + 1) % narenas;
    3062               0 :                 malloc_spin_unlock(&arenas_lock);
    3063                 : #endif
    3064                 :         } else
    3065           11993 :                 ret = arenas[0];
    3066                 : 
    3067                 : #ifdef MOZ_MEMORY_WINDOWS
    3068                 :         TlsSetValue(tlsIndex, ret);
    3069                 : #else
    3070           11993 :         arenas_map = ret;
    3071                 : #endif
    3072                 : 
    3073           11993 :         return (ret);
    3074                 : }
    3075                 : #endif
    3076                 : 
    3077                 : static inline int
    3078         1802517 : arena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b)
    3079                 : {
    3080         1802517 :         uintptr_t a_chunk = (uintptr_t)a;
    3081         1802517 :         uintptr_t b_chunk = (uintptr_t)b;
    3082                 : 
    3083         1802517 :         assert(a != NULL);
    3084         1802517 :         assert(b != NULL);
    3085                 : 
    3086         1802517 :         return ((a_chunk > b_chunk) - (a_chunk < b_chunk));
    3087                 : }
    3088                 : 
    3089                 : /* Wrap red-black tree macros in functions. */
    3090         1137110 : rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
    3091         1137110 :     arena_chunk_t, link_dirty, arena_chunk_comp)
    3092                 : 
    3093                 : static inline int
    3094        20078373 : arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
    3095                 : {
    3096        20078373 :         uintptr_t a_mapelm = (uintptr_t)a;
    3097        20078373 :         uintptr_t b_mapelm = (uintptr_t)b;
    3098                 : 
    3099        20078373 :         assert(a != NULL);
    3100        20078373 :         assert(b != NULL);
    3101                 : 
    3102        20078373 :         return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm));
    3103                 : }
    3104                 : 
    3105                 : /* Wrap red-black tree macros in functions. */
    3106        14611500 : rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, link,
    3107        14611500 :     arena_run_comp)
    3108                 : 
    3109                 : static inline int
    3110        69472920 : arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
    3111                 : {
    3112                 :         int ret;
    3113        69472920 :         size_t a_size = a->bits & ~pagesize_mask;
    3114        69472920 :         size_t b_size = b->bits & ~pagesize_mask;
    3115                 : 
    3116        69472920 :         ret = (a_size > b_size) - (a_size < b_size);
    3117        69472920 :         if (ret == 0) {
    3118                 :                 uintptr_t a_mapelm, b_mapelm;
    3119                 : 
    3120        21029319 :                 if ((a->bits & CHUNK_MAP_KEY) == 0)
    3121        18576310 :                         a_mapelm = (uintptr_t)a;
    3122                 :                 else {
    3123                 :                         /*
    3124                 :                          * Treat keys as though they are lower than anything
    3125                 :                          * else.
    3126                 :                          */
    3127         2453009 :                         a_mapelm = 0;
    3128                 :                 }
    3129        21029319 :                 b_mapelm = (uintptr_t)b;
    3130                 : 
    3131        21029319 :                 ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm);
    3132                 :         }
    3133                 : 
    3134        69472920 :         return (ret);
    3135                 : }
    3136                 : 
    3137                 : /* Wrap red-black tree macros in functions. */
    3138        29949484 : rb_wrap(static, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, link,
    3139        29949484 :     arena_avail_comp)
    3140                 : 
    3141                 : static inline void *
    3142       123481815 : arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
    3143                 : {
    3144                 :         void *ret;
    3145                 :         unsigned i, mask, bit, regind;
    3146                 : 
    3147       123481815 :         assert(run->magic == ARENA_RUN_MAGIC);
    3148       123481815 :         assert(run->regs_minelm < bin->regs_mask_nelms);
    3149                 : 
    3150                 :         /*
    3151                 :          * Move the first check outside the loop, so that run->regs_minelm can
    3152                 :          * be updated unconditionally, without the possibility of updating it
    3153                 :          * multiple times.
    3154                 :          */
    3155       123481815 :         i = run->regs_minelm;
    3156       123481815 :         mask = run->regs_mask[i];
    3157       123481815 :         if (mask != 0) {
    3158                 :                 /* Usable allocation found. */
    3159       120399106 :                 bit = ffs((int)mask) - 1;
    3160                 : 
    3161       120399106 :                 regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
    3162       120399106 :                 assert(regind < bin->nregs);
    3163       240798212 :                 ret = (void *)(((uintptr_t)run) + bin->reg0_offset
    3164       120399106 :                     + (bin->reg_size * regind));
    3165                 : 
    3166                 :                 /* Clear bit. */
    3167       120399106 :                 mask ^= (1U << bit);
    3168       120399106 :                 run->regs_mask[i] = mask;
    3169                 : 
    3170       120399106 :                 return (ret);
    3171                 :         }
    3172                 : 
    3173         3463151 :         for (i++; i < bin->regs_mask_nelms; i++) {
    3174         3463151 :                 mask = run->regs_mask[i];
    3175         3463151 :                 if (mask != 0) {
    3176                 :                         /* Usable allocation found. */
    3177         3082709 :                         bit = ffs((int)mask) - 1;
    3178                 : 
    3179         3082709 :                         regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
    3180         3082709 :                         assert(regind < bin->nregs);
    3181         6165418 :                         ret = (void *)(((uintptr_t)run) + bin->reg0_offset
    3182         3082709 :                             + (bin->reg_size * regind));
    3183                 : 
    3184                 :                         /* Clear bit. */
    3185         3082709 :                         mask ^= (1U << bit);
    3186         3082709 :                         run->regs_mask[i] = mask;
    3187                 : 
    3188                 :                         /*
    3189                 :                          * Make a note that nothing before this element
    3190                 :                          * contains a free region.
    3191                 :                          */
    3192         3082709 :                         run->regs_minelm = i; /* Low payoff: + (mask == 0); */
    3193                 : 
    3194         3082709 :                         return (ret);
    3195                 :                 }
    3196                 :         }
    3197                 :         /* Not reached. */
    3198               0 :         assert(0);
    3199                 :         return (NULL);
    3200                 : }
    3201                 : 
    3202                 : static inline void
    3203       119028910 : arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size)
    3204                 : {
    3205                 :         /*
    3206                 :          * To divide by a number D that is not a power of two we multiply
    3207                 :          * by (2^21 / D) and then right shift by 21 positions.
    3208                 :          *
    3209                 :          *   X / D
    3210                 :          *
    3211                 :          * becomes
    3212                 :          *
    3213                 :          *   (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT
    3214                 :          */
    3215                 : #define SIZE_INV_SHIFT 21
    3216                 : #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1)
    3217                 :         static const unsigned size_invs[] = {
    3218                 :             SIZE_INV(3),
    3219                 :             SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
    3220                 :             SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
    3221                 :             SIZE_INV(12),SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
    3222                 :             SIZE_INV(16),SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
    3223                 :             SIZE_INV(20),SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
    3224                 :             SIZE_INV(24),SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
    3225                 :             SIZE_INV(28),SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
    3226                 : #if (QUANTUM_2POW_MIN < 4)
    3227                 :             ,
    3228                 :             SIZE_INV(32), SIZE_INV(33), SIZE_INV(34), SIZE_INV(35),
    3229                 :             SIZE_INV(36), SIZE_INV(37), SIZE_INV(38), SIZE_INV(39),
    3230                 :             SIZE_INV(40), SIZE_INV(41), SIZE_INV(42), SIZE_INV(43),
    3231                 :             SIZE_INV(44), SIZE_INV(45), SIZE_INV(46), SIZE_INV(47),
    3232                 :             SIZE_INV(48), SIZE_INV(49), SIZE_INV(50), SIZE_INV(51),
    3233                 :             SIZE_INV(52), SIZE_INV(53), SIZE_INV(54), SIZE_INV(55),
    3234                 :             SIZE_INV(56), SIZE_INV(57), SIZE_INV(58), SIZE_INV(59),
    3235                 :             SIZE_INV(60), SIZE_INV(61), SIZE_INV(62), SIZE_INV(63)
    3236                 : #endif
    3237                 :         };
    3238                 :         unsigned diff, regind, elm, bit;
    3239                 : 
    3240       119028910 :         assert(run->magic == ARENA_RUN_MAGIC);
    3241                 :         assert(((sizeof(size_invs)) / sizeof(unsigned)) + 3
    3242                 :             >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN));
    3243                 : 
    3244                 :         /*
    3245                 :          * Avoid doing division with a variable divisor if possible.  Using
    3246                 :          * actual division here can reduce allocator throughput by over 20%!
    3247                 :          */
    3248       119028910 :         diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
    3249       119028910 :         if ((size & (size - 1)) == 0) {
    3250                 :                 /*
    3251                 :                  * log2_table allows fast division of a power of two in the
    3252                 :                  * [1..128] range.
    3253                 :                  *
    3254                 :                  * (x / divisor) becomes (x >> log2_table[divisor - 1]).
    3255                 :                  */
    3256                 :                 static const unsigned char log2_table[] = {
    3257                 :                     0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
    3258                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
    3259                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    3260                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
    3261                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    3262                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    3263                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    3264                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
    3265                 :                 };
    3266                 : 
    3267        90402814 :                 if (size <= 128)
    3268        71000900 :                         regind = (diff >> log2_table[size - 1]);
    3269        19401914 :                 else if (size <= 32768)
    3270        19401914 :                         regind = diff >> (8 + log2_table[(size >> 8) - 1]);
    3271                 :                 else {
    3272                 :                         /*
    3273                 :                          * The run size is too large for us to use the lookup
    3274                 :                          * table.  Use real division.
    3275                 :                          */
    3276               0 :                         regind = diff / size;
    3277                 :                 }
    3278        28626096 :         } else if (size <= ((sizeof(size_invs) / sizeof(unsigned))
    3279                 :             << QUANTUM_2POW_MIN) + 2) {
    3280        28473100 :                 regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff;
    3281        28473100 :                 regind >>= SIZE_INV_SHIFT;
    3282                 :         } else {
    3283                 :                 /*
    3284                 :                  * size_invs isn't large enough to handle this size class, so
    3285                 :                  * calculate regind using actual division.  This only happens
    3286                 :                  * if the user increases small_max via the 'S' runtime
    3287                 :                  * configuration option.
    3288                 :                  */
    3289          152996 :                 regind = diff / size;
    3290                 :         };
    3291       119028910 :         assert(diff == regind * size);
    3292       119028910 :         assert(regind < bin->nregs);
    3293                 : 
    3294       119028910 :         elm = regind >> (SIZEOF_INT_2POW + 3);
    3295       119028910 :         if (elm < run->regs_minelm)
    3296         2461777 :                 run->regs_minelm = elm;
    3297       119028910 :         bit = regind - (elm << (SIZEOF_INT_2POW + 3));
    3298       119028910 :         assert((run->regs_mask[elm] & (1U << bit)) == 0);
    3299       119028910 :         run->regs_mask[elm] |= (1U << bit);
    3300                 : #undef SIZE_INV
    3301                 : #undef SIZE_INV_SHIFT
    3302       119028910 : }
    3303                 : 
    3304                 : static void
    3305         6738038 : arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
    3306                 :     bool zero)
    3307                 : {
    3308                 :         arena_chunk_t *chunk;
    3309                 :         size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i;
    3310                 : 
    3311         6738038 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
    3312         6738038 :         old_ndirty = chunk->ndirty;
    3313         6738038 :         run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk)
    3314                 :             >> pagesize_2pow);
    3315         6738038 :         total_pages = (chunk->map[run_ind].bits & ~pagesize_mask) >>
    3316                 :             pagesize_2pow;
    3317         6738038 :         need_pages = (size >> pagesize_2pow);
    3318         6738038 :         assert(need_pages > 0);
    3319         6738038 :         assert(need_pages <= total_pages);
    3320         6738038 :         rem_pages = total_pages - need_pages;
    3321                 : 
    3322         6738038 :         arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]);
    3323                 : 
    3324                 :         /* Keep track of trailing unused pages for later use. */
    3325         6738038 :         if (rem_pages > 0) {
    3326        10926536 :                 chunk->map[run_ind+need_pages].bits = (rem_pages <<
    3327         5463268 :                     pagesize_2pow) | (chunk->map[run_ind+need_pages].bits &
    3328                 :                     pagesize_mask);
    3329        10926536 :                 chunk->map[run_ind+total_pages-1].bits = (rem_pages <<
    3330         5463268 :                     pagesize_2pow) | (chunk->map[run_ind+total_pages-1].bits &
    3331                 :                     pagesize_mask);
    3332         5463268 :                 arena_avail_tree_insert(&arena->runs_avail,
    3333         5463268 :                     &chunk->map[run_ind+need_pages]);
    3334                 :         }
    3335                 : 
    3336        36556213 :         for (i = 0; i < need_pages; i++) {
    3337                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
    3338                 :                 /*
    3339                 :                  * Commit decommitted pages if necessary.  If a decommitted
    3340                 :                  * page is encountered, commit all needed adjacent decommitted
    3341                 :                  * pages in one operation, in order to reduce system call
    3342                 :                  * overhead.
    3343                 :                  */
    3344        29818175 :                 if (chunk->map[run_ind + i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) {
    3345                 :                         size_t j;
    3346                 : 
    3347                 :                         /*
    3348                 :                          * Advance i+j to just past the index of the last page
    3349                 :                          * to commit.  Clear CHUNK_MAP_DECOMMITTED and
    3350                 :                          * CHUNK_MAP_MADVISED along the way.
    3351                 :                          */
    3352        19513973 :                         for (j = 0; i + j < need_pages && (chunk->map[run_ind +
    3353        16154525 :                             i + j].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED); j++) {
    3354                 :                                 /* DECOMMITTED and MADVISED are mutually exclusive. */
    3355         8076508 :                                 assert(!(chunk->map[run_ind + i + j].bits & CHUNK_MAP_DECOMMITTED &&
    3356                 :                                          chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED));
    3357                 : 
    3358         8076508 :                                 chunk->map[run_ind + i + j].bits &=
    3359                 :                                     ~CHUNK_MAP_MADVISED_OR_DECOMMITTED;
    3360                 :                         }
    3361                 : 
    3362                 : #  ifdef MALLOC_DECOMMIT
    3363                 :                         pages_commit((void *)((uintptr_t)chunk + ((run_ind + i)
    3364                 :                             << pagesize_2pow)), (j << pagesize_2pow));
    3365                 : #    ifdef MALLOC_STATS
    3366                 :                         arena->stats.ncommit++;
    3367                 : #    endif
    3368                 : #  endif
    3369                 : 
    3370                 : #  ifdef MALLOC_STATS
    3371         1679724 :                         arena->stats.committed += j;
    3372                 : #  endif
    3373                 : 
    3374                 : #  ifndef MALLOC_DECOMMIT
    3375                 :                 }
    3376                 : #  else
    3377                 :                 } else /* No need to zero since commit zeros. */
    3378                 : #  endif
    3379                 : 
    3380                 : #endif
    3381                 : 
    3382                 :                 /* Zero if necessary. */
    3383        29818175 :                 if (zero) {
    3384         1405230 :                         if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED)
    3385                 :                             == 0) {
    3386                 :                                 VALGRIND_MALLOCLIKE_BLOCK((void *)((uintptr_t)
    3387                 :                                     chunk + ((run_ind + i) << pagesize_2pow)),
    3388                 :                                     pagesize, 0, false);
    3389         1236668 :                                 memset((void *)((uintptr_t)chunk + ((run_ind
    3390         1236668 :                                     + i) << pagesize_2pow)), 0, pagesize);
    3391                 :                                 VALGRIND_FREELIKE_BLOCK((void *)((uintptr_t)
    3392                 :                                     chunk + ((run_ind + i) << pagesize_2pow)),
    3393                 :                                     0);
    3394                 :                                 /* CHUNK_MAP_ZEROED is cleared below. */
    3395                 :                         }
    3396                 :                 }
    3397                 : 
    3398                 :                 /* Update dirty page accounting. */
    3399        29818175 :                 if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) {
    3400        21741667 :                         chunk->ndirty--;
    3401        21741667 :                         arena->ndirty--;
    3402                 :                         /* CHUNK_MAP_DIRTY is cleared below. */
    3403                 :                 }
    3404                 : 
    3405                 :                 /* Initialize the chunk map. */
    3406        29818175 :                 if (large) {
    3407        12146964 :                         chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE
    3408                 :                             | CHUNK_MAP_ALLOCATED;
    3409                 :                 } else {
    3410        35342422 :                         chunk->map[run_ind + i].bits = (size_t)run
    3411        17671211 :                             | CHUNK_MAP_ALLOCATED;
    3412                 :                 }
    3413                 :         }
    3414                 : 
    3415                 :         /*
    3416                 :          * Set the run size only in the first element for large runs.  This is
    3417                 :          * primarily a debugging aid, since the lack of size info for trailing
    3418                 :          * pages only matters if the application tries to operate on an
    3419                 :          * interior pointer.
    3420                 :          */
    3421         6738038 :         if (large)
    3422         2398466 :                 chunk->map[run_ind].bits |= size;
    3423                 : 
    3424         6738038 :         if (chunk->ndirty == 0 && old_ndirty > 0)
    3425          491283 :                 arena_chunk_tree_dirty_remove(&arena->chunks_dirty, chunk);
    3426         6738038 : }
    3427                 : 
    3428                 : static void
    3429           35193 : arena_chunk_init(arena_t *arena, arena_chunk_t *chunk)
    3430                 : {
    3431                 :         arena_run_t *run;
    3432                 :         size_t i;
    3433                 : 
    3434                 :         VALGRIND_MALLOCLIKE_BLOCK(chunk, (arena_chunk_header_npages <<
    3435                 :             pagesize_2pow), 0, false);
    3436                 : #ifdef MALLOC_STATS
    3437           35193 :         arena->stats.mapped += chunksize;
    3438                 : #endif
    3439                 : 
    3440           35193 :         chunk->arena = arena;
    3441                 : 
    3442                 :         /*
    3443                 :          * Claim that no pages are in use, since the header is merely overhead.
    3444                 :          */
    3445           35193 :         chunk->ndirty = 0;
    3446                 : 
    3447                 :         /* Initialize the map to contain one maximal free untouched run. */
    3448           35193 :         run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages <<
    3449                 :             pagesize_2pow));
    3450           70386 :         for (i = 0; i < arena_chunk_header_npages; i++)
    3451           35193 :                 chunk->map[i].bits = 0;
    3452           35193 :         chunk->map[i].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
    3453         8939022 :         for (i++; i < chunk_npages-1; i++) {
    3454         8903829 :                 chunk->map[i].bits = CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
    3455                 :         }
    3456           35193 :         chunk->map[chunk_npages-1].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
    3457                 : 
    3458                 : #ifdef MALLOC_DECOMMIT
    3459                 :         /*
    3460                 :          * Start out decommitted, in order to force a closer correspondence
    3461                 :          * between dirty pages and committed untouched pages.
    3462                 :          */
    3463                 :         pages_decommit(run, arena_maxclass);
    3464                 : #  ifdef MALLOC_STATS
    3465                 :         arena->stats.ndecommit++;
    3466                 :         arena->stats.decommitted += (chunk_npages - arena_chunk_header_npages);
    3467                 : #  endif
    3468                 : #endif
    3469                 : #ifdef MALLOC_STATS
    3470           35193 :         arena->stats.committed += arena_chunk_header_npages;
    3471                 : #endif
    3472                 : 
    3473                 :         /* Insert the run into the runs_avail tree. */
    3474           35193 :         arena_avail_tree_insert(&arena->runs_avail,
    3475           35193 :             &chunk->map[arena_chunk_header_npages]);
    3476                 : 
    3477                 : #ifdef MALLOC_DOUBLE_PURGE
    3478                 :         LinkedList_Init(&chunk->chunks_madvised_elem);
    3479                 : #endif
    3480           35193 : }
    3481                 : 
    3482                 : static void
    3483           37150 : arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
    3484                 : {
    3485                 : 
    3486           37150 :         if (arena->spare != NULL) {
    3487            2438 :                 if (arena->spare->ndirty > 0) {
    3488            4008 :                         arena_chunk_tree_dirty_remove(
    3489            2004 :                             &chunk->arena->chunks_dirty, arena->spare);
    3490            2004 :                         arena->ndirty -= arena->spare->ndirty;
    3491                 : #ifdef MALLOC_STATS
    3492            2004 :                         arena->stats.committed -= arena->spare->ndirty;
    3493                 : #endif
    3494                 :                 }
    3495                 : 
    3496                 : #ifdef MALLOC_DOUBLE_PURGE
    3497                 :                 /* This is safe to do even if arena->spare is not in the list. */
    3498                 :                 LinkedList_Remove(&arena->spare->chunks_madvised_elem);
    3499                 : #endif
    3500                 : 
    3501                 :                 VALGRIND_FREELIKE_BLOCK(arena->spare, 0);
    3502            2438 :                 chunk_dealloc((void *)arena->spare, chunksize);
    3503                 : #ifdef MALLOC_STATS
    3504            2438 :                 arena->stats.mapped -= chunksize;
    3505            2438 :                 arena->stats.committed -= arena_chunk_header_npages;
    3506                 : #endif
    3507                 :         }
    3508                 : 
    3509                 :         /*
    3510                 :          * Remove run from runs_avail, so that the arena does not use it.
    3511                 :          * Dirty page flushing only uses the chunks_dirty tree, so leaving this
    3512                 :          * chunk in the chunks_* trees is sufficient for that purpose.
    3513                 :          */
    3514           37150 :         arena_avail_tree_remove(&arena->runs_avail,
    3515           37150 :             &chunk->map[arena_chunk_header_npages]);
    3516                 : 
    3517           37150 :         arena->spare = chunk;
    3518           37150 : }
    3519                 : 
    3520                 : static arena_run_t *
    3521         6621364 : arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large,
    3522                 :     bool zero)
    3523                 : {
    3524                 :         arena_run_t *run;
    3525                 :         arena_chunk_map_t *mapelm, key;
    3526                 : 
    3527         6621364 :         assert(size <= arena_maxclass);
    3528         6621364 :         assert((size & pagesize_mask) == 0);
    3529                 : 
    3530                 :         /* Search the arena's chunks for the lowest best fit. */
    3531         6621364 :         key.bits = size | CHUNK_MAP_KEY;
    3532         6621364 :         mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
    3533         6621364 :         if (mapelm != NULL) {
    3534         6554745 :                 arena_chunk_t *chunk =
    3535         6554745 :                     (arena_chunk_t*)CHUNK_ADDR2BASE(mapelm);
    3536        13109490 :                 size_t pageind = ((uintptr_t)mapelm -
    3537         6554745 :                     (uintptr_t)chunk->map) /
    3538                 :                     sizeof(arena_chunk_map_t);
    3539                 : 
    3540         6554745 :                 run = (arena_run_t *)((uintptr_t)chunk + (pageind
    3541         6554745 :                     << pagesize_2pow));
    3542         6554745 :                 arena_run_split(arena, run, size, large, zero);
    3543         6554745 :                 return (run);
    3544                 :         }
    3545                 : 
    3546           66619 :         if (arena->spare != NULL) {
    3547                 :                 /* Use the spare. */
    3548           31426 :                 arena_chunk_t *chunk = arena->spare;
    3549           31426 :                 arena->spare = NULL;
    3550           31426 :                 run = (arena_run_t *)((uintptr_t)chunk +
    3551           31426 :                     (arena_chunk_header_npages << pagesize_2pow));
    3552                 :                 /* Insert the run into the runs_avail tree. */
    3553           31426 :                 arena_avail_tree_insert(&arena->runs_avail,
    3554           31426 :                     &chunk->map[arena_chunk_header_npages]);
    3555           31426 :                 arena_run_split(arena, run, size, large, zero);
    3556           31426 :                 return (run);
    3557                 :         }
    3558                 : 
    3559                 :         /*
    3560                 :          * No usable runs.  Create a new chunk from which to allocate
    3561                 :          * the run.
    3562                 :          */
    3563                 :         {
    3564           35193 :                 arena_chunk_t *chunk = (arena_chunk_t *)
    3565           35193 :                     chunk_alloc(chunksize, true, true);
    3566           35193 :                 if (chunk == NULL)
    3567               0 :                         return (NULL);
    3568                 : 
    3569           35193 :                 arena_chunk_init(arena, chunk);
    3570           35193 :                 run = (arena_run_t *)((uintptr_t)chunk +
    3571           35193 :                     (arena_chunk_header_npages << pagesize_2pow));
    3572                 :         }
    3573                 :         /* Update page map. */
    3574           35193 :         arena_run_split(arena, run, size, large, zero);
    3575           35193 :         return (run);
    3576                 : }
    3577                 : 
    3578                 : static void
    3579            3981 : arena_purge(arena_t *arena)
    3580                 : {
    3581                 :         arena_chunk_t *chunk;
    3582                 :         size_t i, npages;
    3583                 : #ifdef MALLOC_DEBUG
    3584            3981 :         size_t ndirty = 0;
    3585           59413 :         rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty,
    3586                 :             chunk) {
    3587           51451 :                 ndirty += chunk->ndirty;
    3588           51451 :         } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk)
    3589            3981 :         assert(ndirty == arena->ndirty);
    3590                 : #endif
    3591            3981 :         assert(arena->ndirty > opt_dirty_max);
    3592                 : 
    3593                 : #ifdef MALLOC_STATS
    3594            3981 :         arena->stats.npurge++;
    3595                 : #endif
    3596                 : 
    3597                 :         /*
    3598                 :          * Iterate downward through chunks until enough dirty memory has been
    3599                 :          * purged.  Terminate as soon as possible in order to minimize the
    3600                 :          * number of system calls, even if a chunk has only been partially
    3601                 :          * purged.
    3602                 :          */
    3603           42742 :         while (arena->ndirty > (opt_dirty_max >> 1)) {
    3604                 : #ifdef MALLOC_DOUBLE_PURGE
    3605                 :                 bool madvised = false;
    3606                 : #endif
    3607           34780 :                 chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty);
    3608           34780 :                 assert(chunk != NULL);
    3609                 : 
    3610         4675669 :                 for (i = chunk_npages - 1; chunk->ndirty > 0; i--) {
    3611         4644870 :                         assert(i >= arena_chunk_header_npages);
    3612                 : 
    3613         4644870 :                         if (chunk->map[i].bits & CHUNK_MAP_DIRTY) {
    3614                 : #ifdef MALLOC_DECOMMIT
    3615                 :                                 const size_t free_operation = CHUNK_MAP_DECOMMITTED;
    3616                 : #else
    3617          162169 :                                 const size_t free_operation = CHUNK_MAP_MADVISED;
    3618                 : #endif
    3619          162169 :                                 assert((chunk->map[i].bits &
    3620                 :                                         CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
    3621          162169 :                                 chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
    3622                 :                                 /* Find adjacent dirty run(s). */
    3623         2366910 :                                 for (npages = 1;
    3624         4400528 :                                      i > arena_chunk_header_npages &&
    3625         2195787 :                                        (chunk->map[i - 1].bits & CHUNK_MAP_DIRTY);
    3626         2042572 :                                      npages++) {
    3627         2042572 :                                         i--;
    3628         2042572 :                                         assert((chunk->map[i].bits &
    3629                 :                                                 CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
    3630         2042572 :                                         chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
    3631                 :                                 }
    3632          162169 :                                 chunk->ndirty -= npages;
    3633          162169 :                                 arena->ndirty -= npages;
    3634                 : 
    3635                 : #ifdef MALLOC_DECOMMIT
    3636                 :                                 pages_decommit((void *)((uintptr_t)
    3637                 :                                     chunk + (i << pagesize_2pow)),
    3638                 :                                     (npages << pagesize_2pow));
    3639                 : #  ifdef MALLOC_STATS
    3640                 :                                 arena->stats.ndecommit++;
    3641                 :                                 arena->stats.decommitted += npages;
    3642                 : #  endif
    3643                 : #endif
    3644                 : #ifdef MALLOC_STATS
    3645          162169 :                                 arena->stats.committed -= npages;
    3646                 : #endif
    3647                 : 
    3648                 : #ifndef MALLOC_DECOMMIT
    3649          162169 :                                 madvise((void *)((uintptr_t)chunk + (i <<
    3650                 :                                     pagesize_2pow)), (npages << pagesize_2pow),
    3651                 :                                     MADV_FREE);
    3652                 : #  ifdef MALLOC_DOUBLE_PURGE
    3653                 :                                 madvised = true;
    3654                 : #  endif
    3655                 : #endif
    3656                 : #ifdef MALLOC_STATS
    3657          162169 :                                 arena->stats.nmadvise++;
    3658          162169 :                                 arena->stats.purged += npages;
    3659                 : #endif
    3660          162169 :                                 if (arena->ndirty <= (opt_dirty_max >> 1))
    3661            3981 :                                         break;
    3662                 :                         }
    3663                 :                 }
    3664                 : 
    3665           34780 :                 if (chunk->ndirty == 0) {
    3666           32535 :                         arena_chunk_tree_dirty_remove(&arena->chunks_dirty,
    3667                 :                             chunk);
    3668                 :                 }
    3669                 : #ifdef MALLOC_DOUBLE_PURGE
    3670                 :                 if (madvised) {
    3671                 :                         /* The chunk might already be in the list, but this
    3672                 :                          * makes sure it's at the front. */
    3673                 :                         LinkedList_Remove(&chunk->chunks_madvised_elem);
    3674                 :                         LinkedList_InsertHead(&arena->chunks_madvised, &chunk->chunks_madvised_elem);
    3675                 :                 }
    3676                 : #endif
    3677                 :         }
    3678            3981 : }
    3679                 : 
    3680                 : static void
    3681         6178032 : arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
    3682                 : {
    3683                 :         arena_chunk_t *chunk;
    3684                 :         size_t size, run_ind, run_pages;
    3685                 : 
    3686         6178032 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
    3687         6178032 :         run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk)
    3688                 :             >> pagesize_2pow);
    3689         6178032 :         assert(run_ind >= arena_chunk_header_npages);
    3690         6178032 :         assert(run_ind < chunk_npages);
    3691         6178032 :         if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0)
    3692         2276421 :                 size = chunk->map[run_ind].bits & ~pagesize_mask;
    3693                 :         else
    3694         3901611 :                 size = run->bin->run_size;
    3695         6178032 :         run_pages = (size >> pagesize_2pow);
    3696                 : 
    3697                 :         /* Mark pages as unallocated in the chunk map. */
    3698         6178032 :         if (dirty) {
    3699                 :                 size_t i;
    3700                 : 
    3701        34732483 :                 for (i = 0; i < run_pages; i++) {
    3702        28554451 :                         assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY)
    3703                 :                             == 0);
    3704        28554451 :                         chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY;
    3705                 :                 }
    3706                 : 
    3707         6178032 :                 if (chunk->ndirty == 0) {
    3708          556607 :                         arena_chunk_tree_dirty_insert(&arena->chunks_dirty,
    3709                 :                             chunk);
    3710                 :                 }
    3711         6178032 :                 chunk->ndirty += run_pages;
    3712         6178032 :                 arena->ndirty += run_pages;
    3713                 :         } else {
    3714                 :                 size_t i;
    3715                 : 
    3716               0 :                 for (i = 0; i < run_pages; i++) {
    3717               0 :                         chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE |
    3718                 :                             CHUNK_MAP_ALLOCATED);
    3719                 :                 }
    3720                 :         }
    3721         6178032 :         chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
    3722                 :             pagesize_mask);
    3723        12356064 :         chunk->map[run_ind+run_pages-1].bits = size |
    3724         6178032 :             (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
    3725                 : 
    3726                 :         /* Try to coalesce forward. */
    3727        12305487 :         if (run_ind + run_pages < chunk_npages &&
    3728         6127455 :             (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) {
    3729         3799839 :                 size_t nrun_size = chunk->map[run_ind+run_pages].bits &
    3730                 :                     ~pagesize_mask;
    3731                 : 
    3732                 :                 /*
    3733                 :                  * Remove successor from runs_avail; the coalesced run is
    3734                 :                  * inserted later.
    3735                 :                  */
    3736         3799839 :                 arena_avail_tree_remove(&arena->runs_avail,
    3737         3799839 :                     &chunk->map[run_ind+run_pages]);
    3738                 : 
    3739         3799839 :                 size += nrun_size;
    3740         3799839 :                 run_pages = size >> pagesize_2pow;
    3741                 : 
    3742         3799839 :                 assert((chunk->map[run_ind+run_pages-1].bits & ~pagesize_mask)
    3743                 :                     == nrun_size);
    3744         3799839 :                 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
    3745                 :                     pagesize_mask);
    3746         7599678 :                 chunk->map[run_ind+run_pages-1].bits = size |
    3747         3799839 :                     (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
    3748                 :         }
    3749                 : 
    3750                 :         /* Try to coalesce backward. */
    3751         6178032 :         if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits &
    3752                 :             CHUNK_MAP_ALLOCATED) == 0) {
    3753         1025273 :                 size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask;
    3754                 : 
    3755         1025273 :                 run_ind -= prun_size >> pagesize_2pow;
    3756                 : 
    3757                 :                 /*
    3758                 :                  * Remove predecessor from runs_avail; the coalesced run is
    3759                 :                  * inserted later.
    3760                 :                  */
    3761         1025273 :                 arena_avail_tree_remove(&arena->runs_avail,
    3762         1025273 :                     &chunk->map[run_ind]);
    3763                 : 
    3764         1025273 :                 size += prun_size;
    3765         1025273 :                 run_pages = size >> pagesize_2pow;
    3766                 : 
    3767         1025273 :                 assert((chunk->map[run_ind].bits & ~pagesize_mask) ==
    3768                 :                     prun_size);
    3769         1025273 :                 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
    3770                 :                     pagesize_mask);
    3771         2050546 :                 chunk->map[run_ind+run_pages-1].bits = size |
    3772         1025273 :                     (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
    3773                 :         }
    3774                 : 
    3775                 :         /* Insert into runs_avail, now that coalescing is complete. */
    3776         6178032 :         arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]);
    3777                 : 
    3778                 :         /* Deallocate chunk if it is now completely unused. */
    3779        12356064 :         if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask |
    3780         6178032 :             CHUNK_MAP_ALLOCATED)) == arena_maxclass)
    3781           37150 :                 arena_chunk_dealloc(arena, chunk);
    3782                 : 
    3783                 :         /* Enforce opt_dirty_max. */
    3784         6178032 :         if (arena->ndirty > opt_dirty_max)
    3785            3981 :                 arena_purge(arena);
    3786         6178032 : }
    3787                 : 
    3788                 : static void
    3789               0 : arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
    3790                 :     size_t oldsize, size_t newsize)
    3791                 : {
    3792               0 :         size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
    3793               0 :         size_t head_npages = (oldsize - newsize) >> pagesize_2pow;
    3794                 : 
    3795               0 :         assert(oldsize > newsize);
    3796                 : 
    3797                 :         /*
    3798                 :          * Update the chunk map so that arena_run_dalloc() can treat the
    3799                 :          * leading run as separately allocated.
    3800                 :          */
    3801               0 :         chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE |
    3802                 :             CHUNK_MAP_ALLOCATED;
    3803               0 :         chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE |
    3804                 :             CHUNK_MAP_ALLOCATED;
    3805                 : 
    3806               0 :         arena_run_dalloc(arena, run, false);
    3807               0 : }
    3808                 : 
    3809                 : static void
    3810             702 : arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
    3811                 :     size_t oldsize, size_t newsize, bool dirty)
    3812                 : {
    3813             702 :         size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
    3814             702 :         size_t npages = newsize >> pagesize_2pow;
    3815                 : 
    3816             702 :         assert(oldsize > newsize);
    3817                 : 
    3818                 :         /*
    3819                 :          * Update the chunk map so that arena_run_dalloc() can treat the
    3820                 :          * trailing run as separately allocated.
    3821                 :          */
    3822             702 :         chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE |
    3823                 :             CHUNK_MAP_ALLOCATED;
    3824            1404 :         chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE
    3825             702 :             | CHUNK_MAP_ALLOCATED;
    3826                 : 
    3827             702 :         arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
    3828                 :             dirty);
    3829             702 : }
    3830                 : 
    3831                 : static arena_run_t *
    3832         6129663 : arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
    3833                 : {
    3834                 :         arena_chunk_map_t *mapelm;
    3835                 :         arena_run_t *run;
    3836                 :         unsigned i, remainder;
    3837                 : 
    3838                 :         /* Look for a usable run. */
    3839         6129663 :         mapelm = arena_run_tree_first(&bin->runs);
    3840         6129663 :         if (mapelm != NULL) {
    3841                 :                 /* run is guaranteed to have available space. */
    3842         1790091 :                 arena_run_tree_remove(&bin->runs, mapelm);
    3843         1790091 :                 run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
    3844                 : #ifdef MALLOC_STATS
    3845         1790091 :                 bin->stats.reruns++;
    3846                 : #endif
    3847         1790091 :                 return (run);
    3848                 :         }
    3849                 :         /* No existing runs have any space available. */
    3850                 : 
    3851                 :         /* Allocate a new run. */
    3852         4339572 :         run = arena_run_alloc(arena, bin, bin->run_size, false, false);
    3853         4339572 :         if (run == NULL)
    3854               0 :                 return (NULL);
    3855                 :         /*
    3856                 :          * Don't initialize if a race in arena_run_alloc() allowed an existing
    3857                 :          * run to become usable.
    3858                 :          */
    3859         4339572 :         if (run == bin->runcur)
    3860               0 :                 return (run);
    3861                 : 
    3862                 :         VALGRIND_MALLOCLIKE_BLOCK(run, sizeof(arena_run_t) + (sizeof(unsigned) *
    3863                 :             (bin->regs_mask_nelms - 1)), 0, false);
    3864                 : 
    3865                 :         /* Initialize run internals. */
    3866         4339572 :         run->bin = bin;
    3867                 : 
    3868        10790691 :         for (i = 0; i < bin->regs_mask_nelms - 1; i++)
    3869         6451119 :                 run->regs_mask[i] = UINT_MAX;
    3870         4339572 :         remainder = bin->nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1);
    3871         4339572 :         if (remainder == 0)
    3872               0 :                 run->regs_mask[i] = UINT_MAX;
    3873                 :         else {
    3874                 :                 /* The last element has spare bits that need to be unset. */
    3875         4339572 :                 run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3))
    3876         4339572 :                     - remainder));
    3877                 :         }
    3878                 : 
    3879         4339572 :         run->regs_minelm = 0;
    3880                 : 
    3881         4339572 :         run->nfree = bin->nregs;
    3882                 : #ifdef MALLOC_DEBUG
    3883         4339572 :         run->magic = ARENA_RUN_MAGIC;
    3884                 : #endif
    3885                 : 
    3886                 : #ifdef MALLOC_STATS
    3887         4339572 :         bin->stats.nruns++;
    3888         4339572 :         bin->stats.curruns++;
    3889         4339572 :         if (bin->stats.curruns > bin->stats.highruns)
    3890         1127454 :                 bin->stats.highruns = bin->stats.curruns;
    3891                 : #endif
    3892         4339572 :         return (run);
    3893                 : }
    3894                 : 
    3895                 : /* bin->runcur must have space available before this function is called. */
    3896                 : static inline void *
    3897       123481815 : arena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run)
    3898                 : {
    3899                 :         void *ret;
    3900                 : 
    3901       123481815 :         assert(run->magic == ARENA_RUN_MAGIC);
    3902       123481815 :         assert(run->nfree > 0);
    3903                 : 
    3904       123481815 :         ret = arena_run_reg_alloc(run, bin);
    3905       123481815 :         assert(ret != NULL);
    3906       123481815 :         run->nfree--;
    3907                 : 
    3908       123481815 :         return (ret);
    3909                 : }
    3910                 : 
    3911                 : /* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */
    3912                 : static void *
    3913         6129663 : arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
    3914                 : {
    3915                 : 
    3916         6129663 :         bin->runcur = arena_bin_nonfull_run_get(arena, bin);
    3917         6129663 :         if (bin->runcur == NULL)
    3918               0 :                 return (NULL);
    3919         6129663 :         assert(bin->runcur->magic == ARENA_RUN_MAGIC);
    3920         6129663 :         assert(bin->runcur->nfree > 0);
    3921                 : 
    3922         6129663 :         return (arena_bin_malloc_easy(arena, bin, bin->runcur));
    3923                 : }
    3924                 : 
    3925                 : /*
    3926                 :  * Calculate bin->run_size such that it meets the following constraints:
    3927                 :  *
    3928                 :  *   *) bin->run_size >= min_run_size
    3929                 :  *   *) bin->run_size <= arena_maxclass
    3930                 :  *   *) bin->run_size <= RUN_MAX_SMALL
    3931                 :  *   *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
    3932                 :  *
    3933                 :  * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are
    3934                 :  * also calculated here, since these settings are all interdependent.
    3935                 :  */
    3936                 : static size_t
    3937          716436 : arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
    3938                 : {
    3939                 :         size_t try_run_size, good_run_size;
    3940                 :         unsigned good_nregs, good_mask_nelms, good_reg0_offset;
    3941                 :         unsigned try_nregs, try_mask_nelms, try_reg0_offset;
    3942                 : 
    3943          716436 :         assert(min_run_size >= pagesize);
    3944          716436 :         assert(min_run_size <= arena_maxclass);
    3945          716436 :         assert(min_run_size <= RUN_MAX_SMALL);
    3946                 : 
    3947                 :         /*
    3948                 :          * Calculate known-valid settings before entering the run_size
    3949                 :          * expansion loop, so that the first part of the loop always copies
    3950                 :          * valid settings.
    3951                 :          *
    3952                 :          * The do..while loop iteratively reduces the number of regions until
    3953                 :          * the run header and the regions no longer overlap.  A closed formula
    3954                 :          * would be quite messy, since there is an interdependency between the
    3955                 :          * header's mask length and the number of regions.
    3956                 :          */
    3957          716436 :         try_run_size = min_run_size;
    3958          716436 :         try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size)
    3959                 :             + 1; /* Counter-act try_nregs-- in loop. */
    3960                 :         do {
    3961         1472674 :                 try_nregs--;
    3962         2945348 :                 try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
    3963         1472674 :                     ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0);
    3964         1472674 :                 try_reg0_offset = try_run_size - (try_nregs * bin->reg_size);
    3965         1472674 :         } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))
    3966         1472674 :             > try_reg0_offset);
    3967                 : 
    3968                 :         /* run_size expansion loop. */
    3969                 :         do {
    3970                 :                 /*
    3971                 :                  * Copy valid settings before trying more aggressive settings.
    3972                 :                  */
    3973          855743 :                 good_run_size = try_run_size;
    3974          855743 :                 good_nregs = try_nregs;
    3975          855743 :                 good_mask_nelms = try_mask_nelms;
    3976          855743 :                 good_reg0_offset = try_reg0_offset;
    3977                 : 
    3978                 :                 /* Try more aggressive settings. */
    3979          855743 :                 try_run_size += pagesize;
    3980         1711486 :                 try_nregs = ((try_run_size - sizeof(arena_run_t)) /
    3981          855743 :                     bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */
    3982                 :                 do {
    3983         2467724 :                         try_nregs--;
    3984         4935448 :                         try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
    3985         2467724 :                             ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ?
    3986                 :                             1 : 0);
    3987         2467724 :                         try_reg0_offset = try_run_size - (try_nregs *
    3988         2467724 :                             bin->reg_size);
    3989         2467724 :                 } while (sizeof(arena_run_t) + (sizeof(unsigned) *
    3990         2467724 :                     (try_mask_nelms - 1)) > try_reg0_offset);
    3991         1711486 :         } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL
    3992          796040 :             && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
    3993         1611981 :             && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
    3994                 : 
    3995          716436 :         assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1))
    3996                 :             <= good_reg0_offset);
    3997          716436 :         assert((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs);
    3998                 : 
    3999                 :         /* Copy final settings. */
    4000          716436 :         bin->run_size = good_run_size;
    4001          716436 :         bin->nregs = good_nregs;
    4002          716436 :         bin->regs_mask_nelms = good_mask_nelms;
    4003          716436 :         bin->reg0_offset = good_reg0_offset;
    4004                 : 
    4005          716436 :         return (good_run_size);
    4006                 : }
    4007                 : 
    4008                 : #ifdef MALLOC_BALANCE
    4009                 : static inline void
    4010                 : arena_lock_balance(arena_t *arena)
    4011                 : {
    4012                 :         unsigned contention;
    4013                 : 
    4014                 :         contention = malloc_spin_lock(&arena->lock);
    4015                 :         if (narenas > 1) {
    4016                 :                 /*
    4017                 :                  * Calculate the exponentially averaged contention for this
    4018                 :                  * arena.  Due to integer math always rounding down, this value
    4019                 :                  * decays somewhat faster then normal.
    4020                 :                  */
    4021                 :                 arena->contention = (((uint64_t)arena->contention
    4022                 :                     * (uint64_t)((1U << BALANCE_ALPHA_INV_2POW)-1))
    4023                 :                     + (uint64_t)contention) >> BALANCE_ALPHA_INV_2POW;
    4024                 :                 if (arena->contention >= opt_balance_threshold)
    4025                 :                         arena_lock_balance_hard(arena);
    4026                 :         }
    4027                 : }
    4028                 : 
    4029                 : static void
    4030                 : arena_lock_balance_hard(arena_t *arena)
    4031                 : {
    4032                 :         uint32_t ind;
    4033                 : 
    4034                 :         arena->contention = 0;
    4035                 : #ifdef MALLOC_STATS
    4036                 :         arena->stats.nbalance++;
    4037                 : #endif
    4038                 :         ind = PRN(balance, narenas_2pow);
    4039                 :         if (arenas[ind] != NULL) {
    4040                 : #ifdef MOZ_MEMORY_WINDOWS
    4041                 :                 TlsSetValue(tlsIndex, arenas[ind]);
    4042                 : #else
    4043                 :                 arenas_map = arenas[ind];
    4044                 : #endif
    4045                 :         } else {
    4046                 :                 malloc_spin_lock(&arenas_lock);
    4047                 :                 if (arenas[ind] != NULL) {
    4048                 : #ifdef MOZ_MEMORY_WINDOWS
    4049                 :                         TlsSetValue(tlsIndex, arenas[ind]);
    4050                 : #else
    4051                 :                         arenas_map = arenas[ind];
    4052                 : #endif
    4053                 :                 } else {
    4054                 : #ifdef MOZ_MEMORY_WINDOWS
    4055                 :                         TlsSetValue(tlsIndex, arenas_extend(ind));
    4056                 : #else
    4057                 :                         arenas_map = arenas_extend(ind);
    4058                 : #endif
    4059                 :                 }
    4060                 :                 malloc_spin_unlock(&arenas_lock);
    4061                 :         }
    4062                 : }
    4063                 : #endif
    4064                 : 
    4065                 : static inline void *
    4066       123481101 : arena_malloc_small(arena_t *arena, size_t size, bool zero)
    4067                 : {
    4068                 :         void *ret;
    4069                 :         arena_bin_t *bin;
    4070                 :         arena_run_t *run;
    4071                 : 
    4072       123481101 :         if (size < small_min) {
    4073                 :                 /* Tiny. */
    4074        17536923 :                 size = pow2_ceil(size);
    4075        17536740 :                 bin = &arena->bins[ffs((int)(size >> (TINY_MIN_2POW +
    4076                 :                     1)))];
    4077                 : #if (!defined(NDEBUG) || defined(MALLOC_STATS))
    4078                 :                 /*
    4079                 :                  * Bin calculation is always correct, but we may need
    4080                 :                  * to fix size for the purposes of assertions and/or
    4081                 :                  * stats accuracy.
    4082                 :                  */
    4083        17536740 :                 if (size < (1U << TINY_MIN_2POW))
    4084         1194758 :                         size = (1U << TINY_MIN_2POW);
    4085                 : #endif
    4086       105944178 :         } else if (size <= small_max) {
    4087                 :                 /* Quantum-spaced. */
    4088        95606285 :                 size = QUANTUM_CEILING(size);
    4089        95606285 :                 bin = &arena->bins[ntbins + (size >> opt_quantum_2pow)
    4090                 :                     - 1];
    4091                 :         } else {
    4092                 :                 /* Sub-page. */
    4093        10337893 :                 size = pow2_ceil(size);
    4094        20675786 :                 bin = &arena->bins[ntbins + nqbins
    4095        10337893 :                     + (ffs((int)(size >> opt_small_max_2pow)) - 2)];
    4096                 :         }
    4097       123480918 :         assert(size == bin->reg_size);
    4098                 : 
    4099                 : #ifdef MALLOC_BALANCE
    4100                 :         arena_lock_balance(arena);
    4101                 : #else
    4102       123480918 :         malloc_spin_lock(&arena->lock);
    4103                 : #endif
    4104       123481815 :         if ((run = bin->runcur) != NULL && run->nfree > 0)
    4105       117352152 :                 ret = arena_bin_malloc_easy(arena, bin, run);
    4106                 :         else
    4107         6129663 :                 ret = arena_bin_malloc_hard(arena, bin);
    4108                 : 
    4109       123481815 :         if (ret == NULL) {
    4110               0 :                 malloc_spin_unlock(&arena->lock);
    4111               0 :                 return (NULL);
    4112                 :         }
    4113                 : 
    4114                 : #ifdef MALLOC_STATS
    4115       123481815 :         bin->stats.nrequests++;
    4116       123481815 :         arena->stats.nmalloc_small++;
    4117       123481815 :         arena->stats.allocated_small += size;
    4118                 : #endif
    4119       123481815 :         malloc_spin_unlock(&arena->lock);
    4120                 : 
    4121                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
    4122       123481435 :         if (zero == false) {
    4123                 : #ifdef MALLOC_FILL
    4124       116047151 :                 if (opt_junk)
    4125       116047153 :                         memset(ret, 0xa5, size);
    4126               0 :                 else if (opt_zero)
    4127               0 :                         memset(ret, 0, size);
    4128                 : #endif
    4129                 :         } else
    4130         7434284 :                 memset(ret, 0, size);
    4131                 : 
    4132       123481435 :         return (ret);
    4133                 : }
    4134                 : 
    4135                 : static void *
    4136         2281792 : arena_malloc_large(arena_t *arena, size_t size, bool zero)
    4137                 : {
    4138                 :         void *ret;
    4139                 : 
    4140                 :         /* Large allocation. */
    4141         2281792 :         size = PAGE_CEILING(size);
    4142                 : #ifdef MALLOC_BALANCE
    4143                 :         arena_lock_balance(arena);
    4144                 : #else
    4145         2281792 :         malloc_spin_lock(&arena->lock);
    4146                 : #endif
    4147         2281792 :         ret = (void *)arena_run_alloc(arena, NULL, size, true, zero);
    4148         2281792 :         if (ret == NULL) {
    4149               0 :                 malloc_spin_unlock(&arena->lock);
    4150               0 :                 return (NULL);
    4151                 :         }
    4152                 : #ifdef MALLOC_STATS
    4153         2281792 :         arena->stats.nmalloc_large++;
    4154         2281792 :         arena->stats.allocated_large += size;
    4155                 : #endif
    4156         2281792 :         malloc_spin_unlock(&arena->lock);
    4157                 : 
    4158                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
    4159         2281791 :         if (zero == false) {
    4160                 : #ifdef MALLOC_FILL
    4161         1729696 :                 if (opt_junk)
    4162         1729696 :                         memset(ret, 0xa5, size);
    4163               0 :                 else if (opt_zero)
    4164               0 :                         memset(ret, 0, size);
    4165                 : #endif
    4166                 :         }
    4167                 : 
    4168         2281791 :         return (ret);
    4169                 : }
    4170                 : 
    4171                 : static inline void *
    4172       125762809 : arena_malloc(arena_t *arena, size_t size, bool zero)
    4173                 : {
    4174                 : 
    4175       125762809 :         assert(arena != NULL);
    4176       125762809 :         assert(arena->magic == ARENA_MAGIC);
    4177       125762809 :         assert(size != 0);
    4178       125762809 :         assert(QUANTUM_CEILING(size) <= arena_maxclass);
    4179                 : 
    4180       125762809 :         if (size <= bin_maxclass) {
    4181       123481017 :                 return (arena_malloc_small(arena, size, zero));
    4182                 :         } else
    4183         2281792 :                 return (arena_malloc_large(arena, size, zero));
    4184                 : }
    4185                 : 
    4186                 : static inline void *
    4187       113343083 : imalloc(size_t size)
    4188                 : {
    4189                 : 
    4190       113343083 :         assert(size != 0);
    4191                 : 
    4192       113343083 :         if (size <= arena_maxclass)
    4193       113339406 :                 return (arena_malloc(choose_arena(), size, false));
    4194                 :         else
    4195            3677 :                 return (huge_malloc(size, false));
    4196                 : }
    4197                 : 
    4198                 : static inline void *
    4199         7986415 : icalloc(size_t size)
    4200                 : {
    4201                 : 
    4202         7986415 :         if (size <= arena_maxclass)
    4203         7986379 :                 return (arena_malloc(choose_arena(), size, true));
    4204                 :         else
    4205              36 :                 return (huge_malloc(size, true));
    4206                 : }
    4207                 : 
    4208                 : /* Only handles large allocations that require more than page alignment. */
    4209                 : static void *
    4210               0 : arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
    4211                 : {
    4212                 :         void *ret;
    4213                 :         size_t offset;
    4214                 :         arena_chunk_t *chunk;
    4215                 : 
    4216               0 :         assert((size & pagesize_mask) == 0);
    4217               0 :         assert((alignment & pagesize_mask) == 0);
    4218                 : 
    4219                 : #ifdef MALLOC_BALANCE
    4220                 :         arena_lock_balance(arena);
    4221                 : #else
    4222               0 :         malloc_spin_lock(&arena->lock);
    4223                 : #endif
    4224               0 :         ret = (void *)arena_run_alloc(arena, NULL, alloc_size, true, false);
    4225               0 :         if (ret == NULL) {
    4226               0 :                 malloc_spin_unlock(&arena->lock);
    4227               0 :                 return (NULL);
    4228                 :         }
    4229                 : 
    4230               0 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
    4231                 : 
    4232               0 :         offset = (uintptr_t)ret & (alignment - 1);
    4233               0 :         assert((offset & pagesize_mask) == 0);
    4234               0 :         assert(offset < alloc_size);
    4235               0 :         if (offset == 0)
    4236               0 :                 arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, alloc_size, size, false);
    4237                 :         else {
    4238                 :                 size_t leadsize, trailsize;
    4239                 : 
    4240               0 :                 leadsize = alignment - offset;
    4241               0 :                 if (leadsize > 0) {
    4242               0 :                         arena_run_trim_head(arena, chunk, (arena_run_t*)ret, alloc_size,
    4243                 :                             alloc_size - leadsize);
    4244               0 :                         ret = (void *)((uintptr_t)ret + leadsize);
    4245                 :                 }
    4246                 : 
    4247               0 :                 trailsize = alloc_size - leadsize - size;
    4248               0 :                 if (trailsize != 0) {
    4249                 :                         /* Trim trailing space. */
    4250               0 :                         assert(trailsize < alloc_size);
    4251               0 :                         arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, size + trailsize,
    4252                 :                             size, false);
    4253                 :                 }
    4254                 :         }
    4255                 : 
    4256                 : #ifdef MALLOC_STATS
    4257               0 :         arena->stats.nmalloc_large++;
    4258               0 :         arena->stats.allocated_large += size;
    4259                 : #endif
    4260               0 :         malloc_spin_unlock(&arena->lock);
    4261                 : 
    4262                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
    4263                 : #ifdef MALLOC_FILL
    4264               0 :         if (opt_junk)
    4265               0 :                 memset(ret, 0xa5, size);
    4266               0 :         else if (opt_zero)
    4267               0 :                 memset(ret, 0, size);
    4268                 : #endif
    4269               0 :         return (ret);
    4270                 : }
    4271                 : 
    4272                 : static inline void *
    4273           30860 : ipalloc(size_t alignment, size_t size)
    4274                 : {
    4275                 :         void *ret;
    4276                 :         size_t ceil_size;
    4277                 : 
    4278                 :         /*
    4279                 :          * Round size up to the nearest multiple of alignment.
    4280                 :          *
    4281                 :          * This done, we can take advantage of the fact that for each small
    4282                 :          * size class, every object is aligned at the smallest power of two
    4283                 :          * that is non-zero in the base two representation of the size.  For
    4284                 :          * example:
    4285                 :          *
    4286                 :          *   Size |   Base 2 | Minimum alignment
    4287                 :          *   -----+----------+------------------
    4288                 :          *     96 |  1100000 |  32
    4289                 :          *    144 | 10100000 |  32
    4290                 :          *    192 | 11000000 |  64
    4291                 :          *
    4292                 :          * Depending on runtime settings, it is possible that arena_malloc()
    4293                 :          * will further round up to a power of two, but that never causes
    4294                 :          * correctness issues.
    4295                 :          */
    4296           30860 :         ceil_size = (size + (alignment - 1)) & (-alignment);
    4297                 :         /*
    4298                 :          * (ceil_size < size) protects against the combination of maximal
    4299                 :          * alignment and size greater than maximal alignment.
    4300                 :          */
    4301           30860 :         if (ceil_size < size) {
    4302                 :                 /* size_t overflow. */
    4303               0 :                 return (NULL);
    4304                 :         }
    4305                 : 
    4306           30860 :         if (ceil_size <= pagesize || (alignment <= pagesize
    4307              12 :             && ceil_size <= arena_maxclass))
    4308           30860 :                 ret = arena_malloc(choose_arena(), ceil_size, false);
    4309                 :         else {
    4310                 :                 size_t run_size;
    4311                 : 
    4312                 :                 /*
    4313                 :                  * We can't achieve sub-page alignment, so round up alignment
    4314                 :                  * permanently; it makes later calculations simpler.
    4315                 :                  */
    4316               0 :                 alignment = PAGE_CEILING(alignment);
    4317               0 :                 ceil_size = PAGE_CEILING(size);
    4318                 :                 /*
    4319                 :                  * (ceil_size < size) protects against very large sizes within
    4320                 :                  * pagesize of SIZE_T_MAX.
    4321                 :                  *
    4322                 :                  * (ceil_size + alignment < ceil_size) protects against the
    4323                 :                  * combination of maximal alignment and ceil_size large enough
    4324                 :                  * to cause overflow.  This is similar to the first overflow
    4325                 :                  * check above, but it needs to be repeated due to the new
    4326                 :                  * ceil_size value, which may now be *equal* to maximal
    4327                 :                  * alignment, whereas before we only detected overflow if the
    4328                 :                  * original size was *greater* than maximal alignment.
    4329                 :                  */
    4330               0 :                 if (ceil_size < size || ceil_size + alignment < ceil_size) {
    4331                 :                         /* size_t overflow. */
    4332               0 :                         return (NULL);
    4333                 :                 }
    4334                 : 
    4335                 :                 /*
    4336                 :                  * Calculate the size of the over-size run that arena_palloc()
    4337                 :                  * would need to allocate in order to guarantee the alignment.
    4338                 :                  */
    4339               0 :                 if (ceil_size >= alignment)
    4340               0 :                         run_size = ceil_size + alignment - pagesize;
    4341                 :                 else {
    4342                 :                         /*
    4343                 :                          * It is possible that (alignment << 1) will cause
    4344                 :                          * overflow, but it doesn't matter because we also
    4345                 :                          * subtract pagesize, which in the case of overflow
    4346                 :                          * leaves us with a very large run_size.  That causes
    4347                 :                          * the first conditional below to fail, which means
    4348                 :                          * that the bogus run_size value never gets used for
    4349                 :                          * anything important.
    4350                 :                          */
    4351               0 :                         run_size = (alignment << 1) - pagesize;
    4352                 :                 }
    4353                 : 
    4354               0 :                 if (run_size <= arena_maxclass) {
    4355               0 :                         ret = arena_palloc(choose_arena(), alignment, ceil_size,
    4356                 :                             run_size);
    4357               0 :                 } else if (alignment <= chunksize)
    4358               0 :                         ret = huge_malloc(ceil_size, false);
    4359                 :                 else
    4360               0 :                         ret = huge_palloc(alignment, ceil_size);
    4361                 :         }
    4362                 : 
    4363           30860 :         assert(((uintptr_t)ret & (alignment - 1)) == 0);
    4364           30860 :         return (ret);
    4365                 : }
    4366                 : 
    4367                 : /* Return the size of the allocation pointed to by ptr. */
    4368                 : static size_t
    4369        18939099 : arena_salloc(const void *ptr)
    4370                 : {
    4371                 :         size_t ret;
    4372                 :         arena_chunk_t *chunk;
    4373                 :         size_t pageind, mapbits;
    4374                 : 
    4375        18939099 :         assert(ptr != NULL);
    4376        18939099 :         assert(CHUNK_ADDR2BASE(ptr) != ptr);
    4377                 : 
    4378        18939099 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
    4379        18939099 :         pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
    4380        18939099 :         mapbits = chunk->map[pageind].bits;
    4381        18939099 :         assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
    4382        18939099 :         if ((mapbits & CHUNK_MAP_LARGE) == 0) {
    4383        18304078 :                 arena_run_t *run = (arena_run_t *)(mapbits & ~pagesize_mask);
    4384        18304078 :                 assert(run->magic == ARENA_RUN_MAGIC);
    4385        18304078 :                 ret = run->bin->reg_size;
    4386                 :         } else {
    4387          635021 :                 ret = mapbits & ~pagesize_mask;
    4388          635021 :                 assert(ret != 0);
    4389                 :         }
    4390                 : 
    4391        18939099 :         return (ret);
    4392                 : }
    4393                 : 
    4394                 : #if (defined(MALLOC_VALIDATE) || defined(MOZ_MEMORY_DARWIN))
    4395                 : /*
    4396                 :  * Validate ptr before assuming that it points to an allocation.  Currently,
    4397                 :  * the following validation is performed:
    4398                 :  *
    4399                 :  * + Check that ptr is not NULL.
    4400                 :  *
    4401                 :  * + Check that ptr lies within a mapped chunk.
    4402                 :  */
    4403                 : static inline size_t
    4404        14218183 : isalloc_validate(const void *ptr)
    4405                 : {
    4406                 :         arena_chunk_t *chunk;
    4407                 : 
    4408        14218183 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
    4409        14218183 :         if (chunk == NULL)
    4410               0 :                 return (0);
    4411                 : 
    4412        14218183 :         if (malloc_rtree_get(chunk_rtree, (uintptr_t)chunk) == NULL)
    4413               0 :                 return (0);
    4414                 : 
    4415        14218192 :         if (chunk != ptr) {
    4416        14218023 :                 assert(chunk->arena->magic == ARENA_MAGIC);
    4417        14218023 :                 return (arena_salloc(ptr));
    4418                 :         } else {
    4419                 :                 size_t ret;
    4420                 :                 extent_node_t *node;
    4421                 :                 extent_node_t key;
    4422                 : 
    4423                 :                 /* Chunk. */
    4424             169 :                 key.addr = (void *)chunk;
    4425             169 :                 malloc_mutex_lock(&huge_mtx);
    4426             169 :                 node = extent_tree_ad_search(&huge, &key);
    4427             169 :                 if (node != NULL)
    4428             169 :                         ret = node->size;
    4429                 :                 else
    4430               0 :                         ret = 0;
    4431             169 :                 malloc_mutex_unlock(&huge_mtx);
    4432             169 :                 return (ret);
    4433                 :         }
    4434                 : }
    4435                 : #endif
    4436                 : 
    4437                 : static inline size_t
    4438         4723563 : isalloc(const void *ptr)
    4439                 : {
    4440                 :         size_t ret;
    4441                 :         arena_chunk_t *chunk;
    4442                 : 
    4443         4723563 :         assert(ptr != NULL);
    4444                 : 
    4445         4723563 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
    4446         4723563 :         if (chunk != ptr) {
    4447                 :                 /* Region. */
    4448         4721119 :                 assert(chunk->arena->magic == ARENA_MAGIC);
    4449                 : 
    4450         4721119 :                 ret = arena_salloc(ptr);
    4451                 :         } else {
    4452                 :                 extent_node_t *node, key;
    4453                 : 
    4454                 :                 /* Chunk (huge allocation). */
    4455                 : 
    4456            2444 :                 malloc_mutex_lock(&huge_mtx);
    4457                 : 
    4458                 :                 /* Extract from tree of huge allocations. */
    4459            2444 :                 key.addr = __DECONST(void *, ptr);
    4460            2444 :                 node = extent_tree_ad_search(&huge, &key);
    4461            2444 :                 assert(node != NULL);
    4462                 : 
    4463            2444 :                 ret = node->size;
    4464                 : 
    4465            2444 :                 malloc_mutex_unlock(&huge_mtx);
    4466                 :         }
    4467                 : 
    4468         4723563 :         return (ret);
    4469                 : }
    4470                 : 
    4471                 : static inline void
    4472       119028910 : arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
    4473                 :     arena_chunk_map_t *mapelm)
    4474                 : {
    4475                 :         arena_run_t *run;
    4476                 :         arena_bin_t *bin;
    4477                 :         size_t size;
    4478                 : 
    4479       119028910 :         run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
    4480       119028910 :         assert(run->magic == ARENA_RUN_MAGIC);
    4481       119028910 :         bin = run->bin;
    4482       119028910 :         size = bin->reg_size;
    4483                 : 
    4484                 : #ifdef MALLOC_FILL
    4485       119028910 :         if (opt_junk)
    4486       119028910 :                 memset(ptr, 0x5a, size);
    4487                 : #endif
    4488                 : 
    4489       119028910 :         arena_run_reg_dalloc(run, bin, ptr, size);
    4490       119028910 :         run->nfree++;
    4491                 : 
    4492       119028910 :         if (run->nfree == bin->nregs) {
    4493                 :                 /* Deallocate run. */
    4494         3901611 :                 if (run == bin->runcur)
    4495         3346264 :                         bin->runcur = NULL;
    4496          555347 :                 else if (bin->nregs != 1) {
    4497         1110694 :                         size_t run_pageind = (((uintptr_t)run -
    4498          555347 :                             (uintptr_t)chunk)) >> pagesize_2pow;
    4499          555347 :                         arena_chunk_map_t *run_mapelm =
    4500          555347 :                             &chunk->map[run_pageind];
    4501                 :                         /*
    4502                 :                          * This block's conditional is necessary because if the
    4503                 :                          * run only contains one region, then it never gets
    4504                 :                          * inserted into the non-full runs tree.
    4505                 :                          */
    4506          555347 :                         assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
    4507                 :                             run_mapelm);
    4508          555347 :                         arena_run_tree_remove(&bin->runs, run_mapelm);
    4509                 :                 }
    4510                 : #ifdef MALLOC_DEBUG
    4511         3901611 :                 run->magic = 0;
    4512                 : #endif
    4513                 :                 VALGRIND_FREELIKE_BLOCK(run, 0);
    4514         3901611 :                 arena_run_dalloc(arena, run, true);
    4515                 : #ifdef MALLOC_STATS
    4516         3901611 :                 bin->stats.curruns--;
    4517                 : #endif
    4518       115127299 :         } else if (run->nfree == 1 && run != bin->runcur) {
    4519                 :                 /*
    4520                 :                  * Make sure that bin->runcur always refers to the lowest
    4521                 :                  * non-full run, if one exists.
    4522                 :                  */
    4523         2555071 :                 if (bin->runcur == NULL)
    4524           67990 :                         bin->runcur = run;
    4525         2487081 :                 else if ((uintptr_t)run < (uintptr_t)bin->runcur) {
    4526                 :                         /* Switch runcur. */
    4527         1678159 :                         if (bin->runcur->nfree > 0) {
    4528         1623386 :                                 arena_chunk_t *runcur_chunk =
    4529         1623386 :                                     (arena_chunk_t*)CHUNK_ADDR2BASE(bin->runcur);
    4530         1623386 :                                 size_t runcur_pageind =
    4531         3246772 :                                     (((uintptr_t)bin->runcur -
    4532         1623386 :                                     (uintptr_t)runcur_chunk)) >> pagesize_2pow;
    4533         1623386 :                                 arena_chunk_map_t *runcur_mapelm =
    4534         1623386 :                                     &runcur_chunk->map[runcur_pageind];
    4535                 : 
    4536                 :                                 /* Insert runcur. */
    4537         1623386 :                                 assert(arena_run_tree_search(&bin->runs,
    4538                 :                                     runcur_mapelm) == NULL);
    4539         1623386 :                                 arena_run_tree_insert(&bin->runs,
    4540                 :                                     runcur_mapelm);
    4541                 :                         }
    4542         1678159 :                         bin->runcur = run;
    4543                 :                 } else {
    4544         1617844 :                         size_t run_pageind = (((uintptr_t)run -
    4545          808922 :                             (uintptr_t)chunk)) >> pagesize_2pow;
    4546          808922 :                         arena_chunk_map_t *run_mapelm =
    4547          808922 :                             &chunk->map[run_pageind];
    4548                 : 
    4549          808922 :                         assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
    4550                 :                             NULL);
    4551          808922 :                         arena_run_tree_insert(&bin->runs, run_mapelm);
    4552                 :                 }
    4553                 :         }
    4554                 : #ifdef MALLOC_STATS
    4555       119028910 :         arena->stats.allocated_small -= size;
    4556       119028910 :         arena->stats.ndalloc_small++;
    4557                 : #endif
    4558       119028910 : }
    4559                 : 
    4560                 : static void
    4561         2275716 : arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
    4562                 : {
    4563                 :         /* Large allocation. */
    4564         2275716 :         malloc_spin_lock(&arena->lock);
    4565                 : 
    4566                 : #ifdef MALLOC_FILL
    4567                 : #ifndef MALLOC_STATS
    4568                 :         if (opt_junk)
    4569                 : #endif
    4570                 : #endif
    4571                 :         {
    4572         2275719 :                 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
    4573                 :                     pagesize_2pow;
    4574         2275719 :                 size_t size = chunk->map[pageind].bits & ~pagesize_mask;
    4575                 : 
    4576                 : #ifdef MALLOC_FILL
    4577                 : #ifdef MALLOC_STATS
    4578         2275719 :                 if (opt_junk)
    4579                 : #endif
    4580         2275719 :                         memset(ptr, 0x5a, size);
    4581                 : #endif
    4582                 : #ifdef MALLOC_STATS
    4583         2275719 :                 arena->stats.allocated_large -= size;
    4584                 : #endif
    4585                 :         }
    4586                 : #ifdef MALLOC_STATS
    4587         2275719 :         arena->stats.ndalloc_large++;
    4588                 : #endif
    4589                 : 
    4590         2275719 :         arena_run_dalloc(arena, (arena_run_t *)ptr, true);
    4591         2275719 :         malloc_spin_unlock(&arena->lock);
    4592         2275719 : }
    4593                 : 
    4594                 : static inline void
    4595       121279191 : arena_dalloc(void *ptr, size_t offset)
    4596                 : {
    4597                 :         arena_chunk_t *chunk;
    4598                 :         arena_t *arena;
    4599                 :         size_t pageind;
    4600                 :         arena_chunk_map_t *mapelm;
    4601                 : 
    4602       121279191 :         assert(ptr != NULL);
    4603       121279191 :         assert(offset != 0);
    4604       121279191 :         assert(CHUNK_ADDR2OFFSET(ptr) == offset);
    4605                 : 
    4606       121279191 :         chunk = (arena_chunk_t *) ((uintptr_t)ptr - offset);
    4607       121279191 :         arena = chunk->arena;
    4608       121279191 :         assert(arena != NULL);
    4609       121279191 :         assert(arena->magic == ARENA_MAGIC);
    4610                 : 
    4611       121279191 :         pageind = offset >> pagesize_2pow;
    4612       121279191 :         mapelm = &chunk->map[pageind];
    4613       121279191 :         assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
    4614       121279191 :         if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
    4615                 :                 /* Small allocation. */
    4616       119003475 :                 malloc_spin_lock(&arena->lock);
    4617       119028910 :                 arena_dalloc_small(arena, chunk, ptr, mapelm);
    4618       119028910 :                 malloc_spin_unlock(&arena->lock);
    4619                 :         } else
    4620         2275716 :                 arena_dalloc_large(arena, chunk, ptr);
    4621                 :         VALGRIND_FREELIKE_BLOCK(ptr, 0);
    4622       121303509 : }
    4623                 : 
    4624                 : static inline void
    4625         4408108 : idalloc(void *ptr)
    4626                 : {
    4627                 :         size_t offset;
    4628                 : 
    4629         4408108 :         assert(ptr != NULL);
    4630                 : 
    4631         4408108 :         offset = CHUNK_ADDR2OFFSET(ptr);
    4632         4408108 :         if (offset != 0)
    4633         4406384 :                 arena_dalloc(ptr, offset);
    4634                 :         else
    4635            1724 :                 huge_dalloc(ptr);
    4636         4408108 : }
    4637                 : 
    4638                 : static void
    4639             702 : arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
    4640                 :     size_t size, size_t oldsize)
    4641                 : {
    4642                 : 
    4643             702 :         assert(size < oldsize);
    4644                 : 
    4645                 :         /*
    4646                 :          * Shrink the run, and make trailing pages available for other
    4647                 :          * allocations.
    4648                 :          */
    4649                 : #ifdef MALLOC_BALANCE
    4650                 :         arena_lock_balance(arena);
    4651                 : #else
    4652             702 :         malloc_spin_lock(&arena->lock);
    4653                 : #endif
    4654             702 :         arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size,
    4655                 :             true);
    4656                 : #ifdef MALLOC_STATS
    4657             702 :         arena->stats.allocated_large -= oldsize - size;
    4658                 : #endif
    4659             702 :         malloc_spin_unlock(&arena->lock);
    4660             702 : }
    4661                 : 
    4662                 : static bool
    4663          181846 : arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
    4664                 :     size_t size, size_t oldsize)
    4665                 : {
    4666          181846 :         size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow;
    4667          181846 :         size_t npages = oldsize >> pagesize_2pow;
    4668                 : 
    4669          181846 :         assert(oldsize == (chunk->map[pageind].bits & ~pagesize_mask));
    4670                 : 
    4671                 :         /* Try to extend the run. */
    4672          181846 :         assert(size > oldsize);
    4673                 : #ifdef MALLOC_BALANCE
    4674                 :         arena_lock_balance(arena);
    4675                 : #else
    4676          181846 :         malloc_spin_lock(&arena->lock);
    4677                 : #endif
    4678          357265 :         if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits
    4679          420097 :             & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits &
    4680          122339 :             ~pagesize_mask) >= size - oldsize) {
    4681                 :                 /*
    4682                 :                  * The next run is available and sufficiently large.  Split the
    4683                 :                  * following run, then merge the first part with the existing
    4684                 :                  * allocation.
    4685                 :                  */
    4686          233348 :                 arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
    4687          116674 :                     ((pageind+npages) << pagesize_2pow)), size - oldsize, true,
    4688                 :                     false);
    4689                 : 
    4690          116674 :                 chunk->map[pageind].bits = size | CHUNK_MAP_LARGE |
    4691                 :                     CHUNK_MAP_ALLOCATED;
    4692          116674 :                 chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE |
    4693                 :                     CHUNK_MAP_ALLOCATED;
    4694                 : 
    4695                 : #ifdef MALLOC_STATS
    4696          116674 :                 arena->stats.allocated_large += size - oldsize;
    4697                 : #endif
    4698          116674 :                 malloc_spin_unlock(&arena->lock);
    4699          116674 :                 return (false);
    4700                 :         }
    4701           65172 :         malloc_spin_unlock(&arena->lock);
    4702                 : 
    4703           65172 :         return (true);
    4704                 : }
    4705                 : 
    4706                 : /*
    4707                 :  * Try to resize a large allocation, in order to avoid copying.  This will
    4708                 :  * always fail if growing an object, and the following run is already in use.
    4709                 :  */
    4710                 : static bool
    4711          191562 : arena_ralloc_large(void *ptr, size_t size, size_t oldsize)
    4712                 : {
    4713                 :         size_t psize;
    4714                 : 
    4715          191562 :         psize = PAGE_CEILING(size);
    4716          191562 :         if (psize == oldsize) {
    4717                 :                 /* Same size class. */
    4718                 : #ifdef MALLOC_FILL
    4719            9014 :                 if (opt_junk && size < oldsize) {
    4720            9010 :                         memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize -
    4721                 :                             size);
    4722                 :                 }
    4723                 : #endif
    4724            9014 :                 return (false);
    4725                 :         } else {
    4726                 :                 arena_chunk_t *chunk;
    4727                 :                 arena_t *arena;
    4728                 : 
    4729          182548 :                 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
    4730          182548 :                 arena = chunk->arena;
    4731          182548 :                 assert(arena->magic == ARENA_MAGIC);
    4732                 : 
    4733          182548 :                 if (psize < oldsize) {
    4734                 : #ifdef MALLOC_FILL
    4735                 :                         /* Fill before shrinking in order avoid a race. */
    4736             702 :                         if (opt_junk) {
    4737             702 :                                 memset((void *)((uintptr_t)ptr + size), 0x5a,
    4738                 :                                     oldsize - size);
    4739                 :                         }
    4740                 : #endif
    4741             702 :                         arena_ralloc_large_shrink(arena, chunk, ptr, psize,
    4742                 :                             oldsize);
    4743             702 :                         return (false);
    4744                 :                 } else {
    4745          181846 :                         bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
    4746                 :                             psize, oldsize);
    4747                 : #ifdef MALLOC_FILL
    4748          181846 :                         if (ret == false && opt_zero) {
    4749               0 :                                 memset((void *)((uintptr_t)ptr + oldsize), 0,
    4750                 :                                     size - oldsize);
    4751                 :                         }
    4752                 : #endif
    4753          181846 :                         return (ret);
    4754                 :                 }
    4755                 :         }
    4756                 : }
    4757                 : 
    4758                 : static void *
    4759         4721020 : arena_ralloc(void *ptr, size_t size, size_t oldsize)
    4760                 : {
    4761                 :         void *ret;
    4762                 :         size_t copysize;
    4763                 : 
    4764                 :         /* Try to avoid moving the allocation. */
    4765         4721020 :         if (size < small_min) {
    4766          391966 :                 if (oldsize < small_min &&
    4767          111301 :                     ffs((int)(pow2_ceil(size) >> (TINY_MIN_2POW + 1)))
    4768          111301 :                     == ffs((int)(pow2_ceil(oldsize) >> (TINY_MIN_2POW + 1))))
    4769           24671 :                         goto IN_PLACE; /* Same size class. */
    4770         4440355 :         } else if (size <= small_max) {
    4771         6989528 :                 if (oldsize >= small_min && oldsize <= small_max &&
    4772         3462597 :                     (QUANTUM_CEILING(size) >> opt_quantum_2pow)
    4773         3462597 :                     == (QUANTUM_CEILING(oldsize) >> opt_quantum_2pow))
    4774           99224 :                         goto IN_PLACE; /* Same size class. */
    4775          913424 :         } else if (size <= bin_maxclass) {
    4776          912451 :                 if (oldsize > small_max && oldsize <= bin_maxclass &&
    4777          296660 :                     pow2_ceil(size) == pow2_ceil(oldsize))
    4778           64452 :                         goto IN_PLACE; /* Same size class. */
    4779          297633 :         } else if (oldsize > bin_maxclass && oldsize <= arena_maxclass) {
    4780          191562 :                 assert(size > bin_maxclass);
    4781          191562 :                 if (arena_ralloc_large(ptr, size, oldsize) == false)
    4782          126390 :                         return (ptr);
    4783                 :         }
    4784                 : 
    4785                 :         /*
    4786                 :          * If we get here, then size and oldsize are different enough that we
    4787                 :          * need to move the object.  In that case, fall back to allocating new
    4788                 :          * space and copying.
    4789                 :          */
    4790         4406283 :         ret = arena_malloc(choose_arena(), size, false);
    4791         4406282 :         if (ret == NULL)
    4792               0 :                 return (NULL);
    4793                 : 
    4794                 :         /* Junk/zero-filling were already done by arena_malloc(). */
    4795         4406282 :         copysize = (size < oldsize) ? size : oldsize;
    4796                 : #ifdef VM_COPY_MIN
    4797                 :         if (copysize >= VM_COPY_MIN)
    4798                 :                 pages_copy(ret, ptr, copysize);
    4799                 :         else
    4800                 : #endif
    4801         4406282 :                 memcpy(ret, ptr, copysize);
    4802         4406282 :         idalloc(ptr);
    4803         4406283 :         return (ret);
    4804                 : IN_PLACE:
    4805                 : #ifdef MALLOC_FILL
    4806          188347 :         if (opt_junk && size < oldsize)
    4807           98059 :                 memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size);
    4808           90288 :         else if (opt_zero && size > oldsize)
    4809               0 :                 memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
    4810                 : #endif
    4811          188347 :         return (ptr);
    4812                 : }
    4813                 : 
    4814                 : static inline void *
    4815         4723564 : iralloc(void *ptr, size_t size)
    4816                 : {
    4817                 :         size_t oldsize;
    4818                 : 
    4819         4723564 :         assert(ptr != NULL);
    4820         4723564 :         assert(size != 0);
    4821                 : 
    4822         4723564 :         oldsize = isalloc(ptr);
    4823                 : 
    4824                 : #ifndef MALLOC_VALGRIND
    4825         4723563 :         if (size <= arena_maxclass)
    4826         4721018 :                 return (arena_ralloc(ptr, size, oldsize));
    4827                 :         else
    4828            2545 :                 return (huge_ralloc(ptr, size, oldsize));
    4829                 : #else
    4830                 :         /*
    4831                 :          * Valgrind does not provide a public interface for modifying an
    4832                 :          * existing allocation, so use malloc/memcpy/free instead.
    4833                 :          */
    4834                 :         {
    4835                 :                 void *ret = imalloc(size);
    4836                 :                 if (ret != NULL) {
    4837                 :                         if (oldsize < size)
    4838                 :                             memcpy(ret, ptr, oldsize);
    4839                 :                         else
    4840                 :                             memcpy(ret, ptr, size);
    4841                 :                         idalloc(ptr);
    4842                 :                 }
    4843                 :                 return (ret);
    4844                 :         }
    4845                 : #endif
    4846                 : }
    4847                 : 
    4848                 : static bool
    4849           19901 : arena_new(arena_t *arena)
    4850                 : {
    4851                 :         unsigned i;
    4852                 :         arena_bin_t *bin;
    4853                 :         size_t pow2_size, prev_run_size;
    4854                 : 
    4855           19901 :         if (malloc_spin_init(&arena->lock))
    4856               0 :                 return (true);
    4857                 : 
    4858                 : #ifdef MALLOC_STATS
    4859           19901 :         memset(&arena->stats, 0, sizeof(arena_stats_t));
    4860                 : #endif
    4861                 : 
    4862                 :         /* Initialize chunks. */
    4863           19901 :         arena_chunk_tree_dirty_new(&arena->chunks_dirty);
    4864                 : #ifdef MALLOC_DOUBLE_PURGE
    4865                 :         LinkedList_Init(&arena->chunks_madvised);
    4866                 : #endif
    4867           19901 :         arena->spare = NULL;
    4868                 : 
    4869           19901 :         arena->ndirty = 0;
    4870                 : 
    4871           19901 :         arena_avail_tree_new(&arena->runs_avail);
    4872                 : 
    4873                 : #ifdef MALLOC_BALANCE
    4874                 :         arena->contention = 0;
    4875                 : #endif
    4876                 : 
    4877                 :         /* Initialize bins. */
    4878           19901 :         prev_run_size = pagesize;
    4879                 : 
    4880                 :         /* (2^n)-spaced tiny bins. */
    4881           59703 :         for (i = 0; i < ntbins; i++) {
    4882           39802 :                 bin = &arena->bins[i];
    4883           39802 :                 bin->runcur = NULL;
    4884           39802 :                 arena_run_tree_new(&bin->runs);
    4885                 : 
    4886           39802 :                 bin->reg_size = (1U << (TINY_MIN_2POW + i));
    4887                 : 
    4888           39802 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
    4889                 : 
    4890                 : #ifdef MALLOC_STATS
    4891           39802 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
    4892                 : #endif
    4893                 :         }
    4894                 : 
    4895                 :         /* Quantum-spaced bins. */
    4896          656733 :         for (; i < ntbins + nqbins; i++) {
    4897          636832 :                 bin = &arena->bins[i];
    4898          636832 :                 bin->runcur = NULL;
    4899          636832 :                 arena_run_tree_new(&bin->runs);
    4900                 : 
    4901          636832 :                 bin->reg_size = quantum * (i - ntbins + 1);
    4902                 : 
    4903          636832 :                 pow2_size = pow2_ceil(quantum * (i - ntbins + 1));
    4904          636832 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
    4905                 : 
    4906                 : #ifdef MALLOC_STATS
    4907          636832 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
    4908                 : #endif
    4909                 :         }
    4910                 : 
    4911                 :         /* (2^n)-spaced sub-page bins. */
    4912           59703 :         for (; i < ntbins + nqbins + nsbins; i++) {
    4913           39802 :                 bin = &arena->bins[i];
    4914           39802 :                 bin->runcur = NULL;
    4915           39802 :                 arena_run_tree_new(&bin->runs);
    4916                 : 
    4917           39802 :                 bin->reg_size = (small_max << (i - (ntbins + nqbins) + 1));
    4918                 : 
    4919           39802 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
    4920                 : 
    4921                 : #ifdef MALLOC_STATS
    4922           39802 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
    4923                 : #endif
    4924                 :         }
    4925                 : 
    4926                 : #ifdef MALLOC_DEBUG
    4927           19901 :         arena->magic = ARENA_MAGIC;
    4928                 : #endif
    4929                 : 
    4930           19901 :         return (false);
    4931                 : }
    4932                 : 
    4933                 : /* Create a new arena and insert it into the arenas array at index ind. */
    4934                 : static arena_t *
    4935           19901 : arenas_extend(unsigned ind)
    4936                 : {
    4937                 :         arena_t *ret;
    4938                 : 
    4939                 :         /* Allocate enough space for trailing bins. */
    4940           19901 :         ret = (arena_t *)base_alloc(sizeof(arena_t)
    4941           19901 :             + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
    4942           19901 :         if (ret != NULL && arena_new(ret) == false) {
    4943           19901 :                 arenas[ind] = ret;
    4944           19901 :                 return (ret);
    4945                 :         }
    4946                 :         /* Only reached if there is an OOM error. */
    4947                 : 
    4948                 :         /*
    4949                 :          * OOM here is quite inconvenient to propagate, since dealing with it
    4950                 :          * would require a check for failure in the fast path.  Instead, punt
    4951                 :          * by using arenas[0].  In practice, this is an extremely unlikely
    4952                 :          * failure.
    4953                 :          */
    4954               0 :         _malloc_message(_getprogname(),
    4955                 :             ": (malloc) Error initializing arena\n", "", "");
    4956               0 :         if (opt_abort)
    4957               0 :                 abort();
    4958                 : 
    4959               0 :         return (arenas[0]);
    4960                 : }
    4961                 : 
    4962                 : /*
    4963                 :  * End arena.
    4964                 :  */
    4965                 : /******************************************************************************/
    4966                 : /*
    4967                 :  * Begin general internal functions.
    4968                 :  */
    4969                 : 
    4970                 : static void *
    4971            5538 : huge_malloc(size_t size, bool zero)
    4972                 : {
    4973                 :         void *ret;
    4974                 :         size_t csize;
    4975                 :         size_t psize;
    4976                 :         extent_node_t *node;
    4977                 : 
    4978                 :         /* Allocate one or more contiguous chunks for this request. */
    4979                 : 
    4980            5538 :         csize = CHUNK_CEILING(size);
    4981            5538 :         if (csize == 0) {
    4982                 :                 /* size is large enough to cause size_t wrap-around. */
    4983               0 :                 return (NULL);
    4984                 :         }
    4985                 : 
    4986                 :         /* Allocate an extent node with which to track the chunk. */
    4987            5538 :         node = base_node_alloc();
    4988            5538 :         if (node == NULL)
    4989               0 :                 return (NULL);
    4990                 : 
    4991            5538 :         ret = chunk_alloc(csize, zero, true);
    4992            5538 :         if (ret == NULL) {
    4993              19 :                 base_node_dealloc(node);
    4994              19 :                 return (NULL);
    4995                 :         }
    4996                 : 
    4997                 :         /* Insert node into huge. */
    4998            5519 :         node->addr = ret;
    4999            5519 :         psize = PAGE_CEILING(size);
    5000            5519 :         node->size = psize;
    5001                 : 
    5002            5519 :         malloc_mutex_lock(&huge_mtx);
    5003            5519 :         extent_tree_ad_insert(&huge, node);
    5004                 : #ifdef MALLOC_STATS
    5005            5519 :         huge_nmalloc++;
    5006                 : 
    5007                 :         /* Although we allocated space for csize bytes, we indicate that we've
    5008                 :          * allocated only psize bytes.
    5009                 :          *
    5010                 :          * If DECOMMIT is defined, this is a reasonable thing to do, since
    5011                 :          * we'll explicitly decommit the bytes in excess of psize.
    5012                 :          *
    5013                 :          * If DECOMMIT is not defined, then we're relying on the OS to be lazy
    5014                 :          * about how it allocates physical pages to mappings.  If we never
    5015                 :          * touch the pages in excess of psize, the OS won't allocate a physical
    5016                 :          * page, and we won't use more than psize bytes of physical memory.
    5017                 :          *
    5018                 :          * A correct program will only touch memory in excess of how much it
    5019                 :          * requested if it first calls malloc_usable_size and finds out how
    5020                 :          * much space it has to play with.  But because we set node->size =
    5021                 :          * psize above, malloc_usable_size will return psize, not csize, and
    5022                 :          * the program will (hopefully) never touch bytes in excess of psize.
    5023                 :          * Thus those bytes won't take up space in physical memory, and we can
    5024                 :          * reasonably claim we never "allocated" them in the first place. */
    5025            5519 :         huge_allocated += psize;
    5026                 : #endif
    5027            5519 :         malloc_mutex_unlock(&huge_mtx);
    5028                 : 
    5029                 : #ifdef MALLOC_DECOMMIT
    5030                 :         if (csize - psize > 0)
    5031                 :                 pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
    5032                 : #endif
    5033                 : 
    5034                 : #ifdef MALLOC_DECOMMIT
    5035                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, zero);
    5036                 : #else
    5037                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, csize, 0, zero);
    5038                 : #endif
    5039                 : 
    5040                 : #ifdef MALLOC_FILL
    5041            5519 :         if (zero == false) {
    5042            5501 :                 if (opt_junk)
    5043                 : #  ifdef MALLOC_DECOMMIT
    5044                 :                         memset(ret, 0xa5, psize);
    5045                 : #  else
    5046            5501 :                         memset(ret, 0xa5, csize);
    5047                 : #  endif
    5048               0 :                 else if (opt_zero)
    5049                 : #  ifdef MALLOC_DECOMMIT
    5050                 :                         memset(ret, 0, psize);
    5051                 : #  else
    5052               0 :                         memset(ret, 0, csize);
    5053                 : #  endif
    5054                 :         }
    5055                 : #endif
    5056                 : 
    5057            5519 :         return (ret);
    5058                 : }
    5059                 : 
    5060                 : /* Only handles large allocations that require more than chunk alignment. */
    5061                 : static void *
    5062               0 : huge_palloc(size_t alignment, size_t size)
    5063                 : {
    5064                 :         void *ret;
    5065                 :         size_t alloc_size, chunk_size, offset;
    5066                 :         size_t psize;
    5067                 :         extent_node_t *node;
    5068                 :         int pfd;
    5069                 : 
    5070                 :         /*
    5071                 :          * This allocation requires alignment that is even larger than chunk
    5072                 :          * alignment.  This means that huge_malloc() isn't good enough.
    5073                 :          *
    5074                 :          * Allocate almost twice as many chunks as are demanded by the size or
    5075                 :          * alignment, in order to assure the alignment can be achieved, then
    5076                 :          * unmap leading and trailing chunks.
    5077                 :          */
    5078               0 :         assert(alignment >= chunksize);
    5079                 : 
    5080               0 :         chunk_size = CHUNK_CEILING(size);
    5081                 : 
    5082               0 :         if (size >= alignment)
    5083               0 :                 alloc_size = chunk_size + alignment - chunksize;
    5084                 :         else
    5085               0 :                 alloc_size = (alignment << 1) - chunksize;
    5086                 : 
    5087                 :         /* Allocate an extent node with which to track the chunk. */
    5088               0 :         node = base_node_alloc();
    5089               0 :         if (node == NULL)
    5090               0 :                 return (NULL);
    5091                 : 
    5092                 :         /*
    5093                 :          * Windows requires that there be a 1:1 mapping between VM
    5094                 :          * allocation/deallocation operations.  Therefore, take care here to
    5095                 :          * acquire the final result via one mapping operation.
    5096                 :          *
    5097                 :          * The MALLOC_PAGEFILE code also benefits from this mapping algorithm,
    5098                 :          * since it reduces the number of page files.
    5099                 :          */
    5100                 : #ifdef MALLOC_PAGEFILE
    5101                 :         if (opt_pagefile) {
    5102                 :                 pfd = pagefile_init(size);
    5103                 :                 if (pfd == -1)
    5104                 :                         return (NULL);
    5105                 :         } else
    5106                 : #endif
    5107               0 :                 pfd = -1;
    5108                 : #ifdef JEMALLOC_USES_MAP_ALIGN
    5109                 :                 ret = pages_map_align(chunk_size, pfd, alignment);
    5110                 : #else
    5111                 :         do {
    5112                 :                 void *over;
    5113                 : 
    5114               0 :                 over = chunk_alloc(alloc_size, false, false);
    5115               0 :                 if (over == NULL) {
    5116               0 :                         base_node_dealloc(node);
    5117               0 :                         ret = NULL;
    5118               0 :                         goto RETURN;
    5119                 :                 }
    5120                 : 
    5121               0 :                 offset = (uintptr_t)over & (alignment - 1);
    5122               0 :                 assert((offset & chunksize_mask) == 0);
    5123               0 :                 assert(offset < alloc_size);
    5124               0 :                 ret = (void *)((uintptr_t)over + offset);
    5125               0 :                 chunk_dealloc(over, alloc_size);
    5126               0 :                 ret = pages_map(ret, chunk_size, pfd);
    5127                 :                 /*
    5128                 :                  * Failure here indicates a race with another thread, so try
    5129                 :                  * again.
    5130                 :                  */
    5131               0 :         } while (ret == NULL);
    5132                 : #endif
    5133                 :         /* Insert node into huge. */
    5134               0 :         node->addr = ret;
    5135               0 :         psize = PAGE_CEILING(size);
    5136               0 :         node->size = psize;
    5137                 : 
    5138               0 :         malloc_mutex_lock(&huge_mtx);
    5139               0 :         extent_tree_ad_insert(&huge, node);
    5140                 : #ifdef MALLOC_STATS
    5141               0 :         huge_nmalloc++;
    5142                 :         /* See note in huge_alloc() for why huge_allocated += psize is correct
    5143                 :          * here even when DECOMMIT is not defined. */
    5144               0 :         huge_allocated += psize;
    5145                 : #endif
    5146               0 :         malloc_mutex_unlock(&huge_mtx);
    5147                 : 
    5148                 : #ifdef MALLOC_DECOMMIT
    5149                 :         if (chunk_size - psize > 0) {
    5150                 :                 pages_decommit((void *)((uintptr_t)ret + psize),
    5151                 :                     chunk_size - psize);
    5152                 :         }
    5153                 : #endif
    5154                 : 
    5155                 : #ifdef MALLOC_DECOMMIT
    5156                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, false);
    5157                 : #else
    5158                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, chunk_size, 0, false);
    5159                 : #endif
    5160                 : 
    5161                 : #ifdef MALLOC_FILL
    5162               0 :         if (opt_junk)
    5163                 : #  ifdef MALLOC_DECOMMIT
    5164                 :                 memset(ret, 0xa5, psize);
    5165                 : #  else
    5166               0 :                 memset(ret, 0xa5, chunk_size);
    5167                 : #  endif
    5168               0 :         else if (opt_zero)
    5169                 : #  ifdef MALLOC_DECOMMIT
    5170                 :                 memset(ret, 0, psize);
    5171                 : #  else
    5172               0 :                 memset(ret, 0, chunk_size);
    5173                 : #  endif
    5174                 : #endif
    5175                 : 
    5176                 : RETURN:
    5177                 : #ifdef MALLOC_PAGEFILE
    5178                 :         if (pfd != -1)
    5179                 :                 pagefile_close(pfd);
    5180                 : #endif
    5181               0 :         return (ret);
    5182                 : }
    5183                 : 
    5184                 : static void *
    5185            2545 : huge_ralloc(void *ptr, size_t size, size_t oldsize)
    5186                 : {
    5187                 :         void *ret;
    5188                 :         size_t copysize;
    5189                 : 
    5190                 :         /* Avoid moving the allocation if the size class would not change. */
    5191                 : 
    5192            4871 :         if (oldsize > arena_maxclass &&
    5193            2326 :             CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
    5194             720 :                 size_t psize = PAGE_CEILING(size);
    5195                 : #ifdef MALLOC_FILL
    5196             720 :                 if (opt_junk && size < oldsize) {
    5197               0 :                         memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
    5198                 :                             - size);
    5199                 :                 }
    5200                 : #endif
    5201                 : #ifdef MALLOC_DECOMMIT
    5202                 :                 if (psize < oldsize) {
    5203                 :                         extent_node_t *node, key;
    5204                 : 
    5205                 :                         pages_decommit((void *)((uintptr_t)ptr + psize),
    5206                 :                             oldsize - psize);
    5207                 : 
    5208                 :                         /* Update recorded size. */
    5209                 :                         malloc_mutex_lock(&huge_mtx);
    5210                 :                         key.addr = __DECONST(void *, ptr);
    5211                 :                         node = extent_tree_ad_search(&huge, &key);
    5212                 :                         assert(node != NULL);
    5213                 :                         assert(node->size == oldsize);
    5214                 : #  ifdef MALLOC_STATS
    5215                 :                         huge_allocated -= oldsize - psize;
    5216                 : #  endif
    5217                 :                         node->size = psize;
    5218                 :                         malloc_mutex_unlock(&huge_mtx);
    5219                 :                 } else if (psize > oldsize) {
    5220                 :                         pages_commit((void *)((uintptr_t)ptr + oldsize),
    5221                 :                             psize - oldsize);
    5222                 :                 }
    5223                 : #endif
    5224                 : 
    5225                 :                 /* Although we don't have to commit or decommit anything if
    5226                 :                  * DECOMMIT is not defined and the size class didn't change, we
    5227                 :                  * do need to update the recorded size if the size increased,
    5228                 :                  * so malloc_usable_size doesn't return a value smaller than
    5229                 :                  * what was requested via realloc(). */
    5230                 : 
    5231             720 :                 if (psize > oldsize) {
    5232                 :                         /* Update recorded size. */
    5233                 :                         extent_node_t *node, key;
    5234             720 :                         malloc_mutex_lock(&huge_mtx);
    5235             720 :                         key.addr = __DECONST(void *, ptr);
    5236             720 :                         node = extent_tree_ad_search(&huge, &key);
    5237             720 :                         assert(node != NULL);
    5238             720 :                         assert(node->size == oldsize);
    5239                 : #  ifdef MALLOC_STATS
    5240             720 :                         huge_allocated += psize - oldsize;
    5241                 : #  endif
    5242             720 :                         node->size = psize;
    5243             720 :                         malloc_mutex_unlock(&huge_mtx);
    5244                 :                 }
    5245                 : 
    5246                 : #ifdef MALLOC_FILL
    5247             720 :                 if (opt_zero && size > oldsize) {
    5248               0 :                         memset((void *)((uintptr_t)ptr + oldsize), 0, size
    5249                 :                             - oldsize);
    5250                 :                 }
    5251                 : #endif
    5252             720 :                 return (ptr);
    5253                 :         }
    5254                 : 
    5255                 :         /*
    5256                 :          * If we get here, then size and oldsize are different enough that we
    5257                 :          * need to use a different size class.  In that case, fall back to
    5258                 :          * allocating new space and copying.
    5259                 :          */
    5260            1825 :         ret = huge_malloc(size, false);
    5261            1825 :         if (ret == NULL)
    5262               0 :                 return (NULL);
    5263                 : 
    5264            1825 :         copysize = (size < oldsize) ? size : oldsize;
    5265                 : #ifdef VM_COPY_MIN
    5266                 :         if (copysize >= VM_COPY_MIN)
    5267                 :                 pages_copy(ret, ptr, copysize);
    5268                 :         else
    5269                 : #endif
    5270            1825 :                 memcpy(ret, ptr, copysize);
    5271            1825 :         idalloc(ptr);
    5272            1825 :         return (ret);
    5273                 : }
    5274                 : 
    5275                 : static void
    5276            5519 : huge_dalloc(void *ptr)
    5277                 : {
    5278                 :         extent_node_t *node, key;
    5279                 : 
    5280            5519 :         malloc_mutex_lock(&huge_mtx);
    5281                 : 
    5282                 :         /* Extract from tree of huge allocations. */
    5283            5519 :         key.addr = ptr;
    5284            5519 :         node = extent_tree_ad_search(&huge, &key);
    5285            5519 :         assert(node != NULL);
    5286            5519 :         assert(node->addr == ptr);
    5287            5519 :         extent_tree_ad_remove(&huge, node);
    5288                 : 
    5289                 : #ifdef MALLOC_STATS
    5290            5519 :         huge_ndalloc++;
    5291            5519 :         huge_allocated -= node->size;
    5292                 : #endif
    5293                 : 
    5294            5519 :         malloc_mutex_unlock(&huge_mtx);
    5295                 : 
    5296                 :         /* Unmap chunk. */
    5297                 : #ifdef MALLOC_FILL
    5298            5519 :         if (opt_junk)
    5299            5519 :                 memset(node->addr, 0x5a, node->size);
    5300                 : #endif
    5301            5519 :         chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
    5302                 :         VALGRIND_FREELIKE_BLOCK(node->addr, 0);
    5303                 : 
    5304            5519 :         base_node_dealloc(node);
    5305            5519 : }
    5306                 : 
    5307                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5308                 : #ifdef MOZ_MEMORY_BSD
    5309                 : static inline unsigned
    5310                 : malloc_ncpus(void)
    5311                 : {
    5312                 :         unsigned ret;
    5313                 :         int mib[2];
    5314                 :         size_t len;
    5315                 : 
    5316                 :         mib[0] = CTL_HW;
    5317                 :         mib[1] = HW_NCPU;
    5318                 :         len = sizeof(ret);
    5319                 :         if (sysctl(mib, 2, &ret, &len, (void *) 0, 0) == -1) {
    5320                 :                 /* Error. */
    5321                 :                 return (1);
    5322                 :         }
    5323                 : 
    5324                 :         return (ret);
    5325                 : }
    5326                 : #elif (defined(MOZ_MEMORY_LINUX))
    5327                 : #include <fcntl.h>
    5328                 : 
    5329                 : static inline unsigned
    5330                 : malloc_ncpus(void)
    5331                 : {
    5332                 :         unsigned ret;
    5333                 :         int fd, nread, column;
    5334                 :         char buf[1024];
    5335                 :         static const char matchstr[] = "processor\t:";
    5336                 :         int i;
    5337                 : 
    5338                 :         /*
    5339                 :          * sysconf(3) would be the preferred method for determining the number
    5340                 :          * of CPUs, but it uses malloc internally, which causes untennable
    5341                 :          * recursion during malloc initialization.
    5342                 :          */
    5343                 :         fd = open("/proc/cpuinfo", O_RDONLY);
    5344                 :         if (fd == -1)
    5345                 :                 return (1); /* Error. */
    5346                 :         /*
    5347                 :          * Count the number of occurrences of matchstr at the beginnings of
    5348                 :          * lines.  This treats hyperthreaded CPUs as multiple processors.
    5349                 :          */
    5350                 :         column = 0;
    5351                 :         ret = 0;
    5352                 :         while (true) {
    5353                 :                 nread = read(fd, &buf, sizeof(buf));
    5354                 :                 if (nread <= 0)
    5355                 :                         break; /* EOF or error. */
    5356                 :                 for (i = 0;i < nread;i++) {
    5357                 :                         char c = buf[i];
    5358                 :                         if (c == '\n')
    5359                 :                                 column = 0;
    5360                 :                         else if (column != -1) {
    5361                 :                                 if (c == matchstr[column]) {
    5362                 :                                         column++;
    5363                 :                                         if (column == sizeof(matchstr) - 1) {
    5364                 :                                                 column = -1;
    5365                 :                                                 ret++;
    5366                 :                                         }
    5367                 :                                 } else
    5368                 :                                         column = -1;
    5369                 :                         }
    5370                 :                 }
    5371                 :         }
    5372                 : 
    5373                 :         if (ret == 0)
    5374                 :                 ret = 1; /* Something went wrong in the parser. */
    5375                 :         close(fd);
    5376                 : 
    5377                 :         return (ret);
    5378                 : }
    5379                 : #elif (defined(MOZ_MEMORY_DARWIN))
    5380                 : #include <mach/mach_init.h>
    5381                 : #include <mach/mach_host.h>
    5382                 : 
    5383                 : static inline unsigned
    5384                 : malloc_ncpus(void)
    5385                 : {
    5386                 :         kern_return_t error;
    5387                 :         natural_t n;
    5388                 :         processor_info_array_t pinfo;
    5389                 :         mach_msg_type_number_t pinfocnt;
    5390                 : 
    5391                 :         error = host_processor_info(mach_host_self(), PROCESSOR_BASIC_INFO,
    5392                 :                                     &n, &pinfo, &pinfocnt);
    5393                 :         if (error != KERN_SUCCESS)
    5394                 :                 return (1); /* Error. */
    5395                 :         else
    5396                 :                 return (n);
    5397                 : }
    5398                 : #elif (defined(MOZ_MEMORY_SOLARIS))
    5399                 : 
    5400                 : static inline unsigned
    5401                 : malloc_ncpus(void)
    5402                 : {
    5403                 :         return sysconf(_SC_NPROCESSORS_ONLN);
    5404                 : }
    5405                 : #else
    5406                 : static inline unsigned
    5407                 : malloc_ncpus(void)
    5408                 : {
    5409                 : 
    5410                 :         /*
    5411                 :          * We lack a way to determine the number of CPUs on this platform, so
    5412                 :          * assume 1 CPU.
    5413                 :          */
    5414                 :         return (1);
    5415                 : }
    5416                 : #endif
    5417                 : #endif
    5418                 : 
    5419                 : static void
    5420               0 : malloc_print_stats(void)
    5421                 : {
    5422                 : 
    5423               0 :         if (opt_print_stats) {
    5424                 :                 char s[UMAX2S_BUFSIZE];
    5425               0 :                 _malloc_message("___ Begin malloc statistics ___\n", "", "",
    5426                 :                     "");
    5427               0 :                 _malloc_message("Assertions ",
    5428                 : #ifdef NDEBUG
    5429                 :                     "disabled",
    5430                 : #else
    5431                 :                     "enabled",
    5432                 : #endif
    5433                 :                     "\n", "");
    5434               0 :                 _malloc_message("Boolean MALLOC_OPTIONS: ",
    5435                 :                     opt_abort ? "A" : "a", "", "");
    5436                 : #ifdef MALLOC_FILL
    5437               0 :                 _malloc_message(opt_junk ? "J" : "j", "", "", "");
    5438                 : #endif
    5439                 : #ifdef MALLOC_PAGEFILE
    5440                 :                 _malloc_message(opt_pagefile ? "o" : "O", "", "", "");
    5441                 : #endif
    5442               0 :                 _malloc_message("P", "", "", "");
    5443                 : #ifdef MALLOC_UTRACE
    5444               0 :                 _malloc_message(opt_utrace ? "U" : "u", "", "", "");
    5445                 : #endif
    5446                 : #ifdef MALLOC_SYSV
    5447               0 :                 _malloc_message(opt_sysv ? "V" : "v", "", "", "");
    5448                 : #endif
    5449                 : #ifdef MALLOC_XMALLOC
    5450               0 :                 _malloc_message(opt_xmalloc ? "X" : "x", "", "", "");
    5451                 : #endif
    5452                 : #ifdef MALLOC_FILL
    5453               0 :                 _malloc_message(opt_zero ? "Z" : "z", "", "", "");
    5454                 : #endif
    5455               0 :                 _malloc_message("\n", "", "", "");
    5456                 : 
    5457                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5458                 :                 _malloc_message("CPUs: ", umax2s(ncpus, 10, s), "\n", "");
    5459                 : #endif
    5460               0 :                 _malloc_message("Max arenas: ", umax2s(narenas, 10, s), "\n",
    5461                 :                     "");
    5462                 : #ifdef MALLOC_BALANCE
    5463                 :                 _malloc_message("Arena balance threshold: ",
    5464                 :                     umax2s(opt_balance_threshold, 10, s), "\n", "");
    5465                 : #endif
    5466               0 :                 _malloc_message("Pointer size: ", umax2s(sizeof(void *), 10, s),
    5467                 :                     "\n", "");
    5468               0 :                 _malloc_message("Quantum size: ", umax2s(quantum, 10, s), "\n",
    5469                 :                     "");
    5470               0 :                 _malloc_message("Max small size: ", umax2s(small_max, 10, s),
    5471                 :                     "\n", "");
    5472               0 :                 _malloc_message("Max dirty pages per arena: ",
    5473               0 :                     umax2s(opt_dirty_max, 10, s), "\n", "");
    5474                 : 
    5475               0 :                 _malloc_message("Chunk size: ", umax2s(chunksize, 10, s), "",
    5476                 :                     "");
    5477               0 :                 _malloc_message(" (2^", umax2s(opt_chunk_2pow, 10, s), ")\n",
    5478                 :                     "");
    5479                 : 
    5480                 : #ifdef MALLOC_STATS
    5481                 :                 {
    5482                 :                         size_t allocated, mapped;
    5483                 : #ifdef MALLOC_BALANCE
    5484                 :                         uint64_t nbalance = 0;
    5485                 : #endif
    5486                 :                         unsigned i;
    5487                 :                         arena_t *arena;
    5488                 : 
    5489                 :                         /* Calculate and print allocated/mapped stats. */
    5490                 : 
    5491                 :                         /* arenas. */
    5492               0 :                         for (i = 0, allocated = 0; i < narenas; i++) {
    5493               0 :                                 if (arenas[i] != NULL) {
    5494               0 :                                         malloc_spin_lock(&arenas[i]->lock);
    5495               0 :                                         allocated +=
    5496               0 :                                             arenas[i]->stats.allocated_small;
    5497               0 :                                         allocated +=
    5498               0 :                                             arenas[i]->stats.allocated_large;
    5499                 : #ifdef MALLOC_BALANCE
    5500                 :                                         nbalance += arenas[i]->stats.nbalance;
    5501                 : #endif
    5502               0 :                                         malloc_spin_unlock(&arenas[i]->lock);
    5503                 :                                 }
    5504                 :                         }
    5505                 : 
    5506                 :                         /* huge/base. */
    5507               0 :                         malloc_mutex_lock(&huge_mtx);
    5508               0 :                         allocated += huge_allocated;
    5509               0 :                         mapped = stats_chunks.curchunks * chunksize;
    5510               0 :                         malloc_mutex_unlock(&huge_mtx);
    5511                 : 
    5512               0 :                         malloc_mutex_lock(&base_mtx);
    5513               0 :                         mapped += base_mapped;
    5514               0 :                         malloc_mutex_unlock(&base_mtx);
    5515                 : 
    5516                 : #ifdef MOZ_MEMORY_WINDOWS
    5517                 :                         malloc_printf("Allocated: %lu, mapped: %lu\n",
    5518                 :                             allocated, mapped);
    5519                 : #else
    5520               0 :                         malloc_printf("Allocated: %zu, mapped: %zu\n",
    5521                 :                             allocated, mapped);
    5522                 : #endif
    5523                 : 
    5524                 : #ifdef MALLOC_BALANCE
    5525                 :                         malloc_printf("Arena balance reassignments: %llu\n",
    5526                 :                             nbalance);
    5527                 : #endif
    5528                 : 
    5529                 :                         /* Print chunk stats. */
    5530                 :                         {
    5531                 :                                 chunk_stats_t chunks_stats;
    5532                 : 
    5533               0 :                                 malloc_mutex_lock(&huge_mtx);
    5534               0 :                                 chunks_stats = stats_chunks;
    5535               0 :                                 malloc_mutex_unlock(&huge_mtx);
    5536                 : 
    5537               0 :                                 malloc_printf("chunks: nchunks   "
    5538                 :                                     "highchunks    curchunks\n");
    5539               0 :                                 malloc_printf("  %13llu%13lu%13lu\n",
    5540                 :                                     chunks_stats.nchunks,
    5541                 :                                     chunks_stats.highchunks,
    5542                 :                                     chunks_stats.curchunks);
    5543                 :                         }
    5544                 : 
    5545                 :                         /* Print chunk stats. */
    5546               0 :                         malloc_printf(
    5547                 :                             "huge: nmalloc      ndalloc    allocated\n");
    5548                 : #ifdef MOZ_MEMORY_WINDOWS
    5549                 :                         malloc_printf(" %12llu %12llu %12lu\n",
    5550                 :                             huge_nmalloc, huge_ndalloc, huge_allocated);
    5551                 : #else
    5552               0 :                         malloc_printf(" %12llu %12llu %12zu\n",
    5553                 :                             huge_nmalloc, huge_ndalloc, huge_allocated);
    5554                 : #endif
    5555                 :                         /* Print stats for each arena. */
    5556               0 :                         for (i = 0; i < narenas; i++) {
    5557               0 :                                 arena = arenas[i];
    5558               0 :                                 if (arena != NULL) {
    5559               0 :                                         malloc_printf(
    5560                 :                                             "\narenas[%u]:\n", i);
    5561               0 :                                         malloc_spin_lock(&arena->lock);
    5562               0 :                                         stats_print(arena);
    5563               0 :                                         malloc_spin_unlock(&arena->lock);
    5564                 :                                 }
    5565                 :                         }
    5566                 :                 }
    5567                 : #endif /* #ifdef MALLOC_STATS */
    5568               0 :                 _malloc_message("--- End malloc statistics ---\n", "", "", "");
    5569                 :         }
    5570               0 : }
    5571                 : 
    5572                 : /*
    5573                 :  * FreeBSD's pthreads implementation calls malloc(3), so the malloc
    5574                 :  * implementation has to take pains to avoid infinite recursion during
    5575                 :  * initialization.
    5576                 :  */
    5577                 : #if (defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_DARWIN))
    5578                 : #define malloc_init() false
    5579                 : #else
    5580                 : static inline bool
    5581       121360462 : malloc_init(void)
    5582                 : {
    5583                 : 
    5584       121360462 :         if (malloc_initialized == false)
    5585           19901 :                 return (malloc_init_hard());
    5586                 : 
    5587       121340561 :         return (false);
    5588                 : }
    5589                 : #endif
    5590                 : 
    5591                 : #if !defined(MOZ_MEMORY_WINDOWS)
    5592                 : static
    5593                 : #endif
    5594                 : bool
    5595           19901 : malloc_init_hard(void)
    5596                 : {
    5597                 :         unsigned i;
    5598                 :         char buf[PATH_MAX + 1];
    5599                 :         const char *opts;
    5600                 :         long result;
    5601                 : #ifndef MOZ_MEMORY_WINDOWS
    5602                 :         int linklen;
    5603                 : #endif
    5604                 : #ifdef MOZ_MEMORY_DARWIN
    5605                 :     malloc_zone_t* default_zone;
    5606                 : #endif
    5607                 : 
    5608                 : #ifndef MOZ_MEMORY_WINDOWS
    5609           19901 :         malloc_mutex_lock(&init_lock);
    5610                 : #endif
    5611                 : 
    5612           19901 :         if (malloc_initialized) {
    5613                 :                 /*
    5614                 :                  * Another thread initialized the allocator before this one
    5615                 :                  * acquired init_lock.
    5616                 :                  */
    5617                 : #ifndef MOZ_MEMORY_WINDOWS
    5618               0 :                 malloc_mutex_unlock(&init_lock);
    5619                 : #endif
    5620               0 :                 return (false);
    5621                 :         }
    5622                 : 
    5623                 : #ifdef MOZ_MEMORY_WINDOWS
    5624                 :         /* get a thread local storage index */
    5625                 :         tlsIndex = TlsAlloc();
    5626                 : #endif
    5627                 : 
    5628                 :         /* Get page size and number of CPUs */
    5629                 : #ifdef MOZ_MEMORY_WINDOWS
    5630                 :         {
    5631                 :                 SYSTEM_INFO info;
    5632                 : 
    5633                 :                 GetSystemInfo(&info);
    5634                 :                 result = info.dwPageSize;
    5635                 : 
    5636                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5637                 :                 ncpus = info.dwNumberOfProcessors;
    5638                 : #endif
    5639                 :         }
    5640                 : #else
    5641                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    5642                 :         ncpus = malloc_ncpus();
    5643                 : #endif
    5644                 : 
    5645           19901 :         result = sysconf(_SC_PAGESIZE);
    5646           19901 :         assert(result != -1);
    5647                 : #endif
    5648                 : 
    5649                 :         /* We assume that the page size is a power of 2. */
    5650           19901 :         assert(((result - 1) & result) == 0);
    5651                 : #ifdef MALLOC_STATIC_SIZES
    5652           19901 :         if (pagesize % (size_t) result) {
    5653               0 :                 _malloc_message(_getprogname(),
    5654                 :                                 "Compile-time page size does not divide the runtime one.\n",
    5655                 :                                 "", "");
    5656               0 :                 abort();
    5657                 :         }
    5658                 : #else   
    5659                 :         pagesize = (size_t) result;
    5660                 :         pagesize_mask = (size_t) result - 1;
    5661                 :         pagesize_2pow = ffs((int)result) - 1;
    5662                 : #endif
    5663                 :         
    5664                 : #ifdef MALLOC_PAGEFILE
    5665                 :         /*
    5666                 :          * Determine where to create page files.  It is insufficient to
    5667                 :          * unconditionally use P_tmpdir (typically "/tmp"), since for some
    5668                 :          * operating systems /tmp is a separate filesystem that is rather small.
    5669                 :          * Therefore prefer, in order, the following locations:
    5670                 :          *
    5671                 :          * 1) MALLOC_TMPDIR
    5672                 :          * 2) TMPDIR
    5673                 :          * 3) P_tmpdir
    5674                 :          */
    5675                 :         {
    5676                 :                 char *s;
    5677                 :                 size_t slen;
    5678                 :                 static const char suffix[] = "/jemalloc.XXXXXX";
    5679                 : 
    5680                 :                 if ((s = getenv("MALLOC_TMPDIR")) == NULL && (s =
    5681                 :                     getenv("TMPDIR")) == NULL)
    5682                 :                         s = P_tmpdir;
    5683                 :                 slen = strlen(s);
    5684                 :                 if (slen + sizeof(suffix) > sizeof(pagefile_templ)) {
    5685                 :                         _malloc_message(_getprogname(),
    5686                 :                             ": (malloc) Page file path too long\n",
    5687                 :                             "", "");
    5688                 :                         abort();
    5689                 :                 }
    5690                 :                 memcpy(pagefile_templ, s, slen);
    5691                 :                 memcpy(&pagefile_templ[slen], suffix, sizeof(suffix));
    5692                 :         }
    5693                 : #endif
    5694                 : 
    5695           79604 :         for (i = 0; i < 3; i++) {
    5696                 :                 unsigned j;
    5697                 : 
    5698                 :                 /* Get runtime configuration. */
    5699           59703 :                 switch (i) {
    5700                 :                 case 0:
    5701                 : #ifndef MOZ_MEMORY_WINDOWS
    5702           19901 :                         if ((linklen = readlink("/etc/malloc.conf", buf,
    5703                 :                                                 sizeof(buf) - 1)) != -1) {
    5704                 :                                 /*
    5705                 :                                  * Use the contents of the "/etc/malloc.conf"
    5706                 :                                  * symbolic link's name.
    5707                 :                                  */
    5708               0 :                                 buf[linklen] = '\0';
    5709               0 :                                 opts = buf;
    5710                 :                         } else
    5711                 : #endif
    5712                 :                         {
    5713                 :                                 /* No configuration specified. */
    5714           19901 :                                 buf[0] = '\0';
    5715           19901 :                                 opts = buf;
    5716                 :                         }
    5717           19901 :                         break;
    5718                 :                 case 1:
    5719           19901 :                         if (issetugid() == 0 && (opts =
    5720                 :                             getenv("MALLOC_OPTIONS")) != NULL) {
    5721                 :                                 /*
    5722                 :                                  * Do nothing; opts is already initialized to
    5723                 :                                  * the value of the MALLOC_OPTIONS environment
    5724                 :                                  * variable.
    5725                 :                                  */
    5726                 :                         } else {
    5727                 :                                 /* No configuration specified. */
    5728           19901 :                                 buf[0] = '\0';
    5729           19901 :                                 opts = buf;
    5730                 :                         }
    5731           19901 :                         break;
    5732                 :                 case 2:
    5733           19901 :                         if (_malloc_options != NULL) {
    5734                 :                                 /*
    5735                 :                                  * Use options that were compiled into the
    5736                 :                                  * program.
    5737                 :                                  */
    5738               0 :                                 opts = _malloc_options;
    5739                 :                         } else {
    5740                 :                                 /* No configuration specified. */
    5741           19901 :                                 buf[0] = '\0';
    5742           19901 :                                 opts = buf;
    5743                 :                         }
    5744           19901 :                         break;
    5745                 :                 default:
    5746                 :                         /* NOTREACHED */
    5747               0 :                         buf[0] = '\0';
    5748               0 :                         opts = buf;
    5749               0 :                         assert(false);
    5750                 :                 }
    5751                 : 
    5752           59703 :                 for (j = 0; opts[j] != '\0'; j++) {
    5753                 :                         unsigned k, nreps;
    5754                 :                         bool nseen;
    5755                 : 
    5756                 :                         /* Parse repetition count, if any. */
    5757               0 :                         for (nreps = 0, nseen = false;; j++, nseen = true) {
    5758               0 :                                 switch (opts[j]) {
    5759                 :                                         case '0': case '1': case '2': case '3':
    5760                 :                                         case '4': case '5': case '6': case '7':
    5761                 :                                         case '8': case '9':
    5762               0 :                                                 nreps *= 10;
    5763               0 :                                                 nreps += opts[j] - '0';
    5764                 :                                                 break;
    5765                 :                                         default:
    5766               0 :                                                 goto MALLOC_OUT;
    5767                 :                                 }
    5768               0 :                         }
    5769                 : MALLOC_OUT:
    5770               0 :                         if (nseen == false)
    5771               0 :                                 nreps = 1;
    5772                 : 
    5773               0 :                         for (k = 0; k < nreps; k++) {
    5774               0 :                                 switch (opts[j]) {
    5775                 :                                 case 'a':
    5776               0 :                                         opt_abort = false;
    5777               0 :                                         break;
    5778                 :                                 case 'A':
    5779               0 :                                         opt_abort = true;
    5780               0 :                                         break;
    5781                 :                                 case 'b':
    5782                 : #ifdef MALLOC_BALANCE
    5783                 :                                         opt_balance_threshold >>= 1;
    5784                 : #endif
    5785               0 :                                         break;
    5786                 :                                 case 'B':
    5787                 : #ifdef MALLOC_BALANCE
    5788                 :                                         if (opt_balance_threshold == 0)
    5789                 :                                                 opt_balance_threshold = 1;
    5790                 :                                         else if ((opt_balance_threshold << 1)
    5791                 :                                             > opt_balance_threshold)
    5792                 :                                                 opt_balance_threshold <<= 1;
    5793                 : #endif
    5794               0 :                                         break;
    5795                 :                                 case 'f':
    5796               0 :                                         opt_dirty_max >>= 1;
    5797               0 :                                         break;
    5798                 :                                 case 'F':
    5799               0 :                                         if (opt_dirty_max == 0)
    5800               0 :                                                 opt_dirty_max = 1;
    5801               0 :                                         else if ((opt_dirty_max << 1) != 0)
    5802               0 :                                                 opt_dirty_max <<= 1;
    5803               0 :                                         break;
    5804                 : #ifdef MALLOC_FILL
    5805                 :                                 case 'j':
    5806               0 :                                         opt_junk = false;
    5807               0 :                                         break;
    5808                 :                                 case 'J':
    5809               0 :                                         opt_junk = true;
    5810               0 :                                         break;
    5811                 : #endif
    5812                 : #ifndef MALLOC_STATIC_SIZES
    5813                 :                                 case 'k':
    5814                 :                                         /*
    5815                 :                                          * Chunks always require at least one
    5816                 :                                          * header page, so chunks can never be
    5817                 :                                          * smaller than two pages.
    5818                 :                                          */
    5819                 :                                         if (opt_chunk_2pow > pagesize_2pow + 1)
    5820                 :                                                 opt_chunk_2pow--;
    5821                 :                                         break;
    5822                 :                                 case 'K':
    5823                 :                                         if (opt_chunk_2pow + 1 <
    5824                 :                                             (sizeof(size_t) << 3))
    5825                 :                                                 opt_chunk_2pow++;
    5826                 :                                         break;
    5827                 : #endif
    5828                 :                                 case 'n':
    5829               0 :                                         opt_narenas_lshift--;
    5830               0 :                                         break;
    5831                 :                                 case 'N':
    5832               0 :                                         opt_narenas_lshift++;
    5833               0 :                                         break;
    5834                 : #ifdef MALLOC_PAGEFILE
    5835                 :                                 case 'o':
    5836                 :                                         /* Do not over-commit. */
    5837                 :                                         opt_pagefile = true;
    5838                 :                                         break;
    5839                 :                                 case 'O':
    5840                 :                                         /* Allow over-commit. */
    5841                 :                                         opt_pagefile = false;
    5842                 :                                         break;
    5843                 : #endif
    5844                 :                                 case 'p':
    5845               0 :                                         opt_print_stats = false;
    5846               0 :                                         break;
    5847                 :                                 case 'P':
    5848               0 :                                         opt_print_stats = true;
    5849               0 :                                         break;
    5850                 : #ifndef MALLOC_STATIC_SIZES
    5851                 :                                 case 'q':
    5852                 :                                         if (opt_quantum_2pow > QUANTUM_2POW_MIN)
    5853                 :                                                 opt_quantum_2pow--;
    5854                 :                                         break;
    5855                 :                                 case 'Q':
    5856                 :                                         if (opt_quantum_2pow < pagesize_2pow -
    5857                 :                                             1)
    5858                 :                                                 opt_quantum_2pow++;
    5859                 :                                         break;
    5860                 :                                 case 's':
    5861                 :                                         if (opt_small_max_2pow >
    5862                 :                                             QUANTUM_2POW_MIN)
    5863                 :                                                 opt_small_max_2pow--;
    5864                 :                                         break;
    5865                 :                                 case 'S':
    5866                 :                                         if (opt_small_max_2pow < pagesize_2pow
    5867                 :                                             - 1)
    5868                 :                                                 opt_small_max_2pow++;
    5869                 :                                         break;
    5870                 : #endif
    5871                 : #ifdef MALLOC_UTRACE
    5872                 :                                 case 'u':
    5873               0 :                                         opt_utrace = false;
    5874               0 :                                         break;
    5875                 :                                 case 'U':
    5876               0 :                                         opt_utrace = true;
    5877               0 :                                         break;
    5878                 : #endif
    5879                 : #ifdef MALLOC_SYSV
    5880                 :                                 case 'v':
    5881               0 :                                         opt_sysv = false;
    5882               0 :                                         break;
    5883                 :                                 case 'V':
    5884               0 :                                         opt_sysv = true;
    5885               0 :                                         break;
    5886                 : #endif
    5887                 : #ifdef MALLOC_XMALLOC
    5888                 :                                 case 'x':
    5889               0 :                                         opt_xmalloc = false;
    5890               0 :                                         break;
    5891                 :                                 case 'X':
    5892               0 :                                         opt_xmalloc = true;
    5893               0 :                                         break;
    5894                 : #endif
    5895                 : #ifdef MALLOC_FILL
    5896                 :                                 case 'z':
    5897               0 :                                         opt_zero = false;
    5898               0 :                                         break;
    5899                 :                                 case 'Z':
    5900               0 :                                         opt_zero = true;
    5901               0 :                                         break;
    5902                 : #endif
    5903                 :                                 default: {
    5904                 :                                         char cbuf[2];
    5905                 : 
    5906               0 :                                         cbuf[0] = opts[j];
    5907               0 :                                         cbuf[1] = '\0';
    5908               0 :                                         _malloc_message(_getprogname(),
    5909                 :                                             ": (malloc) Unsupported character "
    5910                 :                                             "in malloc options: '", cbuf,
    5911                 :                                             "'\n");
    5912                 :                                 }
    5913                 :                                 }
    5914                 :                         }
    5915                 :                 }
    5916                 :         }
    5917                 : 
    5918                 :         /* Take care to call atexit() only once. */
    5919           19901 :         if (opt_print_stats) {
    5920                 : #ifndef MOZ_MEMORY_WINDOWS
    5921                 :                 /* Print statistics at exit. */
    5922               0 :                 atexit(malloc_print_stats);
    5923                 : #endif
    5924                 :         }
    5925                 : 
    5926                 : #if (!defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN) && !defined(MOZ_MEMORY_ANDROID))
    5927                 :         /* Prevent potential deadlock on malloc locks after fork. */
    5928                 :         /* XXX on Android there is no pthread_atfork, so we specifically
    5929                 :            call _malloc_prefork and _malloc_postfork in process_util_linux.cc */
    5930           19901 :         pthread_atfork(_malloc_prefork, _malloc_postfork, _malloc_postfork);
    5931                 : #endif
    5932                 : 
    5933                 : #ifndef MALLOC_STATIC_SIZES
    5934                 :         /* Set variables according to the value of opt_small_max_2pow. */
    5935                 :         if (opt_small_max_2pow < opt_quantum_2pow)
    5936                 :                 opt_small_max_2pow = opt_quantum_2pow;
    5937                 :         small_max = (1U << opt_small_max_2pow);
    5938                 : 
    5939                 :         /* Set bin-related variables. */
    5940                 :         bin_maxclass = (pagesize >> 1);
    5941                 :         assert(opt_quantum_2pow >= TINY_MIN_2POW);
    5942                 :         ntbins = opt_quantum_2pow - TINY_MIN_2POW;
    5943                 :         assert(ntbins <= opt_quantum_2pow);
    5944                 :         nqbins = (small_max >> opt_quantum_2pow);
    5945                 :         nsbins = pagesize_2pow - opt_small_max_2pow - 1;
    5946                 : 
    5947                 :         /* Set variables according to the value of opt_quantum_2pow. */
    5948                 :         quantum = (1U << opt_quantum_2pow);
    5949                 :         quantum_mask = quantum - 1;
    5950                 :         if (ntbins > 0)
    5951                 :                 small_min = (quantum >> 1) + 1;
    5952                 :         else
    5953                 :                 small_min = 1;
    5954                 :         assert(small_min <= quantum);
    5955                 : 
    5956                 :         /* Set variables according to the value of opt_chunk_2pow. */
    5957                 :         chunksize = (1LU << opt_chunk_2pow);
    5958                 :         chunksize_mask = chunksize - 1;
    5959                 :         chunk_npages = (chunksize >> pagesize_2pow);
    5960                 : 
    5961                 :         arena_chunk_header_npages = calculate_arena_header_pages();
    5962                 :         arena_maxclass = calculate_arena_maxclass();
    5963                 : #endif
    5964                 : 
    5965                 : #ifdef JEMALLOC_USES_MAP_ALIGN
    5966                 :         /*
    5967                 :          * When using MAP_ALIGN, the alignment parameter must be a power of two
    5968                 :          * multiple of the system pagesize, or mmap will fail.
    5969                 :          */
    5970                 :         assert((chunksize % pagesize) == 0);
    5971                 :         assert((1 << (ffs(chunksize / pagesize) - 1)) == (chunksize/pagesize));
    5972                 : #endif
    5973                 : 
    5974           19901 :         UTRACE(0, 0, 0);
    5975                 : 
    5976                 : #ifdef MALLOC_STATS
    5977           19901 :         memset(&stats_chunks, 0, sizeof(chunk_stats_t));
    5978                 : #endif
    5979                 : 
    5980                 :         /* Various sanity checks that regard configuration. */
    5981           19901 :         assert(quantum >= sizeof(void *));
    5982           19901 :         assert(quantum <= pagesize);
    5983           19901 :         assert(chunksize >= pagesize);
    5984           19901 :         assert(quantum * 4 <= chunksize);
    5985                 : 
    5986                 :         /* Initialize chunks data. */
    5987           19901 :         malloc_mutex_init(&huge_mtx);
    5988           19901 :         extent_tree_ad_new(&huge);
    5989                 : #ifdef MALLOC_STATS
    5990           19901 :         huge_nmalloc = 0;
    5991           19901 :         huge_ndalloc = 0;
    5992           19901 :         huge_allocated = 0;
    5993                 : #endif
    5994                 : 
    5995                 :         /* Initialize base allocation data structures. */
    5996                 : #ifdef MALLOC_STATS
    5997           19901 :         base_mapped = 0;
    5998           19901 :         base_committed = 0;
    5999                 : #endif
    6000           19901 :         base_nodes = NULL;
    6001           19901 :         malloc_mutex_init(&base_mtx);
    6002                 : 
    6003                 : #ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE
    6004           19901 :         narenas = 1;
    6005                 : #else
    6006                 :         if (ncpus > 1) {
    6007                 :                 /*
    6008                 :                  * For SMP systems, create four times as many arenas as there
    6009                 :                  * are CPUs by default.
    6010                 :                  */
    6011                 :                 opt_narenas_lshift += 2;
    6012                 :         }
    6013                 : 
    6014                 :         /* Determine how many arenas to use. */
    6015                 :         narenas = ncpus;
    6016                 : #endif
    6017           19901 :         if (opt_narenas_lshift > 0) {
    6018               0 :                 if ((narenas << opt_narenas_lshift) > narenas)
    6019               0 :                         narenas <<= opt_narenas_lshift;
    6020                 :                 /*
    6021                 :                  * Make sure not to exceed the limits of what base_alloc() can
    6022                 :                  * handle.
    6023                 :                  */
    6024               0 :                 if (narenas * sizeof(arena_t *) > chunksize)
    6025               0 :                         narenas = chunksize / sizeof(arena_t *);
    6026           19901 :         } else if (opt_narenas_lshift < 0) {
    6027               0 :                 if ((narenas >> -opt_narenas_lshift) < narenas)
    6028               0 :                         narenas >>= -opt_narenas_lshift;
    6029                 :                 /* Make sure there is at least one arena. */
    6030               0 :                 if (narenas == 0)
    6031               0 :                         narenas = 1;
    6032                 :         }
    6033                 : #ifdef MALLOC_BALANCE
    6034                 :         assert(narenas != 0);
    6035                 :         for (narenas_2pow = 0;
    6036                 :              (narenas >> (narenas_2pow + 1)) != 0;
    6037                 :              narenas_2pow++);
    6038                 : #endif
    6039                 : 
    6040                 : #ifdef NO_TLS
    6041                 :         if (narenas > 1) {
    6042                 :                 static const unsigned primes[] = {1, 3, 5, 7, 11, 13, 17, 19,
    6043                 :                     23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
    6044                 :                     89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149,
    6045                 :                     151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
    6046                 :                     223, 227, 229, 233, 239, 241, 251, 257, 263};
    6047                 :                 unsigned nprimes, parenas;
    6048                 : 
    6049                 :                 /*
    6050                 :                  * Pick a prime number of hash arenas that is more than narenas
    6051                 :                  * so that direct hashing of pthread_self() pointers tends to
    6052                 :                  * spread allocations evenly among the arenas.
    6053                 :                  */
    6054                 :                 assert((narenas & 1) == 0); /* narenas must be even. */
    6055                 :                 nprimes = (sizeof(primes) >> SIZEOF_INT_2POW);
    6056                 :                 parenas = primes[nprimes - 1]; /* In case not enough primes. */
    6057                 :                 for (i = 1; i < nprimes; i++) {
    6058                 :                         if (primes[i] > narenas) {
    6059                 :                                 parenas = primes[i];
    6060                 :                                 break;
    6061                 :                         }
    6062                 :                 }
    6063                 :                 narenas = parenas;
    6064                 :         }
    6065                 : #endif
    6066                 : 
    6067                 : #ifndef NO_TLS
    6068                 : #  ifndef MALLOC_BALANCE
    6069           19901 :         next_arena = 0;
    6070                 : #  endif
    6071                 : #endif
    6072                 : 
    6073                 :         /* Allocate and initialize arenas. */
    6074           19901 :         arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
    6075           19901 :         if (arenas == NULL) {
    6076                 : #ifndef MOZ_MEMORY_WINDOWS
    6077               0 :                 malloc_mutex_unlock(&init_lock);
    6078                 : #endif
    6079               0 :                 return (true);
    6080                 :         }
    6081                 :         /*
    6082                 :          * Zero the array.  In practice, this should always be pre-zeroed,
    6083                 :          * since it was just mmap()ed, but let's be sure.
    6084                 :          */
    6085           19901 :         memset(arenas, 0, sizeof(arena_t *) * narenas);
    6086                 : 
    6087                 :         /*
    6088                 :          * Initialize one arena here.  The rest are lazily created in
    6089                 :          * choose_arena_hard().
    6090                 :          */
    6091           19901 :         arenas_extend(0);
    6092           19901 :         if (arenas[0] == NULL) {
    6093                 : #ifndef MOZ_MEMORY_WINDOWS
    6094               0 :                 malloc_mutex_unlock(&init_lock);
    6095                 : #endif
    6096               0 :                 return (true);
    6097                 :         }
    6098                 : #ifndef NO_TLS
    6099                 :         /*
    6100                 :          * Assign the initial arena to the initial thread, in order to avoid
    6101                 :          * spurious creation of an extra arena if the application switches to
    6102                 :          * threaded mode.
    6103                 :          */
    6104                 : #ifdef MOZ_MEMORY_WINDOWS
    6105                 :         TlsSetValue(tlsIndex, arenas[0]);
    6106                 : #else
    6107           19901 :         arenas_map = arenas[0];
    6108                 : #endif
    6109                 : #endif
    6110                 : 
    6111                 :         /*
    6112                 :          * Seed here for the initial thread, since choose_arena_hard() is only
    6113                 :          * called for other threads.  The seed value doesn't really matter.
    6114                 :          */
    6115                 : #ifdef MALLOC_BALANCE
    6116                 :         SPRN(balance, 42);
    6117                 : #endif
    6118                 : 
    6119           19901 :         malloc_spin_init(&arenas_lock);
    6120                 : 
    6121                 : #ifdef MALLOC_VALIDATE
    6122           19901 :         chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
    6123           19901 :         if (chunk_rtree == NULL)
    6124               0 :                 return (true);
    6125                 : #endif
    6126                 : 
    6127           19901 :         malloc_initialized = true;
    6128                 : 
    6129                 : #if defined(NEEDS_PTHREAD_MMAP_UNALIGNED_TSD)
    6130                 :         if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
    6131                 :                 malloc_printf("<jemalloc>: Error in pthread_key_create()\n");
    6132                 :         }
    6133                 : #endif
    6134                 : 
    6135                 : #ifdef MOZ_MEMORY_DARWIN
    6136                 :         /*
    6137                 :         * Overwrite the default memory allocator to use jemalloc everywhere.
    6138                 :         */
    6139                 :         default_zone = malloc_default_zone();
    6140                 : 
    6141                 :         /*
    6142                 :          * We only use jemalloc with MacOS 10.6 and 10.7.  jemalloc is disabled
    6143                 :          * on 32-bit builds (10.5 and 32-bit 10.6) due to bug 702250, an
    6144                 :          * apparent MacOS bug.  In fact, this code isn't even compiled on
    6145                 :          * 32-bit builds.
    6146                 :          *
    6147                 :          * We'll have to update our code to work with newer versions, because
    6148                 :          * the malloc zone layout is likely to change.
    6149                 :          */
    6150                 : 
    6151                 :         osx_use_jemalloc = (default_zone->version == SNOW_LEOPARD_MALLOC_ZONE_T_VERSION ||
    6152                 :                             default_zone->version == LION_MALLOC_ZONE_T_VERSION);
    6153                 : 
    6154                 :         /* Allow us dynamically turn off jemalloc for testing. */
    6155                 :         if (getenv("NO_MAC_JEMALLOC")) {
    6156                 :                 osx_use_jemalloc = false;
    6157                 : #ifdef __i386__
    6158                 :                 malloc_printf("Warning: NO_MAC_JEMALLOC has no effect on "
    6159                 :                               "i386 machines (such as this one).\n");
    6160                 : #endif
    6161                 :         }
    6162                 : 
    6163                 :         if (osx_use_jemalloc) {
    6164                 :                 /*
    6165                 :                  * Convert the default szone to an "overlay zone" that is capable
    6166                 :                  * of deallocating szone-allocated objects, but allocating new
    6167                 :                  * objects from jemalloc.
    6168                 :                  */
    6169                 :                 size_t size = zone_version_size(default_zone->version);
    6170                 :                 szone2ozone(default_zone, size);
    6171                 :         }
    6172                 :         else {
    6173                 :                 szone = default_zone;
    6174                 :         }
    6175                 : #endif
    6176                 : 
    6177                 : #ifndef MOZ_MEMORY_WINDOWS
    6178           19901 :         malloc_mutex_unlock(&init_lock);
    6179                 : #endif
    6180           19901 :         return (false);
    6181                 : }
    6182                 : 
    6183                 : /* XXX Why not just expose malloc_print_stats()? */
    6184                 : #ifdef MOZ_MEMORY_WINDOWS
    6185                 : void
    6186                 : malloc_shutdown()
    6187                 : {
    6188                 : 
    6189                 :         malloc_print_stats();
    6190                 : }
    6191                 : #endif
    6192                 : 
    6193                 : /*
    6194                 :  * End general internal functions.
    6195                 :  */
    6196                 : /******************************************************************************/
    6197                 : /*
    6198                 :  * Begin malloc(3)-compatible functions.
    6199                 :  */
    6200                 : 
    6201                 : /*
    6202                 :  * Mangle standard interfaces, in order to avoid linking problems.
    6203                 :  */
    6204                 : #if defined(MOZ_MEMORY_DARWIN) || defined(MOZ_MEMORY_WINDOWS) || \
    6205                 :     defined(MOZ_MEMORY_ANDROID)
    6206                 : 
    6207                 : #ifdef MOZ_MEMORY_ANDROID
    6208                 : /*
    6209                 :  * On Android, we use __wrap_* instead of je_* to accomodate with the
    6210                 :  * linker's --wrap option we use. That option prefixes the function
    6211                 :  * names it is given with __wrap_.
    6212                 :  */
    6213                 : #define wrap(a) __wrap_ ## a
    6214                 : 
    6215                 : /* Extra wrappers for NSPR alloc functions */
    6216                 : void *
    6217                 : __wrap_PR_Malloc(size_t size) __attribute__((alias("__wrap_malloc")));
    6218                 : void *
    6219                 : __wrap_PR_Calloc(size_t num, size_t size) __attribute__((alias("__wrap_calloc")));
    6220                 : void *
    6221                 : __wrap_PR_Realloc(void *ptr, size_t size) __attribute__((alias("__wrap_realloc")));
    6222                 : void
    6223                 : __wrap_PR_Free(void *ptr) __attribute__((alias("__wrap_free")));
    6224                 : 
    6225                 : #else
    6226                 : #define wrap(a) je_ ## a
    6227                 : #endif
    6228                 : 
    6229                 : #define malloc(a)               wrap(malloc)(a)
    6230                 : #define memalign(a, b)          wrap(memalign)(a, b)
    6231                 : #define posix_memalign(a, b, c) wrap(posix_memalign)(a, b, c)
    6232                 : #define valloc(a)               wrap(valloc)(a)
    6233                 : #define calloc(a, b)            wrap(calloc)(a, b)
    6234                 : #define realloc(a, b)           wrap(realloc)(a, b)
    6235                 : #define free(a)                 wrap(free)(a)
    6236                 : #define malloc_usable_size(a)   wrap(malloc_usable_size)(a)
    6237                 : 
    6238                 : void *malloc(size_t size);
    6239                 : 
    6240                 : char *
    6241                 : wrap(strndup)(const char *src, size_t len) {
    6242                 :         char* dst = (char*) malloc(len + 1);
    6243                 :         if (dst)
    6244                 :                 strncpy(dst, src, len + 1);
    6245                 :         return dst;
    6246                 : }
    6247                 : 
    6248                 : char *
    6249                 : wrap(strdup)(const char *src) {
    6250                 :         size_t len = strlen(src);
    6251                 :         return wrap(strndup)(src, len);
    6252                 : }
    6253                 : #endif
    6254                 : 
    6255                 : /*
    6256                 :  * Even though we compile with MOZ_MEMORY, we may have to dynamically decide
    6257                 :  * not to use jemalloc, as discussed above. However, we call jemalloc
    6258                 :  * functions directly from mozalloc. Since it's pretty dangerous to mix the
    6259                 :  * allocators, we need to call the OSX allocators from the functions below,
    6260                 :  * when osx_use_jemalloc is not (dynamically) set.
    6261                 :  *
    6262                 :  * Note that we assume jemalloc is enabled on i386.  This is safe because the
    6263                 :  * only i386 versions of MacOS are 10.5 and 10.6, which we support.  We have to
    6264                 :  * do this because madvise isn't in the malloc zone struct for 10.5.
    6265                 :  *
    6266                 :  * This means that NO_MAC_JEMALLOC doesn't work on i386.
    6267                 :  */
    6268                 : #if defined(MOZ_MEMORY_DARWIN) && !defined(__i386__)
    6269                 : #define DARWIN_ONLY(A) if (!osx_use_jemalloc) { A; }
    6270                 : #else
    6271                 : #define DARWIN_ONLY(A)
    6272                 : #endif
    6273                 : 
    6274                 : void *
    6275                 : malloc(size_t size)
    6276                 : {
    6277                 :         void *ret;
    6278                 : 
    6279                 :         DARWIN_ONLY(return (szone->malloc)(szone, size));
    6280                 : 
    6281       108483841 :         if (malloc_init()) {
    6282               0 :                 ret = NULL;
    6283               0 :                 goto RETURN;
    6284                 :         }
    6285                 : 
    6286       108483676 :         if (size == 0) {
    6287                 : #ifdef MALLOC_SYSV
    6288              72 :                 if (opt_sysv == false)
    6289                 : #endif
    6290              72 :                         size = 1;
    6291                 : #ifdef MALLOC_SYSV
    6292                 :                 else {
    6293               0 :                         ret = NULL;
    6294               0 :                         goto RETURN;
    6295                 :                 }
    6296                 : #endif
    6297                 :         }
    6298                 : 
    6299       108483676 :         ret = imalloc(size);
    6300                 : 
    6301                 : RETURN:
    6302       108484005 :         if (ret == NULL) {
    6303                 : #ifdef MALLOC_XMALLOC
    6304               1 :                 if (opt_xmalloc) {
    6305               0 :                         _malloc_message(_getprogname(),
    6306                 :                             ": (malloc) Error in malloc(): out of memory\n", "",
    6307                 :                             "");
    6308               0 :                         abort();
    6309                 :                 }
    6310                 : #endif
    6311               1 :                 errno = ENOMEM;
    6312                 :         }
    6313                 : 
    6314       108484005 :         UTRACE(0, size, ret);
    6315       108484005 :         return (ret);
    6316                 : }
    6317                 : 
    6318                 : /*
    6319                 :  * In ELF systems the default visibility allows symbols to be preempted at
    6320                 :  * runtime. This in turn prevents the uses of memalign in this file from being
    6321                 :  * optimized. What we do in here is define two aliasing symbols (they point to
    6322                 :  * the same code): memalign and memalign_internal. The internal version has
    6323                 :  * hidden visibility and is used in every reference from this file.
    6324                 :  *
    6325                 :  * For more information on this technique, see section 2.2.7 (Avoid Using
    6326                 :  * Exported Symbols) in http://www.akkadia.org/drepper/dsohowto.pdf.
    6327                 :  */
    6328                 : 
    6329                 : #if defined(__GNUC__) && !defined(MOZ_MEMORY_DARWIN)
    6330                 : #define MOZ_MEMORY_ELF
    6331                 : #endif
    6332                 : 
    6333                 : #ifdef MOZ_MEMORY_SOLARIS
    6334                 : #  ifdef __SUNPRO_C
    6335                 : void *
    6336                 : memalign(size_t alignment, size_t size);
    6337                 : #pragma no_inline(memalign)
    6338                 : #  elif (defined(__GNUC__))
    6339                 : __attribute__((noinline))
    6340                 : #  endif
    6341                 : #else
    6342                 : #if (defined(MOZ_MEMORY_ELF))
    6343                 : __attribute__((visibility ("hidden")))
    6344                 : #endif
    6345                 : #endif
    6346                 : 
    6347                 : 
    6348                 : #ifdef MOZ_MEMORY_ELF
    6349                 : #define MEMALIGN memalign_internal
    6350                 : #else
    6351                 : #define MEMALIGN memalign
    6352                 : #endif
    6353                 : 
    6354                 : void *
    6355           30860 : MEMALIGN(size_t alignment, size_t size)
    6356                 : {
    6357                 :         void *ret;
    6358                 : 
    6359                 :         DARWIN_ONLY(return (szone->memalign)(szone, alignment, size));
    6360                 : 
    6361           30860 :         assert(((alignment - 1) & alignment) == 0);
    6362                 : 
    6363           30860 :         if (malloc_init()) {
    6364               0 :                 ret = NULL;
    6365               0 :                 goto RETURN;
    6366                 :         }
    6367                 : 
    6368           30860 :         if (size == 0) {
    6369                 : #ifdef MALLOC_SYSV
    6370               0 :                 if (opt_sysv == false)
    6371                 : #endif
    6372               0 :                         size = 1;
    6373                 : #ifdef MALLOC_SYSV
    6374                 :                 else {
    6375               0 :                         ret = NULL;
    6376               0 :                         goto RETURN;
    6377                 :                 }
    6378                 : #endif
    6379                 :         }
    6380                 : 
    6381           30860 :         alignment = alignment < sizeof(void*) ? sizeof(void*) : alignment;
    6382           30860 :         ret = ipalloc(alignment, size);
    6383                 : 
    6384                 : RETURN:
    6385                 : #ifdef MALLOC_XMALLOC
    6386           30860 :         if (opt_xmalloc && ret == NULL) {
    6387               0 :                 _malloc_message(_getprogname(),
    6388                 :                 ": (malloc) Error in memalign(): out of memory\n", "", "");
    6389               0 :                 abort();
    6390                 :         }
    6391                 : #endif
    6392           30860 :         UTRACE(0, size, ret);
    6393           30860 :         return (ret);
    6394                 : }
    6395                 : 
    6396                 : #ifdef MOZ_MEMORY_ELF
    6397                 : extern void *
    6398                 : memalign(size_t alignment, size_t size) __attribute__((alias ("memalign_internal"), visibility ("default")));
    6399                 : #endif
    6400                 : 
    6401                 : int
    6402                 : posix_memalign(void **memptr, size_t alignment, size_t size)
    6403                 : {
    6404                 :         void *result;
    6405                 : 
    6406                 :         /* Make sure that alignment is a large enough power of 2. */
    6407           30837 :         if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
    6408                 : #ifdef MALLOC_XMALLOC
    6409               0 :                 if (opt_xmalloc) {
    6410               0 :                         _malloc_message(_getprogname(),
    6411                 :                             ": (malloc) Error in posix_memalign(): "
    6412                 :                             "invalid alignment\n", "", "");
    6413               0 :                         abort();
    6414                 :                 }
    6415                 : #endif
    6416               0 :                 return (EINVAL);
    6417                 :         }
    6418                 : 
    6419                 :         /* The 0-->1 size promotion is done in the memalign() call below */
    6420                 : 
    6421           30837 :         result = MEMALIGN(alignment, size);
    6422                 : 
    6423           30837 :         if (result == NULL)
    6424               0 :                 return (ENOMEM);
    6425                 : 
    6426           30837 :         *memptr = result;
    6427           30837 :         return (0);
    6428                 : }
    6429                 : 
    6430                 : void *
    6431                 : valloc(size_t size)
    6432                 : {
    6433               0 :         return (MEMALIGN(pagesize, size));
    6434                 : }
    6435                 : 
    6436                 : void *
    6437                 : calloc(size_t num, size_t size)
    6438                 : {
    6439                 :         void *ret;
    6440                 :         size_t num_size;
    6441                 : 
    6442                 :         DARWIN_ONLY(return (szone->calloc)(szone, num, size));
    6443                 : 
    6444         7986415 :         if (malloc_init()) {
    6445               0 :                 num_size = 0;
    6446               0 :                 ret = NULL;
    6447               0 :                 goto RETURN;
    6448                 :         }
    6449                 : 
    6450         7986414 :         num_size = num * size;
    6451         7986414 :         if (num_size == 0) {
    6452                 : #ifdef MALLOC_SYSV
    6453               0 :                 if ((opt_sysv == false) && ((num == 0) || (size == 0)))
    6454                 : #endif
    6455               0 :                         num_size = 1;
    6456                 : #ifdef MALLOC_SYSV
    6457                 :                 else {
    6458               0 :                         ret = NULL;
    6459               0 :                         goto RETURN;
    6460                 :                 }
    6461                 : #endif
    6462                 :         /*
    6463                 :          * Try to avoid division here.  We know that it isn't possible to
    6464                 :          * overflow during multiplication if neither operand uses any of the
    6465                 :          * most significant half of the bits in a size_t.
    6466                 :          */
    6467         7986414 :         } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
    6468             963 :             && (num_size / size != num)) {
    6469                 :                 /* size_t overflow. */
    6470               0 :                 ret = NULL;
    6471               0 :                 goto RETURN;
    6472                 :         }
    6473                 : 
    6474         7986414 :         ret = icalloc(num_size);
    6475                 : 
    6476                 : RETURN:
    6477         7986415 :         if (ret == NULL) {
    6478                 : #ifdef MALLOC_XMALLOC
    6479              18 :                 if (opt_xmalloc) {
    6480               0 :                         _malloc_message(_getprogname(),
    6481                 :                             ": (malloc) Error in calloc(): out of memory\n", "",
    6482                 :                             "");
    6483               0 :                         abort();
    6484                 :                 }
    6485                 : #endif
    6486              18 :                 errno = ENOMEM;
    6487                 :         }
    6488                 : 
    6489         7986415 :         UTRACE(0, num_size, ret);
    6490         7986415 :         return (ret);
    6491                 : }
    6492                 : 
    6493                 : void *
    6494                 : realloc(void *ptr, size_t size)
    6495                 : {
    6496                 :         void *ret;
    6497                 : 
    6498                 :         DARWIN_ONLY(return (szone->realloc)(szone, ptr, size));
    6499                 : 
    6500         9582979 :         if (size == 0) {
    6501                 : #ifdef MALLOC_SYSV
    6502               0 :                 if (opt_sysv == false)
    6503                 : #endif
    6504               0 :                         size = 1;
    6505                 : #ifdef MALLOC_SYSV
    6506                 :                 else {
    6507               0 :                         if (ptr != NULL)
    6508               0 :                                 idalloc(ptr);
    6509               0 :                         ret = NULL;
    6510               0 :                         goto RETURN;
    6511                 :                 }
    6512                 : #endif
    6513                 :         }
    6514                 : 
    6515         9582979 :         if (ptr != NULL) {
    6516         4723565 :                 assert(malloc_initialized);
    6517                 : 
    6518         4723565 :                 ret = iralloc(ptr, size);
    6519                 : 
    6520         4723565 :                 if (ret == NULL) {
    6521                 : #ifdef MALLOC_XMALLOC
    6522               0 :                         if (opt_xmalloc) {
    6523               0 :                                 _malloc_message(_getprogname(),
    6524                 :                                     ": (malloc) Error in realloc(): out of "
    6525                 :                                     "memory\n", "", "");
    6526               0 :                                 abort();
    6527                 :                         }
    6528                 : #endif
    6529               0 :                         errno = ENOMEM;
    6530                 :                 }
    6531                 :         } else {
    6532         4859414 :                 if (malloc_init())
    6533               0 :                         ret = NULL;
    6534                 :                 else
    6535         4859414 :                         ret = imalloc(size);
    6536                 : 
    6537         4859414 :                 if (ret == NULL) {
    6538                 : #ifdef MALLOC_XMALLOC
    6539               0 :                         if (opt_xmalloc) {
    6540               0 :                                 _malloc_message(_getprogname(),
    6541                 :                                     ": (malloc) Error in realloc(): out of "
    6542                 :                                     "memory\n", "", "");
    6543               0 :                                 abort();
    6544                 :                         }
    6545                 : #endif
    6546               0 :                         errno = ENOMEM;
    6547                 :                 }
    6548                 :         }
    6549                 : 
    6550                 : #ifdef MALLOC_SYSV
    6551                 : RETURN:
    6552                 : #endif
    6553         9582979 :         UTRACE(ptr, size, ret);
    6554         9582979 :         return (ret);
    6555                 : }
    6556                 : 
    6557                 : void
    6558                 : free(void *ptr)
    6559                 : {
    6560                 :         size_t offset;
    6561                 :         
    6562                 :         DARWIN_ONLY((szone->free)(szone, ptr); return);
    6563                 : 
    6564       128640171 :         UTRACE(ptr, 0, 0);
    6565                 : 
    6566                 :         /*
    6567                 :          * A version of idalloc that checks for NULL pointer but only for
    6568                 :          * huge allocations assuming that CHUNK_ADDR2OFFSET(NULL) == 0.
    6569                 :          */
    6570                 :         assert(CHUNK_ADDR2OFFSET(NULL) == 0);
    6571       128640889 :         offset = CHUNK_ADDR2OFFSET(ptr);
    6572       128640889 :         if (offset != 0)
    6573       116879861 :                 arena_dalloc(ptr, offset);
    6574        11761028 :         else if (ptr != NULL)
    6575            3795 :                 huge_dalloc(ptr);
    6576       128658207 : }
    6577                 : 
    6578                 : /*
    6579                 :  * End malloc(3)-compatible functions.
    6580                 :  */
    6581                 : /******************************************************************************/
    6582                 : /*
    6583                 :  * Begin non-standard functions.
    6584                 :  */
    6585                 : 
    6586                 : /* This was added by Mozilla for use by SQLite. */
    6587                 : size_t
    6588         7134088 : je_malloc_usable_size_in_advance(size_t size)
    6589                 : {
    6590                 :         /*
    6591                 :          * This duplicates the logic in imalloc(), arena_malloc() and
    6592                 :          * arena_malloc_small().
    6593                 :          */
    6594         7134088 :         if (size < small_min) {
    6595                 :                 /* Small (tiny). */
    6596          544582 :                 size = pow2_ceil(size);
    6597                 :                 /*
    6598                 :                  * We omit the #ifdefs from arena_malloc_small() --
    6599                 :                  * it can be inaccurate with its size in some cases, but this
    6600                 :                  * function must be accurate.
    6601                 :                  */
    6602          544582 :                 if (size < (1U << TINY_MIN_2POW))
    6603           34768 :                         size = (1U << TINY_MIN_2POW);
    6604         6589506 :         } else if (size <= small_max) {
    6605                 :                 /* Small (quantum-spaced). */
    6606         4271051 :                 size = QUANTUM_CEILING(size);
    6607         2318455 :         } else if (size <= bin_maxclass) {
    6608                 :                 /* Small (sub-page). */
    6609         2190858 :                 size = pow2_ceil(size);
    6610          127597 :         } else if (size <= arena_maxclass) {
    6611                 :                 /* Large. */
    6612          127428 :                 size = PAGE_CEILING(size);
    6613                 :         } else {
    6614                 :                 /*
    6615                 :                  * Huge.  We use PAGE_CEILING to get psize, instead of using
    6616                 :                  * CHUNK_CEILING to get csize.  This ensures that this
    6617                 :                  * malloc_usable_size(malloc(n)) always matches
    6618                 :                  * je_malloc_usable_size_in_advance(n).
    6619                 :                  */
    6620             169 :                 size = PAGE_CEILING(size);
    6621                 :         }
    6622         7134088 :         return size;
    6623                 : }
    6624                 : 
    6625                 : 
    6626                 : #ifdef MOZ_MEMORY_ANDROID
    6627                 : size_t
    6628                 : malloc_usable_size(void *ptr)
    6629                 : #else
    6630                 : size_t
    6631        14218181 : malloc_usable_size(const void *ptr)
    6632                 : #endif
    6633                 : {
    6634                 :         DARWIN_ONLY(return (szone->size)(szone, ptr));
    6635                 : 
    6636                 : #ifdef MALLOC_VALIDATE
    6637        14218181 :         return (isalloc_validate(ptr));
    6638                 : #else
    6639                 :         assert(ptr != NULL);
    6640                 : 
    6641                 :         return (isalloc(ptr));
    6642                 : #endif
    6643                 : }
    6644                 : 
    6645                 : void
    6646               6 : jemalloc_stats(jemalloc_stats_t *stats)
    6647                 : {
    6648                 :         size_t i;
    6649                 : 
    6650               6 :         assert(stats != NULL);
    6651                 : 
    6652                 :         /*
    6653                 :          * Gather runtime settings.
    6654                 :          */
    6655               6 :         stats->opt_abort = opt_abort;
    6656               6 :         stats->opt_junk =
    6657                 : #ifdef MALLOC_FILL
    6658                 :             opt_junk ? true :
    6659                 : #endif
    6660                 :             false;
    6661               6 :         stats->opt_utrace =
    6662                 : #ifdef MALLOC_UTRACE
    6663                 :             opt_utrace ? true :
    6664                 : #endif
    6665                 :             false;
    6666               6 :         stats->opt_sysv =
    6667                 : #ifdef MALLOC_SYSV
    6668                 :             opt_sysv ? true :
    6669                 : #endif
    6670                 :             false;
    6671               6 :         stats->opt_xmalloc =
    6672                 : #ifdef MALLOC_XMALLOC
    6673                 :             opt_xmalloc ? true :
    6674                 : #endif
    6675                 :             false;
    6676               6 :         stats->opt_zero =
    6677                 : #ifdef MALLOC_FILL
    6678                 :             opt_zero ? true :
    6679                 : #endif
    6680                 :             false;
    6681               6 :         stats->narenas = narenas;
    6682               6 :         stats->balance_threshold =
    6683                 : #ifdef MALLOC_BALANCE
    6684                 :             opt_balance_threshold
    6685                 : #else
    6686                 :             SIZE_T_MAX
    6687                 : #endif
    6688                 :             ;
    6689               6 :         stats->quantum = quantum;
    6690               6 :         stats->small_max = small_max;
    6691               6 :         stats->large_max = arena_maxclass;
    6692               6 :         stats->chunksize = chunksize;
    6693               6 :         stats->dirty_max = opt_dirty_max;
    6694                 : 
    6695                 :         /*
    6696                 :          * Gather current memory usage statistics.
    6697                 :          */
    6698               6 :         stats->mapped = 0;
    6699               6 :         stats->committed = 0;
    6700               6 :         stats->allocated = 0;
    6701               6 :         stats->dirty = 0;
    6702                 : 
    6703                 :         /* Get huge mapped/allocated. */
    6704               6 :         malloc_mutex_lock(&huge_mtx);
    6705               6 :         stats->mapped += stats_chunks.curchunks * chunksize;
    6706               6 :         stats->committed += huge_allocated;
    6707               6 :         stats->allocated += huge_allocated;
    6708               6 :         malloc_mutex_unlock(&huge_mtx);
    6709                 : 
    6710                 :         /* Get base mapped. */
    6711               6 :         malloc_mutex_lock(&base_mtx);
    6712               6 :         stats->mapped += base_mapped;
    6713               6 :         assert(base_committed <= base_mapped);
    6714               6 :         stats->committed += base_committed;
    6715               6 :         malloc_mutex_unlock(&base_mtx);
    6716                 : 
    6717                 :         /* Iterate over arenas and their chunks. */
    6718              12 :         for (i = 0; i < narenas; i++) {
    6719               6 :                 arena_t *arena = arenas[i];
    6720               6 :                 if (arena != NULL) {
    6721               6 :                         malloc_spin_lock(&arena->lock);
    6722               6 :                         stats->allocated += arena->stats.allocated_small;
    6723               6 :                         stats->allocated += arena->stats.allocated_large;
    6724               6 :                         stats->committed += (arena->stats.committed <<
    6725                 :                             pagesize_2pow);
    6726               6 :                         stats->dirty += (arena->ndirty << pagesize_2pow);
    6727               6 :                         malloc_spin_unlock(&arena->lock);
    6728                 :                 }
    6729                 :         }
    6730                 : 
    6731               6 :         assert(stats->mapped >= stats->committed);
    6732               6 :         assert(stats->committed >= stats->allocated);
    6733               6 : }
    6734                 : 
    6735                 : #ifdef MALLOC_DOUBLE_PURGE
    6736                 : 
    6737                 : /* Explicitly remove all of this chunk's MADV_FREE'd pages from memory. */
    6738                 : static void
    6739                 : hard_purge_chunk(arena_chunk_t *chunk)
    6740                 : {
    6741                 :         /* See similar logic in arena_purge(). */
    6742                 : 
    6743                 :         size_t i;
    6744                 :         for (i = arena_chunk_header_npages; i < chunk_npages; i++) {
    6745                 :                 /* Find all adjacent pages with CHUNK_MAP_MADVISED set. */
    6746                 :                 size_t npages;
    6747                 :                 for (npages = 0;
    6748                 :                      chunk->map[i + npages].bits & CHUNK_MAP_MADVISED && i + npages < chunk_npages;
    6749                 :                      npages++) {
    6750                 :                         /* Turn off the chunk's MADV_FREED bit and turn on its
    6751                 :                          * DECOMMITTED bit. */
    6752                 :                         assert(!(chunk->map[i + npages].bits & CHUNK_MAP_DECOMMITTED));
    6753                 :                         chunk->map[i + npages].bits ^= CHUNK_MAP_MADVISED_OR_DECOMMITTED;
    6754                 :                 }
    6755                 : 
    6756                 :                 /* We could use mincore to find out which pages are actually
    6757                 :                  * present, but it's not clear that's better. */
    6758                 :                 if (npages > 0) {
    6759                 :                         pages_decommit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
    6760                 :                         pages_commit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
    6761                 :                 }
    6762                 :                 i += npages;
    6763                 :         }
    6764                 : }
    6765                 : 
    6766                 : /* Explicitly remove all of this arena's MADV_FREE'd pages from memory. */
    6767                 : static void
    6768                 : hard_purge_arena(arena_t *arena)
    6769                 : {
    6770                 :         malloc_spin_lock(&arena->lock);
    6771                 : 
    6772                 :         while (!LinkedList_IsEmpty(&arena->chunks_madvised)) {
    6773                 :                 LinkedList* next = arena->chunks_madvised.next;
    6774                 :                 arena_chunk_t *chunk =
    6775                 :                         LinkedList_Get(arena->chunks_madvised.next,
    6776                 :                                        arena_chunk_t, chunks_madvised_elem);
    6777                 :                 hard_purge_chunk(chunk);
    6778                 :                 LinkedList_Remove(&chunk->chunks_madvised_elem);
    6779                 :         }
    6780                 : 
    6781                 :         malloc_spin_unlock(&arena->lock);
    6782                 : }
    6783                 : 
    6784                 : void
    6785                 : jemalloc_purge_freed_pages()
    6786                 : {
    6787                 :         size_t i;
    6788                 :         for (i = 0; i < narenas; i++) {
    6789                 :                 arena_t *arena = arenas[i];
    6790                 :                 if (arena != NULL)
    6791                 :                         hard_purge_arena(arena);
    6792                 :         }
    6793                 : }
    6794                 : 
    6795                 : #else /* !defined MALLOC_DOUBLE_PURGE */
    6796                 : 
    6797                 : void
    6798               0 : jemalloc_purge_freed_pages()
    6799                 : {
    6800                 :         /* Do nothing. */
    6801               0 : }
    6802                 : 
    6803                 : #endif /* defined MALLOC_DOUBLE_PURGE */
    6804                 : 
    6805                 : 
    6806                 : 
    6807                 : #ifdef MOZ_MEMORY_WINDOWS
    6808                 : void*
    6809                 : _recalloc(void *ptr, size_t count, size_t size)
    6810                 : {
    6811                 :         size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
    6812                 :         size_t newsize = count * size;
    6813                 : 
    6814                 :         /*
    6815                 :          * In order for all trailing bytes to be zeroed, the caller needs to
    6816                 :          * use calloc(), followed by recalloc().  However, the current calloc()
    6817                 :          * implementation only zeros the bytes requested, so if recalloc() is
    6818                 :          * to work 100% correctly, calloc() will need to change to zero
    6819                 :          * trailing bytes.
    6820                 :          */
    6821                 : 
    6822                 :         ptr = realloc(ptr, newsize);
    6823                 :         if (ptr != NULL && oldsize < newsize) {
    6824                 :                 memset((void *)((uintptr_t)ptr + oldsize), 0, newsize -
    6825                 :                     oldsize);
    6826                 :         }
    6827                 : 
    6828                 :         return ptr;
    6829                 : }
    6830                 : 
    6831                 : /*
    6832                 :  * This impl of _expand doesn't ever actually expand or shrink blocks: it
    6833                 :  * simply replies that you may continue using a shrunk block.
    6834                 :  */
    6835                 : void*
    6836                 : _expand(void *ptr, size_t newsize)
    6837                 : {
    6838                 :         if (isalloc(ptr) >= newsize)
    6839                 :                 return ptr;
    6840                 : 
    6841                 :         return NULL;
    6842                 : }
    6843                 : 
    6844                 : size_t
    6845                 : _msize(const void *ptr)
    6846                 : {
    6847                 : 
    6848                 :         return malloc_usable_size(ptr);
    6849                 : }
    6850                 : #endif
    6851                 : 
    6852                 : /*
    6853                 :  * End non-standard functions.
    6854                 :  */
    6855                 : /******************************************************************************/
    6856                 : /*
    6857                 :  * Begin library-private functions, used by threading libraries for protection
    6858                 :  * of malloc during fork().  These functions are only called if the program is
    6859                 :  * running in threaded mode, so there is no need to check whether the program
    6860                 :  * is threaded here.
    6861                 :  */
    6862                 : 
    6863                 : #ifdef MOZ_MEMORY_ANDROID
    6864                 : void
    6865                 : #else
    6866                 : static void
    6867                 : #endif
    6868            1097 : _malloc_prefork(void)
    6869                 : {
    6870                 :         unsigned i;
    6871                 : 
    6872                 :         /* Acquire all mutexes in a safe order. */
    6873                 : 
    6874            1097 :         malloc_spin_lock(&arenas_lock);
    6875            2194 :         for (i = 0; i < narenas; i++) {
    6876            1097 :                 if (arenas[i] != NULL)
    6877            1097 :                         malloc_spin_lock(&arenas[i]->lock);
    6878                 :         }
    6879                 : 
    6880            1097 :         malloc_mutex_lock(&base_mtx);
    6881                 : 
    6882            1097 :         malloc_mutex_lock(&huge_mtx);
    6883            1097 : }
    6884                 : 
    6885                 : #ifdef MOZ_MEMORY_ANDROID
    6886                 : void
    6887                 : #else
    6888                 : static void
    6889                 : #endif
    6890            1097 : _malloc_postfork(void)
    6891                 : {
    6892                 :         unsigned i;
    6893                 : 
    6894                 :         /* Release all mutexes, now that fork() has completed. */
    6895                 : 
    6896            1097 :         malloc_mutex_unlock(&huge_mtx);
    6897                 : 
    6898            1097 :         malloc_mutex_unlock(&base_mtx);
    6899                 : 
    6900            2194 :         for (i = 0; i < narenas; i++) {
    6901            1097 :                 if (arenas[i] != NULL)
    6902            1097 :                         malloc_spin_unlock(&arenas[i]->lock);
    6903                 :         }
    6904            1097 :         malloc_spin_unlock(&arenas_lock);
    6905            1097 : }
    6906                 : 
    6907                 : /*
    6908                 :  * End library-private functions.
    6909                 :  */
    6910                 : /******************************************************************************/
    6911                 : 
    6912                 : #ifdef HAVE_DLOPEN
    6913                 : #  include <dlfcn.h>
    6914                 : #endif
    6915                 : 
    6916                 : #ifdef MOZ_MEMORY_DARWIN
    6917                 : 
    6918                 : static void *
    6919                 : zone_malloc(malloc_zone_t *zone, size_t size)
    6920                 : {
    6921                 : 
    6922                 :         return (malloc(size));
    6923                 : }
    6924                 : 
    6925                 : static void *
    6926                 : zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
    6927                 : {
    6928                 : 
    6929                 :         return (calloc(num, size));
    6930                 : }
    6931                 : 
    6932                 : static void *
    6933                 : zone_valloc(malloc_zone_t *zone, size_t size)
    6934                 : {
    6935                 :         void *ret = NULL; /* Assignment avoids useless compiler warning. */
    6936                 : 
    6937                 :         posix_memalign(&ret, pagesize, size);
    6938                 : 
    6939                 :         return (ret);
    6940                 : }
    6941                 : 
    6942                 : static void *
    6943                 : zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
    6944                 : {
    6945                 :         return (memalign(alignment, size));
    6946                 : }
    6947                 : 
    6948                 : static void *
    6949                 : zone_destroy(malloc_zone_t *zone)
    6950                 : {
    6951                 : 
    6952                 :         /* This function should never be called. */
    6953                 :         assert(false);
    6954                 :         return (NULL);
    6955                 : }
    6956                 : 
    6957                 : static size_t
    6958                 : zone_good_size(malloc_zone_t *zone, size_t size)
    6959                 : {
    6960                 :         return je_malloc_usable_size_in_advance(size);
    6961                 : }
    6962                 : 
    6963                 : static size_t
    6964                 : ozone_size(malloc_zone_t *zone, void *ptr)
    6965                 : {
    6966                 :         size_t ret = isalloc_validate(ptr);
    6967                 :         if (ret == 0)
    6968                 :                 ret = szone->size(zone, ptr);
    6969                 : 
    6970                 :         return ret;
    6971                 : }
    6972                 : 
    6973                 : static void
    6974                 : ozone_free(malloc_zone_t *zone, void *ptr)
    6975                 : {
    6976                 :         if (isalloc_validate(ptr) != 0)
    6977                 :                 free(ptr);
    6978                 :         else {
    6979                 :                 size_t size = szone->size(zone, ptr);
    6980                 :                 if (size != 0)
    6981                 :                         (szone->free)(zone, ptr);
    6982                 :                 /* Otherwise we leak. */
    6983                 :         }
    6984                 : }
    6985                 : 
    6986                 : static void *
    6987                 : ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
    6988                 : {
    6989                 :     size_t oldsize;
    6990                 :         if (ptr == NULL)
    6991                 :                 return (malloc(size));
    6992                 : 
    6993                 :         oldsize = isalloc_validate(ptr);
    6994                 :         if (oldsize != 0)
    6995                 :                 return (realloc(ptr, size));
    6996                 :         else {
    6997                 :                 oldsize = szone->size(zone, ptr);
    6998                 :                 if (oldsize == 0)
    6999                 :                         return (malloc(size));
    7000                 :                 else {
    7001                 :                         void *ret = malloc(size);
    7002                 :                         if (ret != NULL) {
    7003                 :                                 memcpy(ret, ptr, (oldsize < size) ? oldsize :
    7004                 :                                     size);
    7005                 :                                 (szone->free)(zone, ptr);
    7006                 :                         }
    7007                 :                         return (ret);
    7008                 :                 }
    7009                 :         }
    7010                 : }
    7011                 : 
    7012                 : static unsigned
    7013                 : ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
    7014                 :     unsigned num_requested)
    7015                 : {
    7016                 :         /* Don't bother implementing this interface, since it isn't required. */
    7017                 :         return 0;
    7018                 : }
    7019                 : 
    7020                 : static void
    7021                 : ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
    7022                 : {
    7023                 :         unsigned i;
    7024                 : 
    7025                 :         for (i = 0; i < num; i++)
    7026                 :                 ozone_free(zone, to_be_freed[i]);
    7027                 : }
    7028                 : 
    7029                 : static void
    7030                 : ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
    7031                 : {
    7032                 :         if (isalloc_validate(ptr) != 0) {
    7033                 :                 assert(isalloc_validate(ptr) == size);
    7034                 :                 free(ptr);
    7035                 :         } else {
    7036                 :                 assert(size == szone->size(zone, ptr));
    7037                 :                 l_szone.m16(zone, ptr, size);
    7038                 :         }
    7039                 : }
    7040                 : 
    7041                 : static void
    7042                 : ozone_force_lock(malloc_zone_t *zone)
    7043                 : {
    7044                 :         /* jemalloc locking is taken care of by the normal jemalloc zone. */
    7045                 :         szone->introspect->force_lock(zone);
    7046                 : }
    7047                 : 
    7048                 : static void
    7049                 : ozone_force_unlock(malloc_zone_t *zone)
    7050                 : {
    7051                 :         /* jemalloc locking is taken care of by the normal jemalloc zone. */
    7052                 :         szone->introspect->force_unlock(zone);
    7053                 : }
    7054                 : 
    7055                 : static size_t
    7056                 : zone_version_size(int version)
    7057                 : {
    7058                 :     switch (version)
    7059                 :     {
    7060                 :         case SNOW_LEOPARD_MALLOC_ZONE_T_VERSION:
    7061                 :             return sizeof(snow_leopard_malloc_zone);
    7062                 :         case LEOPARD_MALLOC_ZONE_T_VERSION:
    7063                 :             return sizeof(leopard_malloc_zone);
    7064                 :         default:
    7065                 :         case LION_MALLOC_ZONE_T_VERSION:
    7066                 :             return sizeof(lion_malloc_zone);
    7067                 :     }
    7068                 : }
    7069                 : 
    7070                 : /*
    7071                 :  * Overlay the default scalable zone (szone) such that existing allocations are
    7072                 :  * drained, and further allocations come from jemalloc. This is necessary
    7073                 :  * because Core Foundation directly accesses and uses the szone before the
    7074                 :  * jemalloc library is even loaded.
    7075                 :  */
    7076                 : static void
    7077                 : szone2ozone(malloc_zone_t *default_zone, size_t size)
    7078                 : {
    7079                 :     lion_malloc_zone *l_zone;
    7080                 :         assert(malloc_initialized);
    7081                 : 
    7082                 :         /*
    7083                 :          * Stash a copy of the original szone so that we can call its
    7084                 :          * functions as needed. Note that internally, the szone stores its
    7085                 :          * bookkeeping data structures immediately following the malloc_zone_t
    7086                 :          * header, so when calling szone functions, we need to pass a pointer to
    7087                 :          * the original zone structure.
    7088                 :          */
    7089                 :         memcpy(szone, default_zone, size);
    7090                 : 
    7091                 :         /* OSX 10.7 allocates the default zone in protected memory. */
    7092                 :         if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
    7093                 :                 void* start_of_page = (void*)((size_t)(default_zone) & ~pagesize_mask);
    7094                 :                 mprotect (start_of_page, size, PROT_READ | PROT_WRITE);
    7095                 :         }
    7096                 : 
    7097                 :         default_zone->size = (void *)ozone_size;
    7098                 :         default_zone->malloc = (void *)zone_malloc;
    7099                 :         default_zone->calloc = (void *)zone_calloc;
    7100                 :         default_zone->valloc = (void *)zone_valloc;
    7101                 :         default_zone->free = (void *)ozone_free;
    7102                 :         default_zone->realloc = (void *)ozone_realloc;
    7103                 :         default_zone->destroy = (void *)zone_destroy;
    7104                 :         default_zone->batch_malloc = NULL;
    7105                 :         default_zone->batch_free = ozone_batch_free;
    7106                 :         default_zone->introspect = ozone_introspect;
    7107                 : 
    7108                 :         /* Don't modify default_zone->zone_name; Mac libc may rely on the name
    7109                 :          * being unchanged.  See Mozilla bug 694896. */
    7110                 : 
    7111                 :         ozone_introspect->enumerator = NULL;
    7112                 :         ozone_introspect->good_size = (void *)zone_good_size;
    7113                 :         ozone_introspect->check = NULL;
    7114                 :         ozone_introspect->print = NULL;
    7115                 :         ozone_introspect->log = NULL;
    7116                 :         ozone_introspect->force_lock = (void *)ozone_force_lock;
    7117                 :         ozone_introspect->force_unlock = (void *)ozone_force_unlock;
    7118                 :         ozone_introspect->statistics = NULL;
    7119                 : 
    7120                 :     /* Platform-dependent structs */
    7121                 :     l_zone = (lion_malloc_zone*)(default_zone);
    7122                 : 
    7123                 :     if (default_zone->version >= SNOW_LEOPARD_MALLOC_ZONE_T_VERSION) {
    7124                 :         l_zone->m15 = (void (*)())zone_memalign;
    7125                 :         l_zone->m16 = (void (*)())ozone_free_definite_size;
    7126                 :         l_ozone_introspect.m9 = NULL;
    7127                 :     }
    7128                 : 
    7129                 :     if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
    7130                 :         l_zone->m17 = NULL;
    7131                 :         l_ozone_introspect.m10 = NULL;
    7132                 :         l_ozone_introspect.m11 = NULL;
    7133                 :         l_ozone_introspect.m12 = NULL;
    7134                 :         l_ozone_introspect.m13 = NULL;
    7135                 :     }
    7136                 : }
    7137                 : 
    7138                 : __attribute__((constructor))
    7139                 : void
    7140                 : jemalloc_darwin_init(void)
    7141                 : {
    7142                 :         if (malloc_init_hard())
    7143                 :                 abort();
    7144                 : }
    7145                 : 
    7146                 : #elif defined(__GLIBC__) && !defined(__UCLIBC__)
    7147                 : /*
    7148                 :  * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
    7149                 :  * to inconsistently reference libc's malloc(3)-compatible functions
    7150                 :  * (bug 493541).
    7151                 :  *
    7152                 :  * These definitions interpose hooks in glibc.  The functions are actually
    7153                 :  * passed an extra argument for the caller return address, which will be
    7154                 :  * ignored.
    7155                 :  */
    7156                 : void (*__free_hook)(void *ptr) = free;
    7157                 : void *(*__malloc_hook)(size_t size) = malloc;
    7158                 : void *(*__realloc_hook)(void *ptr, size_t size) = realloc;
    7159                 : void *(*__memalign_hook)(size_t alignment, size_t size) = MEMALIGN;
    7160                 : 
    7161                 : #elif defined(RTLD_DEEPBIND)
    7162                 : /*
    7163                 :  * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
    7164                 :  * implementations permit similar inconsistencies?  Should STV_SINGLETON
    7165                 :  * visibility be used for interposition where available?
    7166                 :  */
    7167                 : #  error "Interposing malloc is unsafe on this system without libc malloc hooks."
    7168                 : #endif
    7169                 : 
    7170                 : #ifdef MOZ_MEMORY_WINDOWS
    7171                 : /*
    7172                 :  * In the new style jemalloc integration jemalloc is built as a separate
    7173                 :  * shared library.  Since we're no longer hooking into the CRT binary,
    7174                 :  * we need to initialize the heap at the first opportunity we get.
    7175                 :  * DLL_PROCESS_ATTACH in DllMain is that opportunity.
    7176                 :  */
    7177                 : BOOL APIENTRY DllMain(HINSTANCE hModule, 
    7178                 :                       DWORD reason, 
    7179                 :                       LPVOID lpReserved)
    7180                 : {
    7181                 :   switch (reason) {
    7182                 :     case DLL_PROCESS_ATTACH:
    7183                 :       /* Don't force the system to page DllMain back in every time
    7184                 :        * we create/destroy a thread */
    7185                 :       DisableThreadLibraryCalls(hModule);
    7186                 :       /* Initialize the heap */
    7187                 :       malloc_init_hard();
    7188                 :       break;
    7189                 :     
    7190                 :     case DLL_PROCESS_DETACH:
    7191                 :       break;
    7192                 : 
    7193                 :   }
    7194                 : 
    7195                 :   return TRUE;
    7196                 : }
    7197                 : 
    7198                 : /*
    7199                 :  *  There's a fun allocator mismatch in (at least) the VS 2010 CRT
    7200                 :  *  (see the giant comment in this directory's Makefile.in
    7201                 :  *  that gets redirected here to avoid a crash on shutdown.
    7202                 :  */
    7203                 : void
    7204                 : je_dumb_free_thunk(void *ptr)
    7205                 : {
    7206                 :   return; /* shutdown leaks that we don't care about */
    7207                 : }
    7208                 : 
    7209                 : #include <wchar.h>
    7210                 : 
    7211                 : /*
    7212                 :  *  We also need to provide our own impl of wcsdup so that we don't ask
    7213                 :  *  the CRT for memory from its heap (which will then be unfreeable).
    7214                 :  */
    7215                 : wchar_t *je_wcsdup(const wchar_t *src)
    7216                 : {
    7217                 :   size_t len = wcslen(src);
    7218                 :   wchar_t* dst = (wchar_t*)je_malloc((len + 1) * sizeof(wchar_t));
    7219                 :   if(dst)
    7220                 :     wcsncpy(dst, src, len + 1);
    7221                 :   return dst;
    7222                 : }
    7223                 : 
    7224                 : #endif

Generated by: LCOV version 1.7