1 : /* -*- Mode: C; tab-width: 8; c-basic-offset: 8; indent-tabs-mode: t -*- */
2 : /* vim:set softtabstop=8 shiftwidth=8 noet: */
3 : /*-
4 : * Copyright (C) 2006-2008 Jason Evans <jasone@FreeBSD.org>.
5 : * All rights reserved.
6 : *
7 : * Redistribution and use in source and binary forms, with or without
8 : * modification, are permitted provided that the following conditions
9 : * are met:
10 : * 1. Redistributions of source code must retain the above copyright
11 : * notice(s), this list of conditions and the following disclaimer as
12 : * the first lines of this file unmodified other than the possible
13 : * addition of one or more copyright notices.
14 : * 2. Redistributions in binary form must reproduce the above copyright
15 : * notice(s), this list of conditions and the following disclaimer in
16 : * the documentation and/or other materials provided with the
17 : * distribution.
18 : *
19 : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
20 : * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 : * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
23 : * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 : * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 : * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
26 : * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 : * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
28 : * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 : * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 : *
31 : *******************************************************************************
32 : *
33 : * This allocator implementation is designed to provide scalable performance
34 : * for multi-threaded programs on multi-processor systems. The following
35 : * features are included for this purpose:
36 : *
37 : * + Multiple arenas are used if there are multiple CPUs, which reduces lock
38 : * contention and cache sloshing.
39 : *
40 : * + Cache line sharing between arenas is avoided for internal data
41 : * structures.
42 : *
43 : * + Memory is managed in chunks and runs (chunks can be split into runs),
44 : * rather than as individual pages. This provides a constant-time
45 : * mechanism for associating allocations with particular arenas.
46 : *
47 : * Allocation requests are rounded up to the nearest size class, and no record
48 : * of the original request size is maintained. Allocations are broken into
49 : * categories according to size class. Assuming runtime defaults, 4 kB pages
50 : * and a 16 byte quantum on a 32-bit system, the size classes in each category
51 : * are as follows:
52 : *
53 : * |=====================================|
54 : * | Category | Subcategory | Size |
55 : * |=====================================|
56 : * | Small | Tiny | 2 |
57 : * | | | 4 |
58 : * | | | 8 |
59 : * | |----------------+---------|
60 : * | | Quantum-spaced | 16 |
61 : * | | | 32 |
62 : * | | | 48 |
63 : * | | | ... |
64 : * | | | 480 |
65 : * | | | 496 |
66 : * | | | 512 |
67 : * | |----------------+---------|
68 : * | | Sub-page | 1 kB |
69 : * | | | 2 kB |
70 : * |=====================================|
71 : * | Large | 4 kB |
72 : * | | 8 kB |
73 : * | | 12 kB |
74 : * | | ... |
75 : * | | 1012 kB |
76 : * | | 1016 kB |
77 : * | | 1020 kB |
78 : * |=====================================|
79 : * | Huge | 1 MB |
80 : * | | 2 MB |
81 : * | | 3 MB |
82 : * | | ... |
83 : * |=====================================|
84 : *
85 : * NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an
86 : * allocation on Linux or Mac. So on 32-bit *nix, the smallest bucket size is
87 : * 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes.
88 : *
89 : * A different mechanism is used for each category:
90 : *
91 : * Small : Each size class is segregated into its own set of runs. Each run
92 : * maintains a bitmap of which regions are free/allocated.
93 : *
94 : * Large : Each allocation is backed by a dedicated run. Metadata are stored
95 : * in the associated arena chunk header maps.
96 : *
97 : * Huge : Each allocation is backed by a dedicated contiguous set of chunks.
98 : * Metadata are stored in a separate red-black tree.
99 : *
100 : *******************************************************************************
101 : */
102 :
103 : #ifdef MOZ_MEMORY_ANDROID
104 : #define NO_TLS
105 : #define _pthread_self() pthread_self()
106 : #endif
107 :
108 : /*
109 : * On Linux, we use madvise(MADV_DONTNEED) to release memory back to the
110 : * operating system. If we release 1MB of live pages with MADV_DONTNEED, our
111 : * RSS will decrease by 1MB (almost) immediately.
112 : *
113 : * On Mac, we use madvise(MADV_FREE). Unlike MADV_DONTNEED on Linux, MADV_FREE
114 : * on Mac doesn't cause the OS to release the specified pages immediately; the
115 : * OS keeps them in our process until the machine comes under memory pressure.
116 : *
117 : * It's therefore difficult to measure the process's RSS on Mac, since, in the
118 : * absence of memory pressure, the contribution from the heap to RSS will not
119 : * decrease due to our madvise calls.
120 : *
121 : * We therefore define MALLOC_DOUBLE_PURGE on Mac. This causes jemalloc to
122 : * track which pages have been MADV_FREE'd. You can then call
123 : * jemalloc_purge_freed_pages(), which will force the OS to release those
124 : * MADV_FREE'd pages, making the process's RSS reflect its true memory usage.
125 : *
126 : */
127 : #ifdef MOZ_MEMORY_DARWIN
128 : #define MALLOC_DOUBLE_PURGE
129 : #endif
130 :
131 : /*
132 : * MALLOC_PRODUCTION disables assertions and statistics gathering. It also
133 : * defaults the A and J runtime options to off. These settings are appropriate
134 : * for production systems.
135 : */
136 : #ifndef MOZ_MEMORY_DEBUG
137 : # define MALLOC_PRODUCTION
138 : #endif
139 :
140 : /*
141 : * Use only one arena by default. Mozilla does not currently make extensive
142 : * use of concurrent allocation, so the increased fragmentation associated with
143 : * multiple arenas is not warranted.
144 : */
145 : #define MOZ_MEMORY_NARENAS_DEFAULT_ONE
146 :
147 : /*
148 : * MALLOC_STATS enables statistics calculation, and is required for
149 : * jemalloc_stats().
150 : */
151 : #define MALLOC_STATS
152 :
153 : #ifndef MALLOC_PRODUCTION
154 : /*
155 : * MALLOC_DEBUG enables assertions and other sanity checks, and disables
156 : * inline functions.
157 : */
158 : # define MALLOC_DEBUG
159 :
160 : /* Memory filling (junk/zero). */
161 : # define MALLOC_FILL
162 :
163 : /* Allocation tracing. */
164 : # ifndef MOZ_MEMORY_WINDOWS
165 : # define MALLOC_UTRACE
166 : # endif
167 :
168 : /* Support optional abort() on OOM. */
169 : # define MALLOC_XMALLOC
170 :
171 : /* Support SYSV semantics. */
172 : # define MALLOC_SYSV
173 : #endif
174 :
175 : /*
176 : * MALLOC_VALIDATE causes malloc_usable_size() to perform some pointer
177 : * validation. There are many possible errors that validation does not even
178 : * attempt to detect.
179 : */
180 : #define MALLOC_VALIDATE
181 :
182 : /* Embed no-op macros that support memory allocation tracking via valgrind. */
183 : #ifdef MOZ_VALGRIND
184 : # define MALLOC_VALGRIND
185 : #endif
186 : #ifdef MALLOC_VALGRIND
187 : # include <valgrind/valgrind.h>
188 : #else
189 : # define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
190 : # define VALGRIND_FREELIKE_BLOCK(addr, rzB)
191 : #endif
192 :
193 : /*
194 : * MALLOC_BALANCE enables monitoring of arena lock contention and dynamically
195 : * re-balances arena load if exponentially averaged contention exceeds a
196 : * certain threshold.
197 : */
198 : /* #define MALLOC_BALANCE */
199 :
200 : /*
201 : * MALLOC_PAGEFILE causes all mmap()ed memory to be backed by temporary
202 : * files, so that if a chunk is mapped, it is guaranteed to be swappable.
203 : * This avoids asynchronous OOM failures that are due to VM over-commit.
204 : */
205 : /* #define MALLOC_PAGEFILE */
206 :
207 : #ifdef MALLOC_PAGEFILE
208 : /* Write size when initializing a page file. */
209 : # define MALLOC_PAGEFILE_WRITE_SIZE 512
210 : #endif
211 :
212 : #if defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
213 : #define _GNU_SOURCE /* For mremap(2). */
214 : #define issetugid() 0
215 : #if 0 /* Enable in order to test decommit code on Linux. */
216 : # define MALLOC_DECOMMIT
217 : #endif
218 : #endif
219 :
220 : #include <sys/types.h>
221 :
222 : #include <errno.h>
223 : #include <stdlib.h>
224 : #include <limits.h>
225 : #include <stdarg.h>
226 : #include <stdio.h>
227 : #include <string.h>
228 :
229 : #ifdef MOZ_MEMORY_WINDOWS
230 :
231 : /* Some defines from the CRT internal headers that we need here. */
232 : #define _CRT_SPINCOUNT 5000
233 : #define __crtInitCritSecAndSpinCount InitializeCriticalSectionAndSpinCount
234 : #include <io.h>
235 : #include <windows.h>
236 :
237 : #pragma warning( disable: 4267 4996 4146 )
238 :
239 : #define bool BOOL
240 : #define false FALSE
241 : #define true TRUE
242 : #define inline __inline
243 : #define SIZE_T_MAX SIZE_MAX
244 : #define STDERR_FILENO 2
245 : #define PATH_MAX MAX_PATH
246 : #define vsnprintf _vsnprintf
247 :
248 : #ifndef NO_TLS
249 : static unsigned long tlsIndex = 0xffffffff;
250 : #endif
251 :
252 : #define __thread
253 : #define _pthread_self() __threadid()
254 : #define issetugid() 0
255 :
256 : /* use MSVC intrinsics */
257 : #pragma intrinsic(_BitScanForward)
258 : static __forceinline int
259 : ffs(int x)
260 : {
261 : unsigned long i;
262 :
263 : if (_BitScanForward(&i, x) != 0)
264 : return (i + 1);
265 :
266 : return (0);
267 : }
268 :
269 : /* Implement getenv without using malloc */
270 : static char mozillaMallocOptionsBuf[64];
271 :
272 : #define getenv xgetenv
273 : static char *
274 : getenv(const char *name)
275 : {
276 :
277 : if (GetEnvironmentVariableA(name, (LPSTR)&mozillaMallocOptionsBuf,
278 : sizeof(mozillaMallocOptionsBuf)) > 0)
279 : return (mozillaMallocOptionsBuf);
280 :
281 : return (NULL);
282 : }
283 :
284 : typedef unsigned char uint8_t;
285 : typedef unsigned uint32_t;
286 : typedef unsigned long long uint64_t;
287 : typedef unsigned long long uintmax_t;
288 : #if defined(MOZ_MEMORY_SIZEOF_PTR_2POW) && (MOZ_MEMORY_SIZEOF_PTR_2POW == 3)
289 : typedef long long ssize_t;
290 : #else
291 : typedef long ssize_t;
292 : #endif
293 :
294 : #define MALLOC_DECOMMIT
295 : #endif
296 :
297 : #ifndef MOZ_MEMORY_WINDOWS
298 : #ifndef MOZ_MEMORY_SOLARIS
299 : #include <sys/cdefs.h>
300 : #endif
301 : #ifndef __DECONST
302 : # define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
303 : #endif
304 : #ifndef MOZ_MEMORY
305 : __FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z jasone $");
306 : #include "libc_private.h"
307 : #ifdef MALLOC_DEBUG
308 : # define _LOCK_DEBUG
309 : #endif
310 : #include "spinlock.h"
311 : #include "namespace.h"
312 : #endif
313 : #include <sys/mman.h>
314 : #ifndef MADV_FREE
315 : # define MADV_FREE MADV_DONTNEED
316 : #endif
317 : #ifndef MAP_NOSYNC
318 : # define MAP_NOSYNC 0
319 : #endif
320 : #include <sys/param.h>
321 : #ifndef MOZ_MEMORY
322 : #include <sys/stddef.h>
323 : #endif
324 : #include <sys/time.h>
325 : #include <sys/types.h>
326 : #if !defined(MOZ_MEMORY_SOLARIS) && !defined(MOZ_MEMORY_ANDROID)
327 : #include <sys/sysctl.h>
328 : #endif
329 : #include <sys/uio.h>
330 : #ifndef MOZ_MEMORY
331 : #include <sys/ktrace.h> /* Must come after several other sys/ includes. */
332 :
333 : #include <machine/atomic.h>
334 : #include <machine/cpufunc.h>
335 : #include <machine/vmparam.h>
336 : #endif
337 :
338 : #include <errno.h>
339 : #include <limits.h>
340 : #ifndef SIZE_T_MAX
341 : # define SIZE_T_MAX SIZE_MAX
342 : #endif
343 : #include <pthread.h>
344 : #ifdef MOZ_MEMORY_DARWIN
345 : #define _pthread_self pthread_self
346 : #define _pthread_mutex_init pthread_mutex_init
347 : #define _pthread_mutex_trylock pthread_mutex_trylock
348 : #define _pthread_mutex_lock pthread_mutex_lock
349 : #define _pthread_mutex_unlock pthread_mutex_unlock
350 : #endif
351 : #include <sched.h>
352 : #include <stdarg.h>
353 : #include <stdio.h>
354 : #include <stdbool.h>
355 : #include <stdint.h>
356 : #include <stdlib.h>
357 : #include <string.h>
358 : #ifndef MOZ_MEMORY_DARWIN
359 : #include <strings.h>
360 : #endif
361 : #include <unistd.h>
362 :
363 : #ifdef MOZ_MEMORY_DARWIN
364 : #include <libkern/OSAtomic.h>
365 : #include <mach/mach_error.h>
366 : #include <mach/mach_init.h>
367 : #include <mach/vm_map.h>
368 : #include <malloc/malloc.h>
369 : #endif
370 :
371 : #ifndef MOZ_MEMORY
372 : #include "un-namespace.h"
373 : #endif
374 :
375 : #endif
376 :
377 : #include "jemalloc.h"
378 : #include "linkedlist.h"
379 :
380 : /* Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that
381 : * happen to override mmap() and call dlsym() from their overridden
382 : * mmap(). The problem is that dlsym() calls malloc(), and this ends
383 : * up in a dead lock in jemalloc.
384 : * On these systems, we prefer to directly use the system call.
385 : * We do that for Linux systems and kfreebsd with GNU userland.
386 : * Note sanity checks are not done (alignment of offset, ...) because
387 : * the uses of mmap are pretty limited, in jemalloc.
388 : *
389 : * On Alpha, glibc has a bug that prevents syscall() to work for system
390 : * calls with 6 arguments
391 : */
392 : #if (defined(MOZ_MEMORY_LINUX) && !defined(__alpha__)) || \
393 : (defined(MOZ_MEMORY_BSD) && defined(__GLIBC__))
394 : #include <sys/syscall.h>
395 : #if defined(SYS_mmap) || defined(SYS_mmap2)
396 : static inline
397 82582 : void *_mmap(void *addr, size_t length, int prot, int flags,
398 : int fd, off_t offset)
399 : {
400 : /* S390 only passes one argument to the mmap system call, which is a
401 : * pointer to a structure containing the arguments */
402 : #ifdef __s390__
403 : struct {
404 : void *addr;
405 : size_t length;
406 : int prot;
407 : int flags;
408 : int fd;
409 : off_t offset;
410 : } args = { addr, length, prot, flags, fd, offset };
411 : return (void *) syscall(SYS_mmap, &args);
412 : #else
413 : #ifdef SYS_mmap2
414 82582 : return (void *) syscall(SYS_mmap2, addr, length, prot, flags,
415 : fd, offset >> 12);
416 : #else
417 : return (void *) syscall(SYS_mmap, addr, length, prot, flags,
418 : fd, offset);
419 : #endif
420 : #endif
421 : }
422 : #define mmap _mmap
423 : #define munmap(a, l) syscall(SYS_munmap, a, l)
424 : #endif
425 : #endif
426 :
427 : #ifdef MOZ_MEMORY_DARWIN
428 : static const bool __isthreaded = true;
429 : #endif
430 :
431 : #if defined(MOZ_MEMORY_SOLARIS) && defined(MAP_ALIGN) && !defined(JEMALLOC_NEVER_USES_MAP_ALIGN)
432 : #define JEMALLOC_USES_MAP_ALIGN /* Required on Solaris 10. Might improve performance elsewhere. */
433 : #endif
434 :
435 : #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
436 :
437 : #ifdef MOZ_MEMORY_WINDOWS
438 : /* MSVC++ does not support C99 variable-length arrays. */
439 : # define RB_NO_C99_VARARRAYS
440 : #endif
441 : #include "rb.h"
442 :
443 : #ifdef MALLOC_DEBUG
444 : /* Disable inlining to make debugging easier. */
445 : #ifdef inline
446 : #undef inline
447 : #endif
448 :
449 : # define inline
450 : #endif
451 :
452 : /* Size of stack-allocated buffer passed to strerror_r(). */
453 : #define STRERROR_BUF 64
454 :
455 : /* Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. */
456 : # define QUANTUM_2POW_MIN 4
457 : #ifdef MOZ_MEMORY_SIZEOF_PTR_2POW
458 : # define SIZEOF_PTR_2POW MOZ_MEMORY_SIZEOF_PTR_2POW
459 : #else
460 : # define SIZEOF_PTR_2POW 2
461 : #endif
462 : #define PIC
463 : #ifndef MOZ_MEMORY_DARWIN
464 : static const bool __isthreaded = true;
465 : #else
466 : # define NO_TLS
467 : #endif
468 : #if 0
469 : #ifdef __i386__
470 : # define QUANTUM_2POW_MIN 4
471 : # define SIZEOF_PTR_2POW 2
472 : # define CPU_SPINWAIT __asm__ volatile("pause")
473 : #endif
474 : #ifdef __ia64__
475 : # define QUANTUM_2POW_MIN 4
476 : # define SIZEOF_PTR_2POW 3
477 : #endif
478 : #ifdef __alpha__
479 : # define QUANTUM_2POW_MIN 4
480 : # define SIZEOF_PTR_2POW 3
481 : # define NO_TLS
482 : #endif
483 : #ifdef __sparc64__
484 : # define QUANTUM_2POW_MIN 4
485 : # define SIZEOF_PTR_2POW 3
486 : # define NO_TLS
487 : #endif
488 : #ifdef __amd64__
489 : # define QUANTUM_2POW_MIN 4
490 : # define SIZEOF_PTR_2POW 3
491 : # define CPU_SPINWAIT __asm__ volatile("pause")
492 : #endif
493 : #ifdef __arm__
494 : # define QUANTUM_2POW_MIN 3
495 : # define SIZEOF_PTR_2POW 2
496 : # define NO_TLS
497 : #endif
498 : #ifdef __mips__
499 : # define QUANTUM_2POW_MIN 3
500 : # define SIZEOF_PTR_2POW 2
501 : # define NO_TLS
502 : #endif
503 : #ifdef __powerpc__
504 : # define QUANTUM_2POW_MIN 4
505 : # define SIZEOF_PTR_2POW 2
506 : #endif
507 : #endif
508 :
509 : #define SIZEOF_PTR (1U << SIZEOF_PTR_2POW)
510 :
511 : /* sizeof(int) == (1U << SIZEOF_INT_2POW). */
512 : #ifndef SIZEOF_INT_2POW
513 : # define SIZEOF_INT_2POW 2
514 : #endif
515 :
516 : /* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
517 : #if (!defined(PIC) && !defined(NO_TLS))
518 : # define NO_TLS
519 : #endif
520 :
521 : #ifdef NO_TLS
522 : /* MALLOC_BALANCE requires TLS. */
523 : # ifdef MALLOC_BALANCE
524 : # undef MALLOC_BALANCE
525 : # endif
526 : #endif
527 :
528 : /*
529 : * Size and alignment of memory chunks that are allocated by the OS's virtual
530 : * memory system.
531 : */
532 : #define CHUNK_2POW_DEFAULT 20
533 : /* Maximum number of dirty pages per arena. */
534 : #define DIRTY_MAX_DEFAULT (1U << 10)
535 :
536 : /*
537 : * Maximum size of L1 cache line. This is used to avoid cache line aliasing,
538 : * so over-estimates are okay (up to a point), but under-estimates will
539 : * negatively affect performance.
540 : */
541 : #define CACHELINE_2POW 6
542 : #define CACHELINE ((size_t)(1U << CACHELINE_2POW))
543 :
544 : /*
545 : * Smallest size class to support. On Linux and Mac, even malloc(1) must
546 : * reserve a word's worth of memory (see Mozilla bug 691003).
547 : */
548 : #ifdef MOZ_MEMORY_WINDOWS
549 : #define TINY_MIN_2POW 1
550 : #else
551 : #define TINY_MIN_2POW (sizeof(void*) == 8 ? 3 : 2)
552 : #endif
553 :
554 : /*
555 : * Maximum size class that is a multiple of the quantum, but not (necessarily)
556 : * a power of 2. Above this size, allocations are rounded up to the nearest
557 : * power of 2.
558 : */
559 : #define SMALL_MAX_2POW_DEFAULT 9
560 : #define SMALL_MAX_DEFAULT (1U << SMALL_MAX_2POW_DEFAULT)
561 :
562 : /*
563 : * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized
564 : * as small as possible such that this setting is still honored, without
565 : * violating other constraints. The goal is to make runs as small as possible
566 : * without exceeding a per run external fragmentation threshold.
567 : *
568 : * We use binary fixed point math for overhead computations, where the binary
569 : * point is implicitly RUN_BFP bits to the left.
570 : *
571 : * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
572 : * honored for some/all object sizes, since there is one bit of header overhead
573 : * per object (plus a constant). This constraint is relaxed (ignored) for runs
574 : * that are so small that the per-region overhead is greater than:
575 : *
576 : * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
577 : */
578 : #define RUN_BFP 12
579 : /* \/ Implicit binary fixed point. */
580 : #define RUN_MAX_OVRHD 0x0000003dU
581 : #define RUN_MAX_OVRHD_RELAX 0x00001800U
582 :
583 : /* Put a cap on small object run size. This overrides RUN_MAX_OVRHD. */
584 : #define RUN_MAX_SMALL_2POW 15
585 : #define RUN_MAX_SMALL (1U << RUN_MAX_SMALL_2POW)
586 :
587 : /*
588 : * Hyper-threaded CPUs may need a special instruction inside spin loops in
589 : * order to yield to another virtual CPU. If no such instruction is defined
590 : * above, make CPU_SPINWAIT a no-op.
591 : */
592 : #ifndef CPU_SPINWAIT
593 : # define CPU_SPINWAIT
594 : #endif
595 :
596 : /*
597 : * Adaptive spinning must eventually switch to blocking, in order to avoid the
598 : * potential for priority inversion deadlock. Backing off past a certain point
599 : * can actually waste time.
600 : */
601 : #define SPIN_LIMIT_2POW 11
602 :
603 : /*
604 : * Conversion from spinning to blocking is expensive; we use (1U <<
605 : * BLOCK_COST_2POW) to estimate how many more times costly blocking is than
606 : * worst-case spinning.
607 : */
608 : #define BLOCK_COST_2POW 4
609 :
610 : #ifdef MALLOC_BALANCE
611 : /*
612 : * We use an exponential moving average to track recent lock contention,
613 : * where the size of the history window is N, and alpha=2/(N+1).
614 : *
615 : * Due to integer math rounding, very small values here can cause
616 : * substantial degradation in accuracy, thus making the moving average decay
617 : * faster than it would with precise calculation.
618 : */
619 : # define BALANCE_ALPHA_INV_2POW 9
620 :
621 : /*
622 : * Threshold value for the exponential moving contention average at which to
623 : * re-assign a thread.
624 : */
625 : # define BALANCE_THRESHOLD_DEFAULT (1U << (SPIN_LIMIT_2POW-4))
626 : #endif
627 :
628 : /******************************************************************************/
629 :
630 : /* MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. */
631 : #if defined(MALLOC_DECOMMIT) && defined(MALLOC_DOUBLE_PURGE)
632 : #error MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive.
633 : #endif
634 :
635 : /*
636 : * Mutexes based on spinlocks. We can't use normal pthread spinlocks in all
637 : * places, because they require malloc()ed memory, which causes bootstrapping
638 : * issues in some cases.
639 : */
640 : #if defined(MOZ_MEMORY_WINDOWS)
641 : #define malloc_mutex_t CRITICAL_SECTION
642 : #define malloc_spinlock_t CRITICAL_SECTION
643 : #elif defined(MOZ_MEMORY_DARWIN)
644 : typedef struct {
645 : OSSpinLock lock;
646 : } malloc_mutex_t;
647 : typedef struct {
648 : OSSpinLock lock;
649 : } malloc_spinlock_t;
650 : #elif defined(MOZ_MEMORY)
651 : typedef pthread_mutex_t malloc_mutex_t;
652 : typedef pthread_mutex_t malloc_spinlock_t;
653 : #else
654 : /* XXX these should #ifdef these for freebsd (and linux?) only */
655 : typedef struct {
656 : spinlock_t lock;
657 : } malloc_mutex_t;
658 : typedef malloc_spinlock_t malloc_mutex_t;
659 : #endif
660 :
661 : /* Set to true once the allocator has been initialized. */
662 : static bool malloc_initialized = false;
663 :
664 : #if defined(MOZ_MEMORY_WINDOWS)
665 : /* No init lock for Windows. */
666 : #elif defined(MOZ_MEMORY_DARWIN)
667 : static malloc_mutex_t init_lock = {OS_SPINLOCK_INIT};
668 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
669 : static malloc_mutex_t init_lock = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
670 : #elif defined(MOZ_MEMORY)
671 : static malloc_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
672 : #else
673 : static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
674 : #endif
675 :
676 : /******************************************************************************/
677 : /*
678 : * Statistics data structures.
679 : */
680 :
681 : #ifdef MALLOC_STATS
682 :
683 : typedef struct malloc_bin_stats_s malloc_bin_stats_t;
684 : struct malloc_bin_stats_s {
685 : /*
686 : * Number of allocation requests that corresponded to the size of this
687 : * bin.
688 : */
689 : uint64_t nrequests;
690 :
691 : /* Total number of runs created for this bin's size class. */
692 : uint64_t nruns;
693 :
694 : /*
695 : * Total number of runs reused by extracting them from the runs tree for
696 : * this bin's size class.
697 : */
698 : uint64_t reruns;
699 :
700 : /* High-water mark for this bin. */
701 : unsigned long highruns;
702 :
703 : /* Current number of runs in this bin. */
704 : unsigned long curruns;
705 : };
706 :
707 : typedef struct arena_stats_s arena_stats_t;
708 : struct arena_stats_s {
709 : /* Number of bytes currently mapped. */
710 : size_t mapped;
711 :
712 : /*
713 : * Total number of purge sweeps, total number of madvise calls made,
714 : * and total pages purged in order to keep dirty unused memory under
715 : * control.
716 : */
717 : uint64_t npurge;
718 : uint64_t nmadvise;
719 : uint64_t purged;
720 : #ifdef MALLOC_DECOMMIT
721 : /*
722 : * Total number of decommit/commit operations, and total number of
723 : * pages decommitted.
724 : */
725 : uint64_t ndecommit;
726 : uint64_t ncommit;
727 : uint64_t decommitted;
728 : #endif
729 :
730 : /* Current number of committed pages. */
731 : size_t committed;
732 :
733 : /* Per-size-category statistics. */
734 : size_t allocated_small;
735 : uint64_t nmalloc_small;
736 : uint64_t ndalloc_small;
737 :
738 : size_t allocated_large;
739 : uint64_t nmalloc_large;
740 : uint64_t ndalloc_large;
741 :
742 : #ifdef MALLOC_BALANCE
743 : /* Number of times this arena reassigned a thread due to contention. */
744 : uint64_t nbalance;
745 : #endif
746 : };
747 :
748 : typedef struct chunk_stats_s chunk_stats_t;
749 : struct chunk_stats_s {
750 : /* Number of chunks that were allocated. */
751 : uint64_t nchunks;
752 :
753 : /* High-water mark for number of chunks allocated. */
754 : unsigned long highchunks;
755 :
756 : /*
757 : * Current number of chunks allocated. This value isn't maintained for
758 : * any other purpose, so keep track of it in order to be able to set
759 : * highchunks.
760 : */
761 : unsigned long curchunks;
762 : };
763 :
764 : #endif /* #ifdef MALLOC_STATS */
765 :
766 : /******************************************************************************/
767 : /*
768 : * Extent data structures.
769 : */
770 :
771 : /* Tree of extents. */
772 : typedef struct extent_node_s extent_node_t;
773 : struct extent_node_s {
774 : /* Linkage for the size/address-ordered tree. */
775 : rb_node(extent_node_t) link_szad;
776 :
777 : /* Linkage for the address-ordered tree. */
778 : rb_node(extent_node_t) link_ad;
779 :
780 : /* Pointer to the extent that this tree node is responsible for. */
781 : void *addr;
782 :
783 : /* Total region size. */
784 : size_t size;
785 : };
786 : typedef rb_tree(extent_node_t) extent_tree_t;
787 :
788 : /******************************************************************************/
789 : /*
790 : * Radix tree data structures.
791 : */
792 :
793 : #ifdef MALLOC_VALIDATE
794 : /*
795 : * Size of each radix tree node (must be a power of 2). This impacts tree
796 : * depth.
797 : */
798 : # if (SIZEOF_PTR == 4)
799 : # define MALLOC_RTREE_NODESIZE (1U << 14)
800 : # else
801 : # define MALLOC_RTREE_NODESIZE CACHELINE
802 : # endif
803 :
804 : typedef struct malloc_rtree_s malloc_rtree_t;
805 : struct malloc_rtree_s {
806 : malloc_spinlock_t lock;
807 : void **root;
808 : unsigned height;
809 : unsigned level2bits[1]; /* Dynamically sized. */
810 : };
811 : #endif
812 :
813 : /******************************************************************************/
814 : /*
815 : * Arena data structures.
816 : */
817 :
818 : typedef struct arena_s arena_t;
819 : typedef struct arena_bin_s arena_bin_t;
820 :
821 : /* Each element of the chunk map corresponds to one page within the chunk. */
822 : typedef struct arena_chunk_map_s arena_chunk_map_t;
823 : struct arena_chunk_map_s {
824 : /*
825 : * Linkage for run trees. There are two disjoint uses:
826 : *
827 : * 1) arena_t's runs_avail tree.
828 : * 2) arena_run_t conceptually uses this linkage for in-use non-full
829 : * runs, rather than directly embedding linkage.
830 : */
831 : rb_node(arena_chunk_map_t) link;
832 :
833 : /*
834 : * Run address (or size) and various flags are stored together. The bit
835 : * layout looks like (assuming 32-bit system):
836 : *
837 : * ???????? ???????? ????---- -mckdzla
838 : *
839 : * ? : Unallocated: Run address for first/last pages, unset for internal
840 : * pages.
841 : * Small: Run address.
842 : * Large: Run size for first page, unset for trailing pages.
843 : * - : Unused.
844 : * m : MADV_FREE/MADV_DONTNEED'ed?
845 : * c : decommitted?
846 : * k : key?
847 : * d : dirty?
848 : * z : zeroed?
849 : * l : large?
850 : * a : allocated?
851 : *
852 : * Following are example bit patterns for the three types of runs.
853 : *
854 : * r : run address
855 : * s : run size
856 : * x : don't care
857 : * - : 0
858 : * [cdzla] : bit set
859 : *
860 : * Unallocated:
861 : * ssssssss ssssssss ssss---- --c-----
862 : * xxxxxxxx xxxxxxxx xxxx---- ----d---
863 : * ssssssss ssssssss ssss---- -----z--
864 : *
865 : * Small:
866 : * rrrrrrrr rrrrrrrr rrrr---- -------a
867 : * rrrrrrrr rrrrrrrr rrrr---- -------a
868 : * rrrrrrrr rrrrrrrr rrrr---- -------a
869 : *
870 : * Large:
871 : * ssssssss ssssssss ssss---- ------la
872 : * -------- -------- -------- ------la
873 : * -------- -------- -------- ------la
874 : */
875 : size_t bits;
876 :
877 : /* Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether
878 : * MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined.
879 : *
880 : * If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be
881 : * re-committed with pages_commit() before it may be touched. If
882 : * MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined.
883 : *
884 : * If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which
885 : * are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with
886 : * CHUNK_MAP_MADVISED.
887 : *
888 : * Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is
889 : * defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED.
890 : * When it's finally freed with jemalloc_purge_freed_pages, the page is marked
891 : * as CHUNK_MAP_DECOMMITTED.
892 : */
893 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
894 : #define CHUNK_MAP_MADVISED ((size_t)0x40U)
895 : #define CHUNK_MAP_DECOMMITTED ((size_t)0x20U)
896 : #define CHUNK_MAP_MADVISED_OR_DECOMMITTED (CHUNK_MAP_MADVISED | CHUNK_MAP_DECOMMITTED)
897 : #endif
898 : #define CHUNK_MAP_KEY ((size_t)0x10U)
899 : #define CHUNK_MAP_DIRTY ((size_t)0x08U)
900 : #define CHUNK_MAP_ZEROED ((size_t)0x04U)
901 : #define CHUNK_MAP_LARGE ((size_t)0x02U)
902 : #define CHUNK_MAP_ALLOCATED ((size_t)0x01U)
903 : };
904 : typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
905 : typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
906 :
907 : /* Arena chunk header. */
908 : typedef struct arena_chunk_s arena_chunk_t;
909 : struct arena_chunk_s {
910 : /* Arena that owns the chunk. */
911 : arena_t *arena;
912 :
913 : /* Linkage for the arena's chunks_dirty tree. */
914 : rb_node(arena_chunk_t) link_dirty;
915 :
916 : #ifdef MALLOC_DOUBLE_PURGE
917 : /* If we're double-purging, we maintain a linked list of chunks which
918 : * have pages which have been madvise(MADV_FREE)'d but not explicitly
919 : * purged.
920 : *
921 : * We're currently lazy and don't remove a chunk from this list when
922 : * all its madvised pages are recommitted. */
923 : LinkedList chunks_madvised_elem;
924 : #endif
925 :
926 : /* Number of dirty pages. */
927 : size_t ndirty;
928 :
929 : /* Map of pages within chunk that keeps track of free/large/small. */
930 : arena_chunk_map_t map[1]; /* Dynamically sized. */
931 : };
932 : typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
933 :
934 : typedef struct arena_run_s arena_run_t;
935 : struct arena_run_s {
936 : #ifdef MALLOC_DEBUG
937 : uint32_t magic;
938 : # define ARENA_RUN_MAGIC 0x384adf93
939 : #endif
940 :
941 : /* Bin this run is associated with. */
942 : arena_bin_t *bin;
943 :
944 : /* Index of first element that might have a free region. */
945 : unsigned regs_minelm;
946 :
947 : /* Number of free regions in run. */
948 : unsigned nfree;
949 :
950 : /* Bitmask of in-use regions (0: in use, 1: free). */
951 : unsigned regs_mask[1]; /* Dynamically sized. */
952 : };
953 :
954 : struct arena_bin_s {
955 : /*
956 : * Current run being used to service allocations of this bin's size
957 : * class.
958 : */
959 : arena_run_t *runcur;
960 :
961 : /*
962 : * Tree of non-full runs. This tree is used when looking for an
963 : * existing run when runcur is no longer usable. We choose the
964 : * non-full run that is lowest in memory; this policy tends to keep
965 : * objects packed well, and it can also help reduce the number of
966 : * almost-empty chunks.
967 : */
968 : arena_run_tree_t runs;
969 :
970 : /* Size of regions in a run for this bin's size class. */
971 : size_t reg_size;
972 :
973 : /* Total size of a run for this bin's size class. */
974 : size_t run_size;
975 :
976 : /* Total number of regions in a run for this bin's size class. */
977 : uint32_t nregs;
978 :
979 : /* Number of elements in a run's regs_mask for this bin's size class. */
980 : uint32_t regs_mask_nelms;
981 :
982 : /* Offset of first region in a run for this bin's size class. */
983 : uint32_t reg0_offset;
984 :
985 : #ifdef MALLOC_STATS
986 : /* Bin statistics. */
987 : malloc_bin_stats_t stats;
988 : #endif
989 : };
990 :
991 : struct arena_s {
992 : #ifdef MALLOC_DEBUG
993 : uint32_t magic;
994 : # define ARENA_MAGIC 0x947d3d24
995 : #endif
996 :
997 : /* All operations on this arena require that lock be locked. */
998 : #ifdef MOZ_MEMORY
999 : malloc_spinlock_t lock;
1000 : #else
1001 : pthread_mutex_t lock;
1002 : #endif
1003 :
1004 : #ifdef MALLOC_STATS
1005 : arena_stats_t stats;
1006 : #endif
1007 :
1008 : /* Tree of dirty-page-containing chunks this arena manages. */
1009 : arena_chunk_tree_t chunks_dirty;
1010 :
1011 : #ifdef MALLOC_DOUBLE_PURGE
1012 : /* Head of a linked list of MADV_FREE'd-page-containing chunks this
1013 : * arena manages. */
1014 : LinkedList chunks_madvised;
1015 : #endif
1016 :
1017 : /*
1018 : * In order to avoid rapid chunk allocation/deallocation when an arena
1019 : * oscillates right on the cusp of needing a new chunk, cache the most
1020 : * recently freed chunk. The spare is left in the arena's chunk trees
1021 : * until it is deleted.
1022 : *
1023 : * There is one spare chunk per arena, rather than one spare total, in
1024 : * order to avoid interactions between multiple threads that could make
1025 : * a single spare inadequate.
1026 : */
1027 : arena_chunk_t *spare;
1028 :
1029 : /*
1030 : * Current count of pages within unused runs that are potentially
1031 : * dirty, and for which madvise(... MADV_FREE) has not been called. By
1032 : * tracking this, we can institute a limit on how much dirty unused
1033 : * memory is mapped for each arena.
1034 : */
1035 : size_t ndirty;
1036 :
1037 : /*
1038 : * Size/address-ordered tree of this arena's available runs. This tree
1039 : * is used for first-best-fit run allocation.
1040 : */
1041 : arena_avail_tree_t runs_avail;
1042 :
1043 : #ifdef MALLOC_BALANCE
1044 : /*
1045 : * The arena load balancing machinery needs to keep track of how much
1046 : * lock contention there is. This value is exponentially averaged.
1047 : */
1048 : uint32_t contention;
1049 : #endif
1050 :
1051 : /*
1052 : * bins is used to store rings of free regions of the following sizes,
1053 : * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS.
1054 : *
1055 : * bins[i] | size |
1056 : * --------+------+
1057 : * 0 | 2 |
1058 : * 1 | 4 |
1059 : * 2 | 8 |
1060 : * --------+------+
1061 : * 3 | 16 |
1062 : * 4 | 32 |
1063 : * 5 | 48 |
1064 : * 6 | 64 |
1065 : * : :
1066 : * : :
1067 : * 33 | 496 |
1068 : * 34 | 512 |
1069 : * --------+------+
1070 : * 35 | 1024 |
1071 : * 36 | 2048 |
1072 : * --------+------+
1073 : */
1074 : arena_bin_t bins[1]; /* Dynamically sized. */
1075 : };
1076 :
1077 : /******************************************************************************/
1078 : /*
1079 : * Data.
1080 : */
1081 :
1082 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1083 : /* Number of CPUs. */
1084 : static unsigned ncpus;
1085 : #endif
1086 :
1087 : /*
1088 : * When MALLOC_STATIC_SIZES is defined most of the parameters
1089 : * controlling the malloc behavior are defined as compile-time constants
1090 : * for best performance and cannot be altered at runtime.
1091 : */
1092 : #define MALLOC_STATIC_SIZES 1
1093 :
1094 : #ifdef MALLOC_STATIC_SIZES
1095 :
1096 : /*
1097 : * VM page size. It must divide the runtime CPU page size or the code
1098 : * will abort.
1099 : */
1100 : #define pagesize_2pow ((size_t) 12)
1101 : #define pagesize ((size_t) 1 << pagesize_2pow)
1102 : #define pagesize_mask (pagesize - 1)
1103 :
1104 : /* Various quantum-related settings. */
1105 :
1106 : #define QUANTUM_DEFAULT ((size_t) 1 << QUANTUM_2POW_MIN)
1107 : static const size_t quantum = QUANTUM_DEFAULT;
1108 : static const size_t quantum_mask = QUANTUM_DEFAULT - 1;
1109 :
1110 : /* Various bin-related settings. */
1111 :
1112 : static const size_t small_min = (QUANTUM_DEFAULT >> 1) + 1;
1113 : static const size_t small_max = (size_t) SMALL_MAX_DEFAULT;
1114 :
1115 : /* Max size class for bins. */
1116 : static const size_t bin_maxclass = pagesize >> 1;
1117 :
1118 : /* Number of (2^n)-spaced tiny bins. */
1119 : static const unsigned ntbins = (unsigned)
1120 : (QUANTUM_2POW_MIN - TINY_MIN_2POW);
1121 :
1122 : /* Number of quantum-spaced bins. */
1123 : static const unsigned nqbins = (unsigned)
1124 : (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
1125 :
1126 : /* Number of (2^n)-spaced sub-page bins. */
1127 : static const unsigned nsbins = (unsigned)
1128 : (pagesize_2pow -
1129 : SMALL_MAX_2POW_DEFAULT - 1);
1130 :
1131 : #else /* !MALLOC_STATIC_SIZES */
1132 :
1133 : /* VM page size. */
1134 : static size_t pagesize;
1135 : static size_t pagesize_mask;
1136 : static size_t pagesize_2pow;
1137 :
1138 : /* Various bin-related settings. */
1139 : static size_t bin_maxclass; /* Max size class for bins. */
1140 : static unsigned ntbins; /* Number of (2^n)-spaced tiny bins. */
1141 : static unsigned nqbins; /* Number of quantum-spaced bins. */
1142 : static unsigned nsbins; /* Number of (2^n)-spaced sub-page bins. */
1143 : static size_t small_min;
1144 : static size_t small_max;
1145 :
1146 : /* Various quantum-related settings. */
1147 : static size_t quantum;
1148 : static size_t quantum_mask; /* (quantum - 1). */
1149 :
1150 : #endif
1151 :
1152 : /* Various chunk-related settings. */
1153 :
1154 : /*
1155 : * Compute the header size such that it is large enough to contain the page map
1156 : * and enough nodes for the worst case: one node per non-header page plus one
1157 : * extra for situations where we briefly have one more node allocated than we
1158 : * will need.
1159 : */
1160 : #define calculate_arena_header_size() \
1161 : (sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1))
1162 :
1163 : #define calculate_arena_header_pages() \
1164 : ((calculate_arena_header_size() >> pagesize_2pow) + \
1165 : ((calculate_arena_header_size() & pagesize_mask) ? 1 : 0))
1166 :
1167 : /* Max size class for arenas. */
1168 : #define calculate_arena_maxclass() \
1169 : (chunksize - (arena_chunk_header_npages << pagesize_2pow))
1170 :
1171 : #ifdef MALLOC_STATIC_SIZES
1172 : #define CHUNKSIZE_DEFAULT ((size_t) 1 << CHUNK_2POW_DEFAULT)
1173 : static const size_t chunksize = CHUNKSIZE_DEFAULT;
1174 : static const size_t chunksize_mask =CHUNKSIZE_DEFAULT - 1;
1175 : static const size_t chunk_npages = CHUNKSIZE_DEFAULT >> pagesize_2pow;
1176 : #define arena_chunk_header_npages calculate_arena_header_pages()
1177 : #define arena_maxclass calculate_arena_maxclass()
1178 : #else
1179 : static size_t chunksize;
1180 : static size_t chunksize_mask; /* (chunksize - 1). */
1181 : static size_t chunk_npages;
1182 : static size_t arena_chunk_header_npages;
1183 : static size_t arena_maxclass; /* Max size class for arenas. */
1184 : #endif
1185 :
1186 : /********/
1187 : /*
1188 : * Chunks.
1189 : */
1190 :
1191 : #ifdef MALLOC_VALIDATE
1192 : static malloc_rtree_t *chunk_rtree;
1193 : #endif
1194 :
1195 : /* Protects chunk-related data structures. */
1196 : static malloc_mutex_t huge_mtx;
1197 :
1198 : /* Tree of chunks that are stand-alone huge allocations. */
1199 : static extent_tree_t huge;
1200 :
1201 : #ifdef MALLOC_STATS
1202 : /* Huge allocation statistics. */
1203 : static uint64_t huge_nmalloc;
1204 : static uint64_t huge_ndalloc;
1205 : static size_t huge_allocated;
1206 : #endif
1207 :
1208 : #ifdef MALLOC_PAGEFILE
1209 : static char pagefile_templ[PATH_MAX];
1210 : #endif
1211 :
1212 : /****************************/
1213 : /*
1214 : * base (internal allocation).
1215 : */
1216 :
1217 : /*
1218 : * Current pages that are being used for internal memory allocations. These
1219 : * pages are carved up in cacheline-size quanta, so that there is no chance of
1220 : * false cache line sharing.
1221 : */
1222 : static void *base_pages;
1223 : static void *base_next_addr;
1224 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1225 : static void *base_next_decommitted;
1226 : #endif
1227 : static void *base_past_addr; /* Addr immediately past base_pages. */
1228 : static extent_node_t *base_nodes;
1229 : static malloc_mutex_t base_mtx;
1230 : #ifdef MALLOC_STATS
1231 : static size_t base_mapped;
1232 : static size_t base_committed;
1233 : #endif
1234 :
1235 : /********/
1236 : /*
1237 : * Arenas.
1238 : */
1239 :
1240 : /*
1241 : * Arenas that are used to service external requests. Not all elements of the
1242 : * arenas array are necessarily used; arenas are created lazily as needed.
1243 : */
1244 : static arena_t **arenas;
1245 : static unsigned narenas;
1246 : #ifndef NO_TLS
1247 : # ifdef MALLOC_BALANCE
1248 : static unsigned narenas_2pow;
1249 : # else
1250 : static unsigned next_arena;
1251 : # endif
1252 : #endif
1253 : #ifdef MOZ_MEMORY
1254 : static malloc_spinlock_t arenas_lock; /* Protects arenas initialization. */
1255 : #else
1256 : static pthread_mutex_t arenas_lock; /* Protects arenas initialization. */
1257 : #endif
1258 :
1259 : #ifndef NO_TLS
1260 : /*
1261 : * Map of pthread_self() --> arenas[???], used for selecting an arena to use
1262 : * for allocations.
1263 : */
1264 : #ifndef MOZ_MEMORY_WINDOWS
1265 : static __thread arena_t *arenas_map;
1266 : #endif
1267 : #endif
1268 :
1269 : #ifdef MALLOC_STATS
1270 : /* Chunk statistics. */
1271 : static chunk_stats_t stats_chunks;
1272 : #endif
1273 :
1274 : /*******************************/
1275 : /*
1276 : * Runtime configuration options.
1277 : */
1278 : const char *_malloc_options;
1279 :
1280 : #ifndef MALLOC_PRODUCTION
1281 : static bool opt_abort = true;
1282 : #ifdef MALLOC_FILL
1283 : static bool opt_junk = true;
1284 : #endif
1285 : #else
1286 : static bool opt_abort = false;
1287 : #ifdef MALLOC_FILL
1288 : static bool opt_junk = false;
1289 : #endif
1290 : #endif
1291 : static size_t opt_dirty_max = DIRTY_MAX_DEFAULT;
1292 : #ifdef MALLOC_BALANCE
1293 : static uint64_t opt_balance_threshold = BALANCE_THRESHOLD_DEFAULT;
1294 : #endif
1295 : static bool opt_print_stats = false;
1296 : #ifdef MALLOC_STATIC_SIZES
1297 : #define opt_quantum_2pow QUANTUM_2POW_MIN
1298 : #define opt_small_max_2pow SMALL_MAX_2POW_DEFAULT
1299 : #define opt_chunk_2pow CHUNK_2POW_DEFAULT
1300 : #else
1301 : static size_t opt_quantum_2pow = QUANTUM_2POW_MIN;
1302 : static size_t opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
1303 : static size_t opt_chunk_2pow = CHUNK_2POW_DEFAULT;
1304 : #endif
1305 : #ifdef MALLOC_PAGEFILE
1306 : static bool opt_pagefile = false;
1307 : #endif
1308 : #ifdef MALLOC_UTRACE
1309 : static bool opt_utrace = false;
1310 : #endif
1311 : #ifdef MALLOC_SYSV
1312 : static bool opt_sysv = false;
1313 : #endif
1314 : #ifdef MALLOC_XMALLOC
1315 : static bool opt_xmalloc = false;
1316 : #endif
1317 : #ifdef MALLOC_FILL
1318 : static bool opt_zero = false;
1319 : #endif
1320 : static int opt_narenas_lshift = 0;
1321 :
1322 : #ifdef MALLOC_UTRACE
1323 : typedef struct {
1324 : void *p;
1325 : size_t s;
1326 : void *r;
1327 : } malloc_utrace_t;
1328 :
1329 : #define UTRACE(a, b, c) \
1330 : if (opt_utrace) { \
1331 : malloc_utrace_t ut; \
1332 : ut.p = (a); \
1333 : ut.s = (b); \
1334 : ut.r = (c); \
1335 : utrace(&ut, sizeof(ut)); \
1336 : }
1337 : #else
1338 : #define UTRACE(a, b, c)
1339 : #endif
1340 :
1341 : /******************************************************************************/
1342 : /*
1343 : * Begin function prototypes for non-inline static functions.
1344 : */
1345 :
1346 : static char *umax2s(uintmax_t x, unsigned base, char *s);
1347 : static bool malloc_mutex_init(malloc_mutex_t *mutex);
1348 : static bool malloc_spin_init(malloc_spinlock_t *lock);
1349 : static void wrtmessage(const char *p1, const char *p2, const char *p3,
1350 : const char *p4);
1351 : #ifdef MALLOC_STATS
1352 : #ifdef MOZ_MEMORY_DARWIN
1353 : /* Avoid namespace collision with OS X's malloc APIs. */
1354 : #define malloc_printf moz_malloc_printf
1355 : #endif
1356 : static void malloc_printf(const char *format, ...);
1357 : #endif
1358 : static bool base_pages_alloc_mmap(size_t minsize);
1359 : static bool base_pages_alloc(size_t minsize);
1360 : static void *base_alloc(size_t size);
1361 : static void *base_calloc(size_t number, size_t size);
1362 : static extent_node_t *base_node_alloc(void);
1363 : static void base_node_dealloc(extent_node_t *node);
1364 : #ifdef MALLOC_STATS
1365 : static void stats_print(arena_t *arena);
1366 : #endif
1367 : static void *pages_map(void *addr, size_t size, int pfd);
1368 : static void pages_unmap(void *addr, size_t size);
1369 : static void *chunk_alloc_mmap(size_t size, bool pagefile);
1370 : #ifdef MALLOC_PAGEFILE
1371 : static int pagefile_init(size_t size);
1372 : static void pagefile_close(int pfd);
1373 : #endif
1374 : static void *chunk_alloc(size_t size, bool zero, bool pagefile);
1375 : static void chunk_dealloc_mmap(void *chunk, size_t size);
1376 : static void chunk_dealloc(void *chunk, size_t size);
1377 : #ifndef NO_TLS
1378 : static arena_t *choose_arena_hard(void);
1379 : #endif
1380 : static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
1381 : bool large, bool zero);
1382 : static void arena_chunk_init(arena_t *arena, arena_chunk_t *chunk);
1383 : static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
1384 : static arena_run_t *arena_run_alloc(arena_t *arena, arena_bin_t *bin,
1385 : size_t size, bool large, bool zero);
1386 : static void arena_purge(arena_t *arena);
1387 : static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
1388 : static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
1389 : arena_run_t *run, size_t oldsize, size_t newsize);
1390 : static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
1391 : arena_run_t *run, size_t oldsize, size_t newsize, bool dirty);
1392 : static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
1393 : static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
1394 : static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
1395 : #ifdef MALLOC_BALANCE
1396 : static void arena_lock_balance_hard(arena_t *arena);
1397 : #endif
1398 : static void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
1399 : static void *arena_palloc(arena_t *arena, size_t alignment, size_t size,
1400 : size_t alloc_size);
1401 : static size_t arena_salloc(const void *ptr);
1402 : static void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk,
1403 : void *ptr);
1404 : static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
1405 : void *ptr, size_t size, size_t oldsize);
1406 : static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk,
1407 : void *ptr, size_t size, size_t oldsize);
1408 : static bool arena_ralloc_large(void *ptr, size_t size, size_t oldsize);
1409 : static void *arena_ralloc(void *ptr, size_t size, size_t oldsize);
1410 : static bool arena_new(arena_t *arena);
1411 : static arena_t *arenas_extend(unsigned ind);
1412 : static void *huge_malloc(size_t size, bool zero);
1413 : static void *huge_palloc(size_t alignment, size_t size);
1414 : static void *huge_ralloc(void *ptr, size_t size, size_t oldsize);
1415 : static void huge_dalloc(void *ptr);
1416 : static void malloc_print_stats(void);
1417 : #ifndef MOZ_MEMORY_WINDOWS
1418 : static
1419 : #endif
1420 : bool malloc_init_hard(void);
1421 :
1422 : #ifdef MOZ_MEMORY_ANDROID
1423 : void _malloc_prefork(void);
1424 : void _malloc_postfork(void);
1425 : #else
1426 : static void _malloc_prefork(void);
1427 : static void _malloc_postfork(void);
1428 : #endif
1429 :
1430 : #ifdef MOZ_MEMORY_DARWIN
1431 : /*
1432 : * MALLOC_ZONE_T_NOTE
1433 : *
1434 : * On Darwin, we hook into the memory allocator using a malloc_zone_t struct.
1435 : * We must be very careful around this struct because of different behaviour on
1436 : * different versions of OSX.
1437 : *
1438 : * Each of OSX 10.5, 10.6 and 10.7 use different versions of the struct
1439 : * (with version numbers 3, 6 and 8 respectively). The binary we use on each of
1440 : * these platforms will not necessarily be built using the correct SDK [1].
1441 : * This means we need to statically know the correct struct size to use on all
1442 : * OSX releases, and have a fallback for unknown future versions. The struct
1443 : * sizes defined in osx_zone_types.h.
1444 : *
1445 : * For OSX 10.8 and later, we may expect the malloc_zone_t struct to change
1446 : * again, and need to dynamically account for this. By simply leaving
1447 : * malloc_zone_t alone, we don't quite deal with the problem, because there
1448 : * remain calls to jemalloc through the mozalloc interface. We check this
1449 : * dynamically on each allocation, using the CHECK_DARWIN macro and
1450 : * osx_use_jemalloc.
1451 : *
1452 : *
1453 : * [1] Mozilla is built as a universal binary on Mac, supporting i386 and
1454 : * x86_64. The i386 target is built using the 10.5 SDK, even if it runs on
1455 : * 10.6. The x86_64 target is built using the 10.6 SDK, even if it runs on
1456 : * 10.7 or later, or 10.5.
1457 : *
1458 : * FIXME:
1459 : * When later versions of OSX come out (10.8 and up), we need to check their
1460 : * malloc_zone_t versions. If they're greater than 8, we need a new version
1461 : * of malloc_zone_t adapted into osx_zone_types.h.
1462 : */
1463 :
1464 : #include "osx_zone_types.h"
1465 :
1466 : #define LEOPARD_MALLOC_ZONE_T_VERSION 3
1467 : #define SNOW_LEOPARD_MALLOC_ZONE_T_VERSION 6
1468 : #define LION_MALLOC_ZONE_T_VERSION 8
1469 :
1470 : static bool osx_use_jemalloc = false;
1471 :
1472 :
1473 : static lion_malloc_zone l_szone;
1474 : static malloc_zone_t * szone = (malloc_zone_t*)(&l_szone);
1475 :
1476 : static lion_malloc_introspection l_ozone_introspect;
1477 : static malloc_introspection_t * const ozone_introspect =
1478 : (malloc_introspection_t*)(&l_ozone_introspect);
1479 : static void szone2ozone(malloc_zone_t *zone, size_t size);
1480 : static size_t zone_version_size(int version);
1481 : #endif
1482 :
1483 : /*
1484 : * End function prototypes.
1485 : */
1486 : /******************************************************************************/
1487 :
1488 : /*
1489 : * umax2s() provides minimal integer printing functionality, which is
1490 : * especially useful for situations where allocation in vsnprintf() calls would
1491 : * potentially cause deadlock.
1492 : */
1493 : #define UMAX2S_BUFSIZE 65
1494 : char *
1495 0 : umax2s(uintmax_t x, unsigned base, char *s)
1496 : {
1497 : unsigned i;
1498 :
1499 0 : i = UMAX2S_BUFSIZE - 1;
1500 0 : s[i] = '\0';
1501 0 : switch (base) {
1502 : case 10:
1503 : do {
1504 0 : i--;
1505 0 : s[i] = "0123456789"[x % 10];
1506 0 : x /= 10;
1507 0 : } while (x > 0);
1508 0 : break;
1509 : case 16:
1510 : do {
1511 0 : i--;
1512 0 : s[i] = "0123456789abcdef"[x & 0xf];
1513 0 : x >>= 4;
1514 0 : } while (x > 0);
1515 0 : break;
1516 : default:
1517 : do {
1518 0 : i--;
1519 0 : s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % base];
1520 0 : x /= base;
1521 0 : } while (x > 0);
1522 : }
1523 :
1524 0 : return (&s[i]);
1525 : }
1526 :
1527 : static void
1528 0 : wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
1529 : {
1530 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_WINDOWS)
1531 : #define _write write
1532 : #endif
1533 0 : _write(STDERR_FILENO, p1, (unsigned int) strlen(p1));
1534 0 : _write(STDERR_FILENO, p2, (unsigned int) strlen(p2));
1535 0 : _write(STDERR_FILENO, p3, (unsigned int) strlen(p3));
1536 0 : _write(STDERR_FILENO, p4, (unsigned int) strlen(p4));
1537 0 : }
1538 :
1539 : #define _malloc_message malloc_message
1540 :
1541 : void (*_malloc_message)(const char *p1, const char *p2, const char *p3,
1542 : const char *p4) = wrtmessage;
1543 :
1544 : #ifdef MALLOC_DEBUG
1545 : # define assert(e) do { \
1546 : if (!(e)) { \
1547 : char line_buf[UMAX2S_BUFSIZE]; \
1548 : _malloc_message(__FILE__, ":", umax2s(__LINE__, 10, \
1549 : line_buf), ": Failed assertion: "); \
1550 : _malloc_message("\"", #e, "\"\n", ""); \
1551 : abort(); \
1552 : } \
1553 : } while (0)
1554 : #else
1555 : #define assert(e)
1556 : #endif
1557 :
1558 : /******************************************************************************/
1559 : /*
1560 : * Begin mutex. We can't use normal pthread mutexes in all places, because
1561 : * they require malloc()ed memory, which causes bootstrapping issues in some
1562 : * cases.
1563 : */
1564 :
1565 : static bool
1566 99505 : malloc_mutex_init(malloc_mutex_t *mutex)
1567 : {
1568 : #if defined(MOZ_MEMORY_WINDOWS)
1569 : if (__isthreaded)
1570 : if (! __crtInitCritSecAndSpinCount(mutex, _CRT_SPINCOUNT))
1571 : return (true);
1572 : #elif defined(MOZ_MEMORY_DARWIN)
1573 : mutex->lock = OS_SPINLOCK_INIT;
1574 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1575 : pthread_mutexattr_t attr;
1576 99505 : if (pthread_mutexattr_init(&attr) != 0)
1577 0 : return (true);
1578 99505 : pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
1579 99505 : if (pthread_mutex_init(mutex, &attr) != 0) {
1580 0 : pthread_mutexattr_destroy(&attr);
1581 0 : return (true);
1582 : }
1583 99505 : pthread_mutexattr_destroy(&attr);
1584 : #elif defined(MOZ_MEMORY)
1585 : if (pthread_mutex_init(mutex, NULL) != 0)
1586 : return (true);
1587 : #else
1588 : static const spinlock_t lock = _SPINLOCK_INITIALIZER;
1589 :
1590 : mutex->lock = lock;
1591 : #endif
1592 99505 : return (false);
1593 : }
1594 :
1595 : static inline void
1596 261618339 : malloc_mutex_lock(malloc_mutex_t *mutex)
1597 : {
1598 :
1599 : #if defined(MOZ_MEMORY_WINDOWS)
1600 : EnterCriticalSection(mutex);
1601 : #elif defined(MOZ_MEMORY_DARWIN)
1602 : OSSpinLockLock(&mutex->lock);
1603 : #elif defined(MOZ_MEMORY)
1604 261618339 : pthread_mutex_lock(mutex);
1605 : #else
1606 : if (__isthreaded)
1607 : _SPINLOCK(&mutex->lock);
1608 : #endif
1609 261646910 : }
1610 :
1611 : static inline void
1612 261645882 : malloc_mutex_unlock(malloc_mutex_t *mutex)
1613 : {
1614 :
1615 : #if defined(MOZ_MEMORY_WINDOWS)
1616 : LeaveCriticalSection(mutex);
1617 : #elif defined(MOZ_MEMORY_DARWIN)
1618 : OSSpinLockUnlock(&mutex->lock);
1619 : #elif defined(MOZ_MEMORY)
1620 261645882 : pthread_mutex_unlock(mutex);
1621 : #else
1622 : if (__isthreaded)
1623 : _SPINUNLOCK(&mutex->lock);
1624 : #endif
1625 261645682 : }
1626 :
1627 : static bool
1628 0 : malloc_spin_init(malloc_spinlock_t *lock)
1629 : {
1630 : #if defined(MOZ_MEMORY_WINDOWS)
1631 : if (__isthreaded)
1632 : if (! __crtInitCritSecAndSpinCount(lock, _CRT_SPINCOUNT))
1633 : return (true);
1634 : #elif defined(MOZ_MEMORY_DARWIN)
1635 : lock->lock = OS_SPINLOCK_INIT;
1636 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1637 : pthread_mutexattr_t attr;
1638 0 : if (pthread_mutexattr_init(&attr) != 0)
1639 0 : return (true);
1640 0 : pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
1641 0 : if (pthread_mutex_init(lock, &attr) != 0) {
1642 0 : pthread_mutexattr_destroy(&attr);
1643 0 : return (true);
1644 : }
1645 0 : pthread_mutexattr_destroy(&attr);
1646 : #elif defined(MOZ_MEMORY)
1647 : if (pthread_mutex_init(lock, NULL) != 0)
1648 : return (true);
1649 : #else
1650 : lock->lock = _SPINLOCK_INITIALIZER;
1651 : #endif
1652 0 : return (false);
1653 : }
1654 :
1655 : static inline void
1656 0 : malloc_spin_lock(malloc_spinlock_t *lock)
1657 : {
1658 :
1659 : #if defined(MOZ_MEMORY_WINDOWS)
1660 : EnterCriticalSection(lock);
1661 : #elif defined(MOZ_MEMORY_DARWIN)
1662 : OSSpinLockLock(&lock->lock);
1663 : #elif defined(MOZ_MEMORY)
1664 0 : pthread_mutex_lock(lock);
1665 : #else
1666 : if (__isthreaded)
1667 : _SPINLOCK(&lock->lock);
1668 : #endif
1669 0 : }
1670 :
1671 : static inline void
1672 0 : malloc_spin_unlock(malloc_spinlock_t *lock)
1673 : {
1674 : #if defined(MOZ_MEMORY_WINDOWS)
1675 : LeaveCriticalSection(lock);
1676 : #elif defined(MOZ_MEMORY_DARWIN)
1677 : OSSpinLockUnlock(&lock->lock);
1678 : #elif defined(MOZ_MEMORY)
1679 0 : pthread_mutex_unlock(lock);
1680 : #else
1681 : if (__isthreaded)
1682 : _SPINUNLOCK(&lock->lock);
1683 : #endif
1684 0 : }
1685 :
1686 : /*
1687 : * End mutex.
1688 : */
1689 : /******************************************************************************/
1690 : /*
1691 : * Begin spin lock. Spin locks here are actually adaptive mutexes that block
1692 : * after a period of spinning, because unbounded spinning would allow for
1693 : * priority inversion.
1694 : */
1695 :
1696 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_DARWIN)
1697 : # define malloc_spin_init malloc_mutex_init
1698 : # define malloc_spin_lock malloc_mutex_lock
1699 : # define malloc_spin_unlock malloc_mutex_unlock
1700 : #endif
1701 :
1702 : #ifndef MOZ_MEMORY
1703 : /*
1704 : * We use an unpublished interface to initialize pthread mutexes with an
1705 : * allocation callback, in order to avoid infinite recursion.
1706 : */
1707 : int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
1708 : void *(calloc_cb)(size_t, size_t));
1709 :
1710 : __weak_reference(_pthread_mutex_init_calloc_cb_stub,
1711 : _pthread_mutex_init_calloc_cb);
1712 :
1713 : int
1714 : _pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
1715 : void *(calloc_cb)(size_t, size_t))
1716 : {
1717 :
1718 : return (0);
1719 : }
1720 :
1721 : static bool
1722 : malloc_spin_init(pthread_mutex_t *lock)
1723 : {
1724 :
1725 : if (_pthread_mutex_init_calloc_cb(lock, base_calloc) != 0)
1726 : return (true);
1727 :
1728 : return (false);
1729 : }
1730 :
1731 : static inline unsigned
1732 : malloc_spin_lock(pthread_mutex_t *lock)
1733 : {
1734 : unsigned ret = 0;
1735 :
1736 : if (__isthreaded) {
1737 : if (_pthread_mutex_trylock(lock) != 0) {
1738 : unsigned i;
1739 : volatile unsigned j;
1740 :
1741 : /* Exponentially back off. */
1742 : for (i = 1; i <= SPIN_LIMIT_2POW; i++) {
1743 : for (j = 0; j < (1U << i); j++)
1744 : ret++;
1745 :
1746 : CPU_SPINWAIT;
1747 : if (_pthread_mutex_trylock(lock) == 0)
1748 : return (ret);
1749 : }
1750 :
1751 : /*
1752 : * Spinning failed. Block until the lock becomes
1753 : * available, in order to avoid indefinite priority
1754 : * inversion.
1755 : */
1756 : _pthread_mutex_lock(lock);
1757 : assert((ret << BLOCK_COST_2POW) != 0);
1758 : return (ret << BLOCK_COST_2POW);
1759 : }
1760 : }
1761 :
1762 : return (ret);
1763 : }
1764 :
1765 : static inline void
1766 : malloc_spin_unlock(pthread_mutex_t *lock)
1767 : {
1768 :
1769 : if (__isthreaded)
1770 : _pthread_mutex_unlock(lock);
1771 : }
1772 : #endif
1773 :
1774 : /*
1775 : * End spin lock.
1776 : */
1777 : /******************************************************************************/
1778 : /*
1779 : * Begin Utility functions/macros.
1780 : */
1781 :
1782 : /* Return the chunk address for allocation address a. */
1783 : #define CHUNK_ADDR2BASE(a) \
1784 : ((void *)((uintptr_t)(a) & ~chunksize_mask))
1785 :
1786 : /* Return the chunk offset of address a. */
1787 : #define CHUNK_ADDR2OFFSET(a) \
1788 : ((size_t)((uintptr_t)(a) & chunksize_mask))
1789 :
1790 : /* Return the smallest chunk multiple that is >= s. */
1791 : #define CHUNK_CEILING(s) \
1792 : (((s) + chunksize_mask) & ~chunksize_mask)
1793 :
1794 : /* Return the smallest cacheline multiple that is >= s. */
1795 : #define CACHELINE_CEILING(s) \
1796 : (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1))
1797 :
1798 : /* Return the smallest quantum multiple that is >= a. */
1799 : #define QUANTUM_CEILING(a) \
1800 : (((a) + quantum_mask) & ~quantum_mask)
1801 :
1802 : /* Return the smallest pagesize multiple that is >= s. */
1803 : #define PAGE_CEILING(s) \
1804 : (((s) + pagesize_mask) & ~pagesize_mask)
1805 :
1806 : /* Compute the smallest power of 2 that is >= x. */
1807 : static inline size_t
1808 32082902 : pow2_ceil(size_t x)
1809 : {
1810 :
1811 32082902 : x--;
1812 32082902 : x |= x >> 1;
1813 32082902 : x |= x >> 2;
1814 32082902 : x |= x >> 4;
1815 32082902 : x |= x >> 8;
1816 32082902 : x |= x >> 16;
1817 : #if (SIZEOF_PTR == 8)
1818 : x |= x >> 32;
1819 : #endif
1820 32082902 : x++;
1821 32082902 : return (x);
1822 : }
1823 :
1824 : #ifdef MALLOC_BALANCE
1825 : /*
1826 : * Use a simple linear congruential pseudo-random number generator:
1827 : *
1828 : * prn(y) = (a*x + c) % m
1829 : *
1830 : * where the following constants ensure maximal period:
1831 : *
1832 : * a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4.
1833 : * c == Odd number (relatively prime to 2^n).
1834 : * m == 2^32
1835 : *
1836 : * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
1837 : *
1838 : * This choice of m has the disadvantage that the quality of the bits is
1839 : * proportional to bit position. For example. the lowest bit has a cycle of 2,
1840 : * the next has a cycle of 4, etc. For this reason, we prefer to use the upper
1841 : * bits.
1842 : */
1843 : # define PRN_DEFINE(suffix, var, a, c) \
1844 : static inline void \
1845 : sprn_##suffix(uint32_t seed) \
1846 : { \
1847 : var = seed; \
1848 : } \
1849 : \
1850 : static inline uint32_t \
1851 : prn_##suffix(uint32_t lg_range) \
1852 : { \
1853 : uint32_t ret, x; \
1854 : \
1855 : assert(lg_range > 0); \
1856 : assert(lg_range <= 32); \
1857 : \
1858 : x = (var * (a)) + (c); \
1859 : var = x; \
1860 : ret = x >> (32 - lg_range); \
1861 : \
1862 : return (ret); \
1863 : }
1864 : # define SPRN(suffix, seed) sprn_##suffix(seed)
1865 : # define PRN(suffix, lg_range) prn_##suffix(lg_range)
1866 : #endif
1867 :
1868 : #ifdef MALLOC_BALANCE
1869 : /* Define the PRNG used for arena assignment. */
1870 : static __thread uint32_t balance_x;
1871 : PRN_DEFINE(balance, balance_x, 1297, 1301)
1872 : #endif
1873 :
1874 : #ifdef MALLOC_UTRACE
1875 : static int
1876 0 : utrace(const void *addr, size_t len)
1877 : {
1878 0 : malloc_utrace_t *ut = (malloc_utrace_t *)addr;
1879 : char buf_a[UMAX2S_BUFSIZE];
1880 : char buf_b[UMAX2S_BUFSIZE];
1881 :
1882 0 : assert(len == sizeof(malloc_utrace_t));
1883 :
1884 0 : if (ut->p == NULL && ut->s == 0 && ut->r == NULL) {
1885 0 : _malloc_message(
1886 0 : umax2s(getpid(), 10, buf_a),
1887 : " x USER malloc_init()\n", "", "");
1888 0 : } else if (ut->p == NULL && ut->r != NULL) {
1889 0 : _malloc_message(
1890 0 : umax2s(getpid(), 10, buf_a),
1891 : " x USER 0x",
1892 0 : umax2s((uintptr_t)ut->r, 16, buf_b),
1893 : " = malloc(");
1894 0 : _malloc_message(
1895 0 : umax2s(ut->s, 10, buf_a),
1896 : ")\n", "", "");
1897 0 : } else if (ut->p != NULL && ut->r != NULL) {
1898 0 : _malloc_message(
1899 0 : umax2s(getpid(), 10, buf_a),
1900 : " x USER 0x",
1901 0 : umax2s((uintptr_t)ut->r, 16, buf_b),
1902 : " = realloc(0x");
1903 0 : _malloc_message(
1904 0 : umax2s((uintptr_t)ut->p, 16, buf_a),
1905 : ", ",
1906 0 : umax2s(ut->s, 10, buf_b),
1907 : ")\n");
1908 : } else {
1909 0 : _malloc_message(
1910 0 : umax2s(getpid(), 10, buf_a),
1911 : " x USER free(0x",
1912 0 : umax2s((uintptr_t)ut->p, 16, buf_b),
1913 : ")\n");
1914 : }
1915 :
1916 0 : return (0);
1917 : }
1918 : #endif
1919 :
1920 : static inline const char *
1921 0 : _getprogname(void)
1922 : {
1923 :
1924 0 : return ("<jemalloc>");
1925 : }
1926 :
1927 : #ifdef MALLOC_STATS
1928 : /*
1929 : * Print to stderr in such a way as to (hopefully) avoid memory allocation.
1930 : */
1931 : static void
1932 0 : malloc_printf(const char *format, ...)
1933 : {
1934 : char buf[4096];
1935 : va_list ap;
1936 :
1937 0 : va_start(ap, format);
1938 0 : vsnprintf(buf, sizeof(buf), format, ap);
1939 0 : va_end(ap);
1940 0 : _malloc_message(buf, "", "", "");
1941 0 : }
1942 : #endif
1943 :
1944 : /******************************************************************************/
1945 :
1946 : static inline void
1947 0 : pages_decommit(void *addr, size_t size)
1948 : {
1949 :
1950 : #ifdef MOZ_MEMORY_WINDOWS
1951 : VirtualFree(addr, size, MEM_DECOMMIT);
1952 : #else
1953 0 : if (mmap(addr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1,
1954 : 0) == MAP_FAILED)
1955 0 : abort();
1956 : #endif
1957 0 : }
1958 :
1959 : static inline void
1960 0 : pages_commit(void *addr, size_t size)
1961 : {
1962 :
1963 : # ifdef MOZ_MEMORY_WINDOWS
1964 : VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
1965 : # else
1966 0 : if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE |
1967 : MAP_ANON, -1, 0) == MAP_FAILED)
1968 0 : abort();
1969 : # endif
1970 0 : }
1971 :
1972 : static bool
1973 19901 : base_pages_alloc_mmap(size_t minsize)
1974 : {
1975 : bool ret;
1976 : size_t csize;
1977 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1978 : size_t pminsize;
1979 : #endif
1980 : int pfd;
1981 :
1982 19901 : assert(minsize != 0);
1983 19901 : csize = CHUNK_CEILING(minsize);
1984 : #ifdef MALLOC_PAGEFILE
1985 : if (opt_pagefile) {
1986 : pfd = pagefile_init(csize);
1987 : if (pfd == -1)
1988 : return (true);
1989 : } else
1990 : #endif
1991 19901 : pfd = -1;
1992 19901 : base_pages = pages_map(NULL, csize, pfd);
1993 19901 : if (base_pages == NULL) {
1994 0 : ret = true;
1995 0 : goto RETURN;
1996 : }
1997 19901 : base_next_addr = base_pages;
1998 19901 : base_past_addr = (void *)((uintptr_t)base_pages + csize);
1999 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
2000 : /*
2001 : * Leave enough pages for minsize committed, since otherwise they would
2002 : * have to be immediately recommitted.
2003 : */
2004 19901 : pminsize = PAGE_CEILING(minsize);
2005 19901 : base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize);
2006 : # if defined(MALLOC_DECOMMIT)
2007 : if (pminsize < csize)
2008 : pages_decommit(base_next_decommitted, csize - pminsize);
2009 : # endif
2010 : # ifdef MALLOC_STATS
2011 19901 : base_mapped += csize;
2012 19901 : base_committed += pminsize;
2013 : # endif
2014 : #endif
2015 :
2016 19901 : ret = false;
2017 : RETURN:
2018 : #ifdef MALLOC_PAGEFILE
2019 : if (pfd != -1)
2020 : pagefile_close(pfd);
2021 : #endif
2022 19901 : return (false);
2023 : }
2024 :
2025 : static bool
2026 19901 : base_pages_alloc(size_t minsize)
2027 : {
2028 :
2029 19901 : if (base_pages_alloc_mmap(minsize) == false)
2030 19901 : return (false);
2031 :
2032 0 : return (true);
2033 : }
2034 :
2035 : static void *
2036 80680 : base_alloc(size_t size)
2037 : {
2038 : void *ret;
2039 : size_t csize;
2040 :
2041 : /* Round size up to nearest multiple of the cacheline size. */
2042 80680 : csize = CACHELINE_CEILING(size);
2043 :
2044 80680 : malloc_mutex_lock(&base_mtx);
2045 : /* Make sure there's enough space for the allocation. */
2046 80680 : if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
2047 19901 : if (base_pages_alloc(csize)) {
2048 0 : malloc_mutex_unlock(&base_mtx);
2049 0 : return (NULL);
2050 : }
2051 : }
2052 : /* Allocate. */
2053 80680 : ret = base_next_addr;
2054 80680 : base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
2055 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
2056 : /* Make sure enough pages are committed for the new allocation. */
2057 80680 : if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) {
2058 19901 : void *pbase_next_addr =
2059 19901 : (void *)(PAGE_CEILING((uintptr_t)base_next_addr));
2060 :
2061 : # ifdef MALLOC_DECOMMIT
2062 : pages_commit(base_next_decommitted, (uintptr_t)pbase_next_addr -
2063 : (uintptr_t)base_next_decommitted);
2064 : # endif
2065 19901 : base_next_decommitted = pbase_next_addr;
2066 : # ifdef MALLOC_STATS
2067 39802 : base_committed += (uintptr_t)pbase_next_addr -
2068 19901 : (uintptr_t)base_next_decommitted;
2069 : # endif
2070 : }
2071 : #endif
2072 80680 : malloc_mutex_unlock(&base_mtx);
2073 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
2074 :
2075 80680 : return (ret);
2076 : }
2077 :
2078 : static void *
2079 39802 : base_calloc(size_t number, size_t size)
2080 : {
2081 : void *ret;
2082 :
2083 39802 : ret = base_alloc(number * size);
2084 : #ifdef MALLOC_VALGRIND
2085 : if (ret != NULL) {
2086 : VALGRIND_FREELIKE_BLOCK(ret, 0);
2087 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, true);
2088 : }
2089 : #endif
2090 39802 : memset(ret, 0, number * size);
2091 :
2092 39802 : return (ret);
2093 : }
2094 :
2095 : static extent_node_t *
2096 5538 : base_node_alloc(void)
2097 : {
2098 : extent_node_t *ret;
2099 :
2100 5538 : malloc_mutex_lock(&base_mtx);
2101 5538 : if (base_nodes != NULL) {
2102 4462 : ret = base_nodes;
2103 4462 : base_nodes = *(extent_node_t **)ret;
2104 : VALGRIND_FREELIKE_BLOCK(ret, 0);
2105 : VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(extent_node_t), 0, false);
2106 4462 : malloc_mutex_unlock(&base_mtx);
2107 : } else {
2108 1076 : malloc_mutex_unlock(&base_mtx);
2109 1076 : ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
2110 : }
2111 :
2112 5538 : return (ret);
2113 : }
2114 :
2115 : static void
2116 5538 : base_node_dealloc(extent_node_t *node)
2117 : {
2118 :
2119 5538 : malloc_mutex_lock(&base_mtx);
2120 : VALGRIND_FREELIKE_BLOCK(node, 0);
2121 : VALGRIND_MALLOCLIKE_BLOCK(node, sizeof(extent_node_t *), 0, false);
2122 5538 : *(extent_node_t **)node = base_nodes;
2123 5538 : base_nodes = node;
2124 5538 : malloc_mutex_unlock(&base_mtx);
2125 5538 : }
2126 :
2127 : /******************************************************************************/
2128 :
2129 : #ifdef MALLOC_STATS
2130 : static void
2131 0 : stats_print(arena_t *arena)
2132 : {
2133 : unsigned i, gap_start;
2134 :
2135 : #ifdef MOZ_MEMORY_WINDOWS
2136 : malloc_printf("dirty: %Iu page%s dirty, %I64u sweep%s,"
2137 : " %I64u madvise%s, %I64u page%s purged\n",
2138 : arena->ndirty, arena->ndirty == 1 ? "" : "s",
2139 : arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
2140 : arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
2141 : arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
2142 : # ifdef MALLOC_DECOMMIT
2143 : malloc_printf("decommit: %I64u decommit%s, %I64u commit%s,"
2144 : " %I64u page%s decommitted\n",
2145 : arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
2146 : arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
2147 : arena->stats.decommitted,
2148 : (arena->stats.decommitted == 1) ? "" : "s");
2149 : # endif
2150 :
2151 : malloc_printf(" allocated nmalloc ndalloc\n");
2152 : malloc_printf("small: %12Iu %12I64u %12I64u\n",
2153 : arena->stats.allocated_small, arena->stats.nmalloc_small,
2154 : arena->stats.ndalloc_small);
2155 : malloc_printf("large: %12Iu %12I64u %12I64u\n",
2156 : arena->stats.allocated_large, arena->stats.nmalloc_large,
2157 : arena->stats.ndalloc_large);
2158 : malloc_printf("total: %12Iu %12I64u %12I64u\n",
2159 : arena->stats.allocated_small + arena->stats.allocated_large,
2160 : arena->stats.nmalloc_small + arena->stats.nmalloc_large,
2161 : arena->stats.ndalloc_small + arena->stats.ndalloc_large);
2162 : malloc_printf("mapped: %12Iu\n", arena->stats.mapped);
2163 : #else
2164 0 : malloc_printf("dirty: %zu page%s dirty, %llu sweep%s,"
2165 : " %llu madvise%s, %llu page%s purged\n",
2166 0 : arena->ndirty, arena->ndirty == 1 ? "" : "s",
2167 0 : arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
2168 0 : arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
2169 0 : arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
2170 : # ifdef MALLOC_DECOMMIT
2171 : malloc_printf("decommit: %llu decommit%s, %llu commit%s,"
2172 : " %llu page%s decommitted\n",
2173 : arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
2174 : arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
2175 : arena->stats.decommitted,
2176 : (arena->stats.decommitted == 1) ? "" : "s");
2177 : # endif
2178 :
2179 0 : malloc_printf(" allocated nmalloc ndalloc\n");
2180 0 : malloc_printf("small: %12zu %12llu %12llu\n",
2181 : arena->stats.allocated_small, arena->stats.nmalloc_small,
2182 : arena->stats.ndalloc_small);
2183 0 : malloc_printf("large: %12zu %12llu %12llu\n",
2184 : arena->stats.allocated_large, arena->stats.nmalloc_large,
2185 : arena->stats.ndalloc_large);
2186 0 : malloc_printf("total: %12zu %12llu %12llu\n",
2187 0 : arena->stats.allocated_small + arena->stats.allocated_large,
2188 0 : arena->stats.nmalloc_small + arena->stats.nmalloc_large,
2189 0 : arena->stats.ndalloc_small + arena->stats.ndalloc_large);
2190 0 : malloc_printf("mapped: %12zu\n", arena->stats.mapped);
2191 : #endif
2192 0 : malloc_printf("bins: bin size regs pgs requests newruns"
2193 : " reruns maxruns curruns\n");
2194 0 : for (i = 0, gap_start = UINT_MAX; i < ntbins + nqbins + nsbins; i++) {
2195 0 : if (arena->bins[i].stats.nrequests == 0) {
2196 0 : if (gap_start == UINT_MAX)
2197 0 : gap_start = i;
2198 : } else {
2199 0 : if (gap_start != UINT_MAX) {
2200 0 : if (i > gap_start + 1) {
2201 : /* Gap of more than one size class. */
2202 0 : malloc_printf("[%u..%u]\n",
2203 : gap_start, i - 1);
2204 : } else {
2205 : /* Gap of one size class. */
2206 0 : malloc_printf("[%u]\n", gap_start);
2207 : }
2208 0 : gap_start = UINT_MAX;
2209 : }
2210 0 : malloc_printf(
2211 : #if defined(MOZ_MEMORY_WINDOWS)
2212 : "%13u %1s %4u %4u %3u %9I64u %9I64u"
2213 : " %9I64u %7u %7u\n",
2214 : #else
2215 : "%13u %1s %4u %4u %3u %9llu %9llu"
2216 : " %9llu %7lu %7lu\n",
2217 : #endif
2218 : i,
2219 0 : i < ntbins ? "T" : i < ntbins + nqbins ? "Q" : "S",
2220 : arena->bins[i].reg_size,
2221 : arena->bins[i].nregs,
2222 0 : arena->bins[i].run_size >> pagesize_2pow,
2223 : arena->bins[i].stats.nrequests,
2224 : arena->bins[i].stats.nruns,
2225 : arena->bins[i].stats.reruns,
2226 : arena->bins[i].stats.highruns,
2227 : arena->bins[i].stats.curruns);
2228 : }
2229 : }
2230 0 : if (gap_start != UINT_MAX) {
2231 0 : if (i > gap_start + 1) {
2232 : /* Gap of more than one size class. */
2233 0 : malloc_printf("[%u..%u]\n", gap_start, i - 1);
2234 : } else {
2235 : /* Gap of one size class. */
2236 0 : malloc_printf("[%u]\n", gap_start);
2237 : }
2238 : }
2239 0 : }
2240 : #endif
2241 :
2242 : /*
2243 : * End Utility functions/macros.
2244 : */
2245 : /******************************************************************************/
2246 : /*
2247 : * Begin extent tree code.
2248 : */
2249 :
2250 : static inline int
2251 0 : extent_szad_comp(extent_node_t *a, extent_node_t *b)
2252 : {
2253 : int ret;
2254 0 : size_t a_size = a->size;
2255 0 : size_t b_size = b->size;
2256 :
2257 0 : ret = (a_size > b_size) - (a_size < b_size);
2258 0 : if (ret == 0) {
2259 0 : uintptr_t a_addr = (uintptr_t)a->addr;
2260 0 : uintptr_t b_addr = (uintptr_t)b->addr;
2261 :
2262 0 : ret = (a_addr > b_addr) - (a_addr < b_addr);
2263 : }
2264 :
2265 0 : return (ret);
2266 : }
2267 :
2268 : /* Wrap red-black tree macros in functions. */
2269 0 : rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
2270 0 : link_szad, extent_szad_comp)
2271 :
2272 : static inline int
2273 33113 : extent_ad_comp(extent_node_t *a, extent_node_t *b)
2274 : {
2275 33113 : uintptr_t a_addr = (uintptr_t)a->addr;
2276 33113 : uintptr_t b_addr = (uintptr_t)b->addr;
2277 :
2278 33113 : return ((a_addr > b_addr) - (a_addr < b_addr));
2279 : }
2280 :
2281 : /* Wrap red-black tree macros in functions. */
2282 39791 : rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
2283 39791 : extent_ad_comp)
2284 :
2285 : /*
2286 : * End extent tree code.
2287 : */
2288 : /******************************************************************************/
2289 : /*
2290 : * Begin chunk management functions.
2291 : */
2292 :
2293 : #ifdef MOZ_MEMORY_WINDOWS
2294 :
2295 : static void *
2296 : pages_map(void *addr, size_t size, int pfd)
2297 : {
2298 : void *ret = NULL;
2299 : ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
2300 : PAGE_READWRITE);
2301 : return (ret);
2302 : }
2303 :
2304 : static void
2305 : pages_unmap(void *addr, size_t size)
2306 : {
2307 : if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
2308 : _malloc_message(_getprogname(),
2309 : ": (malloc) Error in VirtualFree()\n", "", "");
2310 : if (opt_abort)
2311 : abort();
2312 : }
2313 : }
2314 : #else
2315 : #ifdef JEMALLOC_USES_MAP_ALIGN
2316 : static void *
2317 : pages_map_align(size_t size, int pfd, size_t alignment)
2318 : {
2319 : void *ret;
2320 :
2321 : /*
2322 : * We don't use MAP_FIXED here, because it can cause the *replacement*
2323 : * of existing mappings, and we only want to create new mappings.
2324 : */
2325 : #ifdef MALLOC_PAGEFILE
2326 : if (pfd != -1) {
2327 : ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
2328 : MAP_NOSYNC | MAP_ALIGN, pfd, 0);
2329 : } else
2330 : #endif
2331 : {
2332 : ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
2333 : MAP_NOSYNC | MAP_ALIGN | MAP_ANON, -1, 0);
2334 : }
2335 : assert(ret != NULL);
2336 :
2337 : if (ret == MAP_FAILED)
2338 : ret = NULL;
2339 : return (ret);
2340 : }
2341 : #endif
2342 :
2343 : static void *
2344 82582 : pages_map(void *addr, size_t size, int pfd)
2345 : {
2346 : void *ret;
2347 : #if defined(__ia64__)
2348 : /*
2349 : * The JS engine assumes that all allocated pointers have their high 17 bits clear,
2350 : * which ia64's mmap doesn't support directly. However, we can emulate it by passing
2351 : * mmap an "addr" parameter with those bits clear. The mmap will return that address,
2352 : * or the nearest available memory above that address, providing a near-guarantee
2353 : * that those bits are clear. If they are not, we return NULL below to indicate
2354 : * out-of-memory.
2355 : *
2356 : * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual
2357 : * address space.
2358 : *
2359 : * See Bug 589735 for more information.
2360 : */
2361 : bool check_placement = true;
2362 : if (addr == NULL) {
2363 : addr = (void*)0x0000070000000000;
2364 : check_placement = false;
2365 : }
2366 : #endif
2367 :
2368 : /*
2369 : * We don't use MAP_FIXED here, because it can cause the *replacement*
2370 : * of existing mappings, and we only want to create new mappings.
2371 : */
2372 : #ifdef MALLOC_PAGEFILE
2373 : if (pfd != -1) {
2374 : ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
2375 : MAP_NOSYNC, pfd, 0);
2376 : } else
2377 : #endif
2378 : {
2379 82582 : ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
2380 : MAP_ANON, -1, 0);
2381 : }
2382 82582 : assert(ret != NULL);
2383 :
2384 82582 : if (ret == MAP_FAILED) {
2385 19 : ret = NULL;
2386 : }
2387 : #if defined(__ia64__)
2388 : /*
2389 : * If the allocated memory doesn't have its upper 17 bits clear, consider it
2390 : * as out of memory.
2391 : */
2392 : else if ((long long)ret & 0xffff800000000000) {
2393 : munmap(ret, size);
2394 : ret = NULL;
2395 : }
2396 : /* If the caller requested a specific memory location, verify that's what mmap returned. */
2397 : else if (check_placement && ret != addr) {
2398 : #else
2399 82563 : else if (addr != NULL && ret != addr) {
2400 : #endif
2401 : /*
2402 : * We succeeded in mapping memory, but not in the right place.
2403 : */
2404 0 : if (munmap(ret, size) == -1) {
2405 : char buf[STRERROR_BUF];
2406 :
2407 0 : strerror_r(errno, buf, sizeof(buf));
2408 0 : _malloc_message(_getprogname(),
2409 : ": (malloc) Error in munmap(): ", buf, "\n");
2410 0 : if (opt_abort)
2411 0 : abort();
2412 : }
2413 0 : ret = NULL;
2414 : }
2415 :
2416 : #if defined(__ia64__)
2417 : assert(ret == NULL || (!check_placement && ret != NULL)
2418 : || (check_placement && ret == addr));
2419 : #else
2420 82582 : assert(ret == NULL || (addr == NULL && ret != addr)
2421 : || (addr != NULL && ret == addr));
2422 : #endif
2423 82582 : return (ret);
2424 : }
2425 :
2426 : static void
2427 40007 : pages_unmap(void *addr, size_t size)
2428 : {
2429 :
2430 40007 : if (munmap(addr, size) == -1) {
2431 : char buf[STRERROR_BUF];
2432 :
2433 0 : strerror_r(errno, buf, sizeof(buf));
2434 0 : _malloc_message(_getprogname(),
2435 : ": (malloc) Error in munmap(): ", buf, "\n");
2436 0 : if (opt_abort)
2437 0 : abort();
2438 : }
2439 40007 : }
2440 : #endif
2441 :
2442 : #ifdef MOZ_MEMORY_DARWIN
2443 : #define VM_COPY_MIN (pagesize << 5)
2444 : static inline void
2445 : pages_copy(void *dest, const void *src, size_t n)
2446 : {
2447 :
2448 : assert((void *)((uintptr_t)dest & ~pagesize_mask) == dest);
2449 : assert(n >= VM_COPY_MIN);
2450 : assert((void *)((uintptr_t)src & ~pagesize_mask) == src);
2451 :
2452 : vm_copy(mach_task_self(), (vm_address_t)src, (vm_size_t)n,
2453 : (vm_address_t)dest);
2454 : }
2455 : #endif
2456 :
2457 : #ifdef MALLOC_VALIDATE
2458 : static inline malloc_rtree_t *
2459 19901 : malloc_rtree_new(unsigned bits)
2460 : {
2461 : malloc_rtree_t *ret;
2462 : unsigned bits_per_level, height, i;
2463 :
2464 39802 : bits_per_level = ffs(pow2_ceil((MALLOC_RTREE_NODESIZE /
2465 19901 : sizeof(void *)))) - 1;
2466 19901 : height = bits / bits_per_level;
2467 19901 : if (height * bits_per_level != bits)
2468 0 : height++;
2469 19901 : assert(height * bits_per_level >= bits);
2470 :
2471 19901 : ret = (malloc_rtree_t*)base_calloc(1, sizeof(malloc_rtree_t) +
2472 : (sizeof(unsigned) * (height - 1)));
2473 19901 : if (ret == NULL)
2474 0 : return (NULL);
2475 :
2476 19901 : malloc_spin_init(&ret->lock);
2477 19901 : ret->height = height;
2478 19901 : if (bits_per_level * height > bits)
2479 0 : ret->level2bits[0] = bits % bits_per_level;
2480 : else
2481 19901 : ret->level2bits[0] = bits_per_level;
2482 19901 : for (i = 1; i < height; i++)
2483 0 : ret->level2bits[i] = bits_per_level;
2484 :
2485 19901 : ret->root = (void**)base_calloc(1, sizeof(void *) << ret->level2bits[0]);
2486 19901 : if (ret->root == NULL) {
2487 : /*
2488 : * We leak the rtree here, since there's no generic base
2489 : * deallocation.
2490 : */
2491 0 : return (NULL);
2492 : }
2493 :
2494 19901 : return (ret);
2495 : }
2496 :
2497 : #define MALLOC_RTREE_GET_GENERATE(f) \
2498 : /* The least significant bits of the key are ignored. */ \
2499 : static inline void * \
2500 : f(malloc_rtree_t *rtree, uintptr_t key) \
2501 : { \
2502 : void *ret; \
2503 : uintptr_t subkey; \
2504 : unsigned i, lshift, height, bits; \
2505 : void **node, **child; \
2506 : \
2507 : MALLOC_RTREE_LOCK(&rtree->lock); \
2508 : for (i = lshift = 0, height = rtree->height, node = rtree->root;\
2509 : i < height - 1; \
2510 : i++, lshift += bits, node = child) { \
2511 : bits = rtree->level2bits[i]; \
2512 : subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); \
2513 : child = (void**)node[subkey]; \
2514 : if (child == NULL) { \
2515 : MALLOC_RTREE_UNLOCK(&rtree->lock); \
2516 : return (NULL); \
2517 : } \
2518 : } \
2519 : \
2520 : /* \
2521 : * node is a leaf, so it contains values rather than node \
2522 : * pointers. \
2523 : */ \
2524 : bits = rtree->level2bits[i]; \
2525 : subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); \
2526 : ret = node[subkey]; \
2527 : MALLOC_RTREE_UNLOCK(&rtree->lock); \
2528 : \
2529 : MALLOC_RTREE_GET_VALIDATE \
2530 : return (ret); \
2531 : }
2532 :
2533 : #ifdef MALLOC_DEBUG
2534 : # define MALLOC_RTREE_LOCK(l) malloc_spin_lock(l)
2535 : # define MALLOC_RTREE_UNLOCK(l) malloc_spin_unlock(l)
2536 : # define MALLOC_RTREE_GET_VALIDATE
2537 14218142 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get_locked)
2538 : # undef MALLOC_RTREE_LOCK
2539 : # undef MALLOC_RTREE_UNLOCK
2540 : # undef MALLOC_RTREE_GET_VALIDATE
2541 : #endif
2542 :
2543 : #define MALLOC_RTREE_LOCK(l)
2544 : #define MALLOC_RTREE_UNLOCK(l)
2545 : #ifdef MALLOC_DEBUG
2546 : /*
2547 : * Suppose that it were possible for a jemalloc-allocated chunk to be
2548 : * munmap()ped, followed by a different allocator in another thread re-using
2549 : * overlapping virtual memory, all without invalidating the cached rtree
2550 : * value. The result would be a false positive (the rtree would claim that
2551 : * jemalloc owns memory that it had actually discarded). I don't think this
2552 : * scenario is possible, but the following assertion is a prudent sanity
2553 : * check.
2554 : */
2555 : # define MALLOC_RTREE_GET_VALIDATE \
2556 : assert(malloc_rtree_get_locked(rtree, key) == ret);
2557 : #else
2558 : # define MALLOC_RTREE_GET_VALIDATE
2559 : #endif
2560 14218181 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get)
2561 : #undef MALLOC_RTREE_LOCK
2562 : #undef MALLOC_RTREE_UNLOCK
2563 : #undef MALLOC_RTREE_GET_VALIDATE
2564 :
2565 : static inline bool
2566 48669 : malloc_rtree_set(malloc_rtree_t *rtree, uintptr_t key, void *val)
2567 : {
2568 : uintptr_t subkey;
2569 : unsigned i, lshift, height, bits;
2570 : void **node, **child;
2571 :
2572 48669 : malloc_spin_lock(&rtree->lock);
2573 97338 : for (i = lshift = 0, height = rtree->height, node = rtree->root;
2574 48669 : i < height - 1;
2575 0 : i++, lshift += bits, node = child) {
2576 0 : bits = rtree->level2bits[i];
2577 0 : subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
2578 0 : child = (void**)node[subkey];
2579 0 : if (child == NULL) {
2580 0 : child = (void**)base_calloc(1, sizeof(void *) <<
2581 0 : rtree->level2bits[i+1]);
2582 0 : if (child == NULL) {
2583 0 : malloc_spin_unlock(&rtree->lock);
2584 0 : return (true);
2585 : }
2586 0 : node[subkey] = child;
2587 : }
2588 : }
2589 :
2590 : /* node is a leaf, so it contains values rather than node pointers. */
2591 48669 : bits = rtree->level2bits[i];
2592 48669 : subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
2593 48669 : node[subkey] = val;
2594 48669 : malloc_spin_unlock(&rtree->lock);
2595 :
2596 48669 : return (false);
2597 : }
2598 : #endif
2599 :
2600 : #if defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)
2601 :
2602 : /* Allocate an aligned chunk while maintaining a 1:1 correspondence between
2603 : * mmap and unmap calls. This is important on Windows, but not elsewhere. */
2604 : static void *
2605 : chunk_alloc_mmap(size_t size, bool pagefile)
2606 : {
2607 : void *ret;
2608 : #ifndef JEMALLOC_USES_MAP_ALIGN
2609 : size_t offset;
2610 : #endif
2611 : int pfd;
2612 :
2613 : #ifdef MALLOC_PAGEFILE
2614 : if (opt_pagefile && pagefile) {
2615 : pfd = pagefile_init(size);
2616 : if (pfd == -1)
2617 : return (NULL);
2618 : } else
2619 : #endif
2620 : pfd = -1;
2621 :
2622 : #ifdef JEMALLOC_USES_MAP_ALIGN
2623 : ret = pages_map_align(size, pfd, chunksize);
2624 : #else
2625 : ret = pages_map(NULL, size, pfd);
2626 : if (ret == NULL)
2627 : goto RETURN;
2628 :
2629 : offset = CHUNK_ADDR2OFFSET(ret);
2630 : if (offset != 0) {
2631 : /* Deallocate, then try to allocate at (ret + size - offset). */
2632 : pages_unmap(ret, size);
2633 : ret = pages_map((void *)((uintptr_t)ret + size - offset), size,
2634 : pfd);
2635 : while (ret == NULL) {
2636 : /*
2637 : * Over-allocate in order to map a memory region that
2638 : * is definitely large enough.
2639 : */
2640 : ret = pages_map(NULL, size + chunksize, -1);
2641 : if (ret == NULL)
2642 : goto RETURN;
2643 : /*
2644 : * Deallocate, then allocate the correct size, within
2645 : * the over-sized mapping.
2646 : */
2647 : offset = CHUNK_ADDR2OFFSET(ret);
2648 : pages_unmap(ret, size + chunksize);
2649 : if (offset == 0)
2650 : ret = pages_map(ret, size, pfd);
2651 : else {
2652 : ret = pages_map((void *)((uintptr_t)ret +
2653 : chunksize - offset), size, pfd);
2654 : }
2655 : /*
2656 : * Failure here indicates a race with another thread, so
2657 : * try again.
2658 : */
2659 : }
2660 : }
2661 : RETURN:
2662 : #endif
2663 : #ifdef MALLOC_PAGEFILE
2664 : if (pfd != -1)
2665 : pagefile_close(pfd);
2666 : #endif
2667 : #ifdef MALLOC_STATS
2668 : if (ret != NULL)
2669 : stats_chunks.nchunks += (size / chunksize);
2670 : #endif
2671 : return (ret);
2672 : }
2673 :
2674 : #else /* ! (defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)) */
2675 :
2676 : /*
2677 : * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
2678 : * potentially avoid some system calls.
2679 : */
2680 : #ifndef NO_TLS
2681 : static __thread bool mmap_unaligned_tls __attribute__((tls_model("initial-exec")));
2682 : #define MMAP_UNALIGNED_GET() mmap_unaligned_tls
2683 : #define MMAP_UNALIGNED_SET(v) do { \
2684 : mmap_unaligned_tls = (v); \
2685 : } while (0)
2686 : #else
2687 : #define NEEDS_PTHREAD_MMAP_UNALIGNED_TSD
2688 : static pthread_key_t mmap_unaligned_tsd;
2689 : #define MMAP_UNALIGNED_GET() ((bool)pthread_getspecific(mmap_unaligned_tsd))
2690 : #define MMAP_UNALIGNED_SET(v) do { \
2691 : pthread_setspecific(mmap_unaligned_tsd, (void *)(v)); \
2692 : } while (0)
2693 : #endif
2694 :
2695 : /* chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked from upstream
2696 : * jemalloc 2.2.3 to fix Mozilla bug 694896, enable jemalloc on Mac 10.7. */
2697 :
2698 : static void *
2699 7837 : chunk_alloc_mmap_slow(size_t size, bool unaligned)
2700 : {
2701 : void *ret;
2702 : size_t offset;
2703 :
2704 : /* Beware size_t wrap-around. */
2705 7837 : if (size + chunksize <= size)
2706 0 : return (NULL);
2707 :
2708 7837 : ret = pages_map(NULL, size + chunksize, -1);
2709 7837 : if (ret == NULL)
2710 0 : return (NULL);
2711 :
2712 : /* Clean up unneeded leading/trailing space. */
2713 7837 : offset = CHUNK_ADDR2OFFSET(ret);
2714 7837 : if (offset != 0) {
2715 : /* Note that mmap() returned an unaligned mapping. */
2716 2263 : unaligned = true;
2717 :
2718 : /* Leading space. */
2719 2263 : pages_unmap(ret, chunksize - offset);
2720 :
2721 2263 : ret = (void *)((uintptr_t)ret +
2722 2263 : (chunksize - offset));
2723 :
2724 : /* Trailing space. */
2725 2263 : pages_unmap((void *)((uintptr_t)ret + size),
2726 : offset);
2727 : } else {
2728 : /* Trailing space only. */
2729 5574 : pages_unmap((void *)((uintptr_t)ret + size),
2730 : chunksize);
2731 : }
2732 :
2733 : /*
2734 : * If mmap() returned an aligned mapping, reset mmap_unaligned so that
2735 : * the next chunk_alloc_mmap() execution tries the fast allocation
2736 : * method.
2737 : */
2738 7837 : if (unaligned == false)
2739 5574 : MMAP_UNALIGNED_SET(false);
2740 :
2741 7837 : return (ret);
2742 : }
2743 :
2744 : static void *
2745 40731 : chunk_alloc_mmap(size_t size, bool pagefile)
2746 : {
2747 : void *ret;
2748 :
2749 : /*
2750 : * Ideally, there would be a way to specify alignment to mmap() (like
2751 : * NetBSD has), but in the absence of such a feature, we have to work
2752 : * hard to efficiently create aligned mappings. The reliable, but
2753 : * slow method is to create a mapping that is over-sized, then trim the
2754 : * excess. However, that always results in at least one call to
2755 : * pages_unmap().
2756 : *
2757 : * A more optimistic approach is to try mapping precisely the right
2758 : * amount, then try to append another mapping if alignment is off. In
2759 : * practice, this works out well as long as the application is not
2760 : * interleaving mappings via direct mmap() calls. If we do run into a
2761 : * situation where there is an interleaved mapping and we are unable to
2762 : * extend an unaligned mapping, our best option is to switch to the
2763 : * slow method until mmap() returns another aligned mapping. This will
2764 : * tend to leave a gap in the memory map that is too small to cause
2765 : * later problems for the optimistic method.
2766 : *
2767 : * Another possible confounding factor is address space layout
2768 : * randomization (ASLR), which causes mmap(2) to disregard the
2769 : * requested address. mmap_unaligned tracks whether the previous
2770 : * chunk_alloc_mmap() execution received any unaligned or relocated
2771 : * mappings, and if so, the current execution will immediately fall
2772 : * back to the slow method. However, we keep track of whether the fast
2773 : * method would have succeeded, and if so, we make a note to try the
2774 : * fast method next time.
2775 : */
2776 :
2777 40731 : if (MMAP_UNALIGNED_GET() == false) {
2778 : size_t offset;
2779 :
2780 32894 : ret = pages_map(NULL, size, -1);
2781 32894 : if (ret == NULL)
2782 19 : return (NULL);
2783 :
2784 32875 : offset = CHUNK_ADDR2OFFSET(ret);
2785 32875 : if (offset != 0) {
2786 21950 : MMAP_UNALIGNED_SET(true);
2787 : /* Try to extend chunk boundary. */
2788 21950 : if (pages_map((void *)((uintptr_t)ret + size),
2789 : chunksize - offset, -1) == NULL) {
2790 : /*
2791 : * Extension failed. Clean up, then revert to
2792 : * the reliable-but-expensive method.
2793 : */
2794 0 : pages_unmap(ret, size);
2795 0 : ret = chunk_alloc_mmap_slow(size, true);
2796 : } else {
2797 : /* Clean up unneeded leading space. */
2798 21950 : pages_unmap(ret, chunksize - offset);
2799 21950 : ret = (void *)((uintptr_t)ret + (chunksize -
2800 : offset));
2801 : }
2802 : }
2803 : } else
2804 7837 : ret = chunk_alloc_mmap_slow(size, false);
2805 :
2806 40712 : return (ret);
2807 : }
2808 :
2809 : #endif /* defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE) */
2810 :
2811 : #ifdef MALLOC_PAGEFILE
2812 : static int
2813 : pagefile_init(size_t size)
2814 : {
2815 : int ret;
2816 : size_t i;
2817 : char pagefile_path[PATH_MAX];
2818 : char zbuf[MALLOC_PAGEFILE_WRITE_SIZE];
2819 :
2820 : /*
2821 : * Create a temporary file, then immediately unlink it so that it will
2822 : * not persist.
2823 : */
2824 : strcpy(pagefile_path, pagefile_templ);
2825 : ret = mkstemp(pagefile_path);
2826 : if (ret == -1)
2827 : return (ret);
2828 : if (unlink(pagefile_path)) {
2829 : char buf[STRERROR_BUF];
2830 :
2831 : strerror_r(errno, buf, sizeof(buf));
2832 : _malloc_message(_getprogname(), ": (malloc) Error in unlink(\"",
2833 : pagefile_path, "\"):");
2834 : _malloc_message(buf, "\n", "", "");
2835 : if (opt_abort)
2836 : abort();
2837 : }
2838 :
2839 : /*
2840 : * Write sequential zeroes to the file in order to assure that disk
2841 : * space is committed, with minimal fragmentation. It would be
2842 : * sufficient to write one zero per disk block, but that potentially
2843 : * results in more system calls, for no real gain.
2844 : */
2845 : memset(zbuf, 0, sizeof(zbuf));
2846 : for (i = 0; i < size; i += sizeof(zbuf)) {
2847 : if (write(ret, zbuf, sizeof(zbuf)) != sizeof(zbuf)) {
2848 : if (errno != ENOSPC) {
2849 : char buf[STRERROR_BUF];
2850 :
2851 : strerror_r(errno, buf, sizeof(buf));
2852 : _malloc_message(_getprogname(),
2853 : ": (malloc) Error in write(): ", buf, "\n");
2854 : if (opt_abort)
2855 : abort();
2856 : }
2857 : pagefile_close(ret);
2858 : return (-1);
2859 : }
2860 : }
2861 :
2862 : return (ret);
2863 : }
2864 :
2865 : static void
2866 : pagefile_close(int pfd)
2867 : {
2868 :
2869 : if (close(pfd)) {
2870 : char buf[STRERROR_BUF];
2871 :
2872 : strerror_r(errno, buf, sizeof(buf));
2873 : _malloc_message(_getprogname(),
2874 : ": (malloc) Error in close(): ", buf, "\n");
2875 : if (opt_abort)
2876 : abort();
2877 : }
2878 : }
2879 : #endif
2880 :
2881 : static void *
2882 40731 : chunk_alloc(size_t size, bool zero, bool pagefile)
2883 : {
2884 : void *ret;
2885 :
2886 40731 : assert(size != 0);
2887 40731 : assert((size & chunksize_mask) == 0);
2888 :
2889 40731 : ret = chunk_alloc_mmap(size, pagefile);
2890 40731 : if (ret != NULL) {
2891 40712 : goto RETURN;
2892 : }
2893 :
2894 : /* All strategies for allocation failed. */
2895 19 : ret = NULL;
2896 : RETURN:
2897 : #ifdef MALLOC_STATS
2898 40731 : if (ret != NULL)
2899 40712 : stats_chunks.curchunks += (size / chunksize);
2900 40731 : if (stats_chunks.curchunks > stats_chunks.highchunks)
2901 35557 : stats_chunks.highchunks = stats_chunks.curchunks;
2902 : #endif
2903 :
2904 : #ifdef MALLOC_VALIDATE
2905 40731 : if (ret != NULL) {
2906 40712 : if (malloc_rtree_set(chunk_rtree, (uintptr_t)ret, ret)) {
2907 0 : chunk_dealloc(ret, size);
2908 0 : return (NULL);
2909 : }
2910 : }
2911 : #endif
2912 :
2913 40731 : assert(CHUNK_ADDR2BASE(ret) == ret);
2914 40731 : return (ret);
2915 : }
2916 :
2917 : static void
2918 7957 : chunk_dealloc_mmap(void *chunk, size_t size)
2919 : {
2920 :
2921 7957 : pages_unmap(chunk, size);
2922 7957 : }
2923 :
2924 : static void
2925 7957 : chunk_dealloc(void *chunk, size_t size)
2926 : {
2927 :
2928 7957 : assert(chunk != NULL);
2929 7957 : assert(CHUNK_ADDR2BASE(chunk) == chunk);
2930 7957 : assert(size != 0);
2931 7957 : assert((size & chunksize_mask) == 0);
2932 :
2933 : #ifdef MALLOC_STATS
2934 7957 : stats_chunks.curchunks -= (size / chunksize);
2935 : #endif
2936 : #ifdef MALLOC_VALIDATE
2937 7957 : malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL);
2938 : #endif
2939 :
2940 7957 : chunk_dealloc_mmap(chunk, size);
2941 7957 : }
2942 :
2943 : /*
2944 : * End chunk management functions.
2945 : */
2946 : /******************************************************************************/
2947 : /*
2948 : * Begin arena.
2949 : */
2950 :
2951 : /*
2952 : * Choose an arena based on a per-thread value (fast-path code, calls slow-path
2953 : * code if necessary).
2954 : */
2955 : static inline arena_t *
2956 125762912 : choose_arena(void)
2957 : {
2958 : arena_t *ret;
2959 :
2960 : /*
2961 : * We can only use TLS if this is a PIC library, since for the static
2962 : * library version, libc's malloc is used by TLS allocation, which
2963 : * introduces a bootstrapping issue.
2964 : */
2965 : #ifndef NO_TLS
2966 125762912 : if (__isthreaded == false) {
2967 : /* Avoid the overhead of TLS for single-threaded operation. */
2968 0 : return (arenas[0]);
2969 : }
2970 :
2971 : # ifdef MOZ_MEMORY_WINDOWS
2972 : ret = (arena_t*)TlsGetValue(tlsIndex);
2973 : # else
2974 125762912 : ret = arenas_map;
2975 : # endif
2976 :
2977 125762912 : if (ret == NULL) {
2978 11993 : ret = choose_arena_hard();
2979 11993 : assert(ret != NULL);
2980 : }
2981 : #else
2982 : if (__isthreaded && narenas > 1) {
2983 : unsigned long ind;
2984 :
2985 : /*
2986 : * Hash _pthread_self() to one of the arenas. There is a prime
2987 : * number of arenas, so this has a reasonable chance of
2988 : * working. Even so, the hashing can be easily thwarted by
2989 : * inconvenient _pthread_self() values. Without specific
2990 : * knowledge of how _pthread_self() calculates values, we can't
2991 : * easily do much better than this.
2992 : */
2993 : ind = (unsigned long) _pthread_self() % narenas;
2994 :
2995 : /*
2996 : * Optimistially assume that arenas[ind] has been initialized.
2997 : * At worst, we find out that some other thread has already
2998 : * done so, after acquiring the lock in preparation. Note that
2999 : * this lazy locking also has the effect of lazily forcing
3000 : * cache coherency; without the lock acquisition, there's no
3001 : * guarantee that modification of arenas[ind] by another thread
3002 : * would be seen on this CPU for an arbitrary amount of time.
3003 : *
3004 : * In general, this approach to modifying a synchronized value
3005 : * isn't a good idea, but in this case we only ever modify the
3006 : * value once, so things work out well.
3007 : */
3008 : ret = arenas[ind];
3009 : if (ret == NULL) {
3010 : /*
3011 : * Avoid races with another thread that may have already
3012 : * initialized arenas[ind].
3013 : */
3014 : malloc_spin_lock(&arenas_lock);
3015 : if (arenas[ind] == NULL)
3016 : ret = arenas_extend((unsigned)ind);
3017 : else
3018 : ret = arenas[ind];
3019 : malloc_spin_unlock(&arenas_lock);
3020 : }
3021 : } else
3022 : ret = arenas[0];
3023 : #endif
3024 :
3025 125762912 : assert(ret != NULL);
3026 125762912 : return (ret);
3027 : }
3028 :
3029 : #ifndef NO_TLS
3030 : /*
3031 : * Choose an arena based on a per-thread value (slow-path code only, called
3032 : * only by choose_arena()).
3033 : */
3034 : static arena_t *
3035 11993 : choose_arena_hard(void)
3036 : {
3037 : arena_t *ret;
3038 :
3039 11993 : assert(__isthreaded);
3040 :
3041 : #ifdef MALLOC_BALANCE
3042 : /* Seed the PRNG used for arena load balancing. */
3043 : SPRN(balance, (uint32_t)(uintptr_t)(_pthread_self()));
3044 : #endif
3045 :
3046 11993 : if (narenas > 1) {
3047 : #ifdef MALLOC_BALANCE
3048 : unsigned ind;
3049 :
3050 : ind = PRN(balance, narenas_2pow);
3051 : if ((ret = arenas[ind]) == NULL) {
3052 : malloc_spin_lock(&arenas_lock);
3053 : if ((ret = arenas[ind]) == NULL)
3054 : ret = arenas_extend(ind);
3055 : malloc_spin_unlock(&arenas_lock);
3056 : }
3057 : #else
3058 0 : malloc_spin_lock(&arenas_lock);
3059 0 : if ((ret = arenas[next_arena]) == NULL)
3060 0 : ret = arenas_extend(next_arena);
3061 0 : next_arena = (next_arena + 1) % narenas;
3062 0 : malloc_spin_unlock(&arenas_lock);
3063 : #endif
3064 : } else
3065 11993 : ret = arenas[0];
3066 :
3067 : #ifdef MOZ_MEMORY_WINDOWS
3068 : TlsSetValue(tlsIndex, ret);
3069 : #else
3070 11993 : arenas_map = ret;
3071 : #endif
3072 :
3073 11993 : return (ret);
3074 : }
3075 : #endif
3076 :
3077 : static inline int
3078 1802517 : arena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b)
3079 : {
3080 1802517 : uintptr_t a_chunk = (uintptr_t)a;
3081 1802517 : uintptr_t b_chunk = (uintptr_t)b;
3082 :
3083 1802517 : assert(a != NULL);
3084 1802517 : assert(b != NULL);
3085 :
3086 1802517 : return ((a_chunk > b_chunk) - (a_chunk < b_chunk));
3087 : }
3088 :
3089 : /* Wrap red-black tree macros in functions. */
3090 1137110 : rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
3091 1137110 : arena_chunk_t, link_dirty, arena_chunk_comp)
3092 :
3093 : static inline int
3094 20078373 : arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
3095 : {
3096 20078373 : uintptr_t a_mapelm = (uintptr_t)a;
3097 20078373 : uintptr_t b_mapelm = (uintptr_t)b;
3098 :
3099 20078373 : assert(a != NULL);
3100 20078373 : assert(b != NULL);
3101 :
3102 20078373 : return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm));
3103 : }
3104 :
3105 : /* Wrap red-black tree macros in functions. */
3106 14611500 : rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, link,
3107 14611500 : arena_run_comp)
3108 :
3109 : static inline int
3110 69472920 : arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
3111 : {
3112 : int ret;
3113 69472920 : size_t a_size = a->bits & ~pagesize_mask;
3114 69472920 : size_t b_size = b->bits & ~pagesize_mask;
3115 :
3116 69472920 : ret = (a_size > b_size) - (a_size < b_size);
3117 69472920 : if (ret == 0) {
3118 : uintptr_t a_mapelm, b_mapelm;
3119 :
3120 21029319 : if ((a->bits & CHUNK_MAP_KEY) == 0)
3121 18576310 : a_mapelm = (uintptr_t)a;
3122 : else {
3123 : /*
3124 : * Treat keys as though they are lower than anything
3125 : * else.
3126 : */
3127 2453009 : a_mapelm = 0;
3128 : }
3129 21029319 : b_mapelm = (uintptr_t)b;
3130 :
3131 21029319 : ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm);
3132 : }
3133 :
3134 69472920 : return (ret);
3135 : }
3136 :
3137 : /* Wrap red-black tree macros in functions. */
3138 29949484 : rb_wrap(static, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, link,
3139 29949484 : arena_avail_comp)
3140 :
3141 : static inline void *
3142 123481815 : arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
3143 : {
3144 : void *ret;
3145 : unsigned i, mask, bit, regind;
3146 :
3147 123481815 : assert(run->magic == ARENA_RUN_MAGIC);
3148 123481815 : assert(run->regs_minelm < bin->regs_mask_nelms);
3149 :
3150 : /*
3151 : * Move the first check outside the loop, so that run->regs_minelm can
3152 : * be updated unconditionally, without the possibility of updating it
3153 : * multiple times.
3154 : */
3155 123481815 : i = run->regs_minelm;
3156 123481815 : mask = run->regs_mask[i];
3157 123481815 : if (mask != 0) {
3158 : /* Usable allocation found. */
3159 120399106 : bit = ffs((int)mask) - 1;
3160 :
3161 120399106 : regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
3162 120399106 : assert(regind < bin->nregs);
3163 240798212 : ret = (void *)(((uintptr_t)run) + bin->reg0_offset
3164 120399106 : + (bin->reg_size * regind));
3165 :
3166 : /* Clear bit. */
3167 120399106 : mask ^= (1U << bit);
3168 120399106 : run->regs_mask[i] = mask;
3169 :
3170 120399106 : return (ret);
3171 : }
3172 :
3173 3463151 : for (i++; i < bin->regs_mask_nelms; i++) {
3174 3463151 : mask = run->regs_mask[i];
3175 3463151 : if (mask != 0) {
3176 : /* Usable allocation found. */
3177 3082709 : bit = ffs((int)mask) - 1;
3178 :
3179 3082709 : regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
3180 3082709 : assert(regind < bin->nregs);
3181 6165418 : ret = (void *)(((uintptr_t)run) + bin->reg0_offset
3182 3082709 : + (bin->reg_size * regind));
3183 :
3184 : /* Clear bit. */
3185 3082709 : mask ^= (1U << bit);
3186 3082709 : run->regs_mask[i] = mask;
3187 :
3188 : /*
3189 : * Make a note that nothing before this element
3190 : * contains a free region.
3191 : */
3192 3082709 : run->regs_minelm = i; /* Low payoff: + (mask == 0); */
3193 :
3194 3082709 : return (ret);
3195 : }
3196 : }
3197 : /* Not reached. */
3198 0 : assert(0);
3199 : return (NULL);
3200 : }
3201 :
3202 : static inline void
3203 119028910 : arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size)
3204 : {
3205 : /*
3206 : * To divide by a number D that is not a power of two we multiply
3207 : * by (2^21 / D) and then right shift by 21 positions.
3208 : *
3209 : * X / D
3210 : *
3211 : * becomes
3212 : *
3213 : * (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT
3214 : */
3215 : #define SIZE_INV_SHIFT 21
3216 : #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1)
3217 : static const unsigned size_invs[] = {
3218 : SIZE_INV(3),
3219 : SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
3220 : SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
3221 : SIZE_INV(12),SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
3222 : SIZE_INV(16),SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
3223 : SIZE_INV(20),SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
3224 : SIZE_INV(24),SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
3225 : SIZE_INV(28),SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
3226 : #if (QUANTUM_2POW_MIN < 4)
3227 : ,
3228 : SIZE_INV(32), SIZE_INV(33), SIZE_INV(34), SIZE_INV(35),
3229 : SIZE_INV(36), SIZE_INV(37), SIZE_INV(38), SIZE_INV(39),
3230 : SIZE_INV(40), SIZE_INV(41), SIZE_INV(42), SIZE_INV(43),
3231 : SIZE_INV(44), SIZE_INV(45), SIZE_INV(46), SIZE_INV(47),
3232 : SIZE_INV(48), SIZE_INV(49), SIZE_INV(50), SIZE_INV(51),
3233 : SIZE_INV(52), SIZE_INV(53), SIZE_INV(54), SIZE_INV(55),
3234 : SIZE_INV(56), SIZE_INV(57), SIZE_INV(58), SIZE_INV(59),
3235 : SIZE_INV(60), SIZE_INV(61), SIZE_INV(62), SIZE_INV(63)
3236 : #endif
3237 : };
3238 : unsigned diff, regind, elm, bit;
3239 :
3240 119028910 : assert(run->magic == ARENA_RUN_MAGIC);
3241 : assert(((sizeof(size_invs)) / sizeof(unsigned)) + 3
3242 : >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN));
3243 :
3244 : /*
3245 : * Avoid doing division with a variable divisor if possible. Using
3246 : * actual division here can reduce allocator throughput by over 20%!
3247 : */
3248 119028910 : diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
3249 119028910 : if ((size & (size - 1)) == 0) {
3250 : /*
3251 : * log2_table allows fast division of a power of two in the
3252 : * [1..128] range.
3253 : *
3254 : * (x / divisor) becomes (x >> log2_table[divisor - 1]).
3255 : */
3256 : static const unsigned char log2_table[] = {
3257 : 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
3258 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
3259 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3260 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
3261 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3262 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3263 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
3264 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
3265 : };
3266 :
3267 90402814 : if (size <= 128)
3268 71000900 : regind = (diff >> log2_table[size - 1]);
3269 19401914 : else if (size <= 32768)
3270 19401914 : regind = diff >> (8 + log2_table[(size >> 8) - 1]);
3271 : else {
3272 : /*
3273 : * The run size is too large for us to use the lookup
3274 : * table. Use real division.
3275 : */
3276 0 : regind = diff / size;
3277 : }
3278 28626096 : } else if (size <= ((sizeof(size_invs) / sizeof(unsigned))
3279 : << QUANTUM_2POW_MIN) + 2) {
3280 28473100 : regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff;
3281 28473100 : regind >>= SIZE_INV_SHIFT;
3282 : } else {
3283 : /*
3284 : * size_invs isn't large enough to handle this size class, so
3285 : * calculate regind using actual division. This only happens
3286 : * if the user increases small_max via the 'S' runtime
3287 : * configuration option.
3288 : */
3289 152996 : regind = diff / size;
3290 : };
3291 119028910 : assert(diff == regind * size);
3292 119028910 : assert(regind < bin->nregs);
3293 :
3294 119028910 : elm = regind >> (SIZEOF_INT_2POW + 3);
3295 119028910 : if (elm < run->regs_minelm)
3296 2461777 : run->regs_minelm = elm;
3297 119028910 : bit = regind - (elm << (SIZEOF_INT_2POW + 3));
3298 119028910 : assert((run->regs_mask[elm] & (1U << bit)) == 0);
3299 119028910 : run->regs_mask[elm] |= (1U << bit);
3300 : #undef SIZE_INV
3301 : #undef SIZE_INV_SHIFT
3302 119028910 : }
3303 :
3304 : static void
3305 6738038 : arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
3306 : bool zero)
3307 : {
3308 : arena_chunk_t *chunk;
3309 : size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i;
3310 :
3311 6738038 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
3312 6738038 : old_ndirty = chunk->ndirty;
3313 6738038 : run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk)
3314 : >> pagesize_2pow);
3315 6738038 : total_pages = (chunk->map[run_ind].bits & ~pagesize_mask) >>
3316 : pagesize_2pow;
3317 6738038 : need_pages = (size >> pagesize_2pow);
3318 6738038 : assert(need_pages > 0);
3319 6738038 : assert(need_pages <= total_pages);
3320 6738038 : rem_pages = total_pages - need_pages;
3321 :
3322 6738038 : arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]);
3323 :
3324 : /* Keep track of trailing unused pages for later use. */
3325 6738038 : if (rem_pages > 0) {
3326 10926536 : chunk->map[run_ind+need_pages].bits = (rem_pages <<
3327 5463268 : pagesize_2pow) | (chunk->map[run_ind+need_pages].bits &
3328 : pagesize_mask);
3329 10926536 : chunk->map[run_ind+total_pages-1].bits = (rem_pages <<
3330 5463268 : pagesize_2pow) | (chunk->map[run_ind+total_pages-1].bits &
3331 : pagesize_mask);
3332 5463268 : arena_avail_tree_insert(&arena->runs_avail,
3333 5463268 : &chunk->map[run_ind+need_pages]);
3334 : }
3335 :
3336 36556213 : for (i = 0; i < need_pages; i++) {
3337 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
3338 : /*
3339 : * Commit decommitted pages if necessary. If a decommitted
3340 : * page is encountered, commit all needed adjacent decommitted
3341 : * pages in one operation, in order to reduce system call
3342 : * overhead.
3343 : */
3344 29818175 : if (chunk->map[run_ind + i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) {
3345 : size_t j;
3346 :
3347 : /*
3348 : * Advance i+j to just past the index of the last page
3349 : * to commit. Clear CHUNK_MAP_DECOMMITTED and
3350 : * CHUNK_MAP_MADVISED along the way.
3351 : */
3352 19513973 : for (j = 0; i + j < need_pages && (chunk->map[run_ind +
3353 16154525 : i + j].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED); j++) {
3354 : /* DECOMMITTED and MADVISED are mutually exclusive. */
3355 8076508 : assert(!(chunk->map[run_ind + i + j].bits & CHUNK_MAP_DECOMMITTED &&
3356 : chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED));
3357 :
3358 8076508 : chunk->map[run_ind + i + j].bits &=
3359 : ~CHUNK_MAP_MADVISED_OR_DECOMMITTED;
3360 : }
3361 :
3362 : # ifdef MALLOC_DECOMMIT
3363 : pages_commit((void *)((uintptr_t)chunk + ((run_ind + i)
3364 : << pagesize_2pow)), (j << pagesize_2pow));
3365 : # ifdef MALLOC_STATS
3366 : arena->stats.ncommit++;
3367 : # endif
3368 : # endif
3369 :
3370 : # ifdef MALLOC_STATS
3371 1679724 : arena->stats.committed += j;
3372 : # endif
3373 :
3374 : # ifndef MALLOC_DECOMMIT
3375 : }
3376 : # else
3377 : } else /* No need to zero since commit zeros. */
3378 : # endif
3379 :
3380 : #endif
3381 :
3382 : /* Zero if necessary. */
3383 29818175 : if (zero) {
3384 1405230 : if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED)
3385 : == 0) {
3386 : VALGRIND_MALLOCLIKE_BLOCK((void *)((uintptr_t)
3387 : chunk + ((run_ind + i) << pagesize_2pow)),
3388 : pagesize, 0, false);
3389 1236668 : memset((void *)((uintptr_t)chunk + ((run_ind
3390 1236668 : + i) << pagesize_2pow)), 0, pagesize);
3391 : VALGRIND_FREELIKE_BLOCK((void *)((uintptr_t)
3392 : chunk + ((run_ind + i) << pagesize_2pow)),
3393 : 0);
3394 : /* CHUNK_MAP_ZEROED is cleared below. */
3395 : }
3396 : }
3397 :
3398 : /* Update dirty page accounting. */
3399 29818175 : if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) {
3400 21741667 : chunk->ndirty--;
3401 21741667 : arena->ndirty--;
3402 : /* CHUNK_MAP_DIRTY is cleared below. */
3403 : }
3404 :
3405 : /* Initialize the chunk map. */
3406 29818175 : if (large) {
3407 12146964 : chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE
3408 : | CHUNK_MAP_ALLOCATED;
3409 : } else {
3410 35342422 : chunk->map[run_ind + i].bits = (size_t)run
3411 17671211 : | CHUNK_MAP_ALLOCATED;
3412 : }
3413 : }
3414 :
3415 : /*
3416 : * Set the run size only in the first element for large runs. This is
3417 : * primarily a debugging aid, since the lack of size info for trailing
3418 : * pages only matters if the application tries to operate on an
3419 : * interior pointer.
3420 : */
3421 6738038 : if (large)
3422 2398466 : chunk->map[run_ind].bits |= size;
3423 :
3424 6738038 : if (chunk->ndirty == 0 && old_ndirty > 0)
3425 491283 : arena_chunk_tree_dirty_remove(&arena->chunks_dirty, chunk);
3426 6738038 : }
3427 :
3428 : static void
3429 35193 : arena_chunk_init(arena_t *arena, arena_chunk_t *chunk)
3430 : {
3431 : arena_run_t *run;
3432 : size_t i;
3433 :
3434 : VALGRIND_MALLOCLIKE_BLOCK(chunk, (arena_chunk_header_npages <<
3435 : pagesize_2pow), 0, false);
3436 : #ifdef MALLOC_STATS
3437 35193 : arena->stats.mapped += chunksize;
3438 : #endif
3439 :
3440 35193 : chunk->arena = arena;
3441 :
3442 : /*
3443 : * Claim that no pages are in use, since the header is merely overhead.
3444 : */
3445 35193 : chunk->ndirty = 0;
3446 :
3447 : /* Initialize the map to contain one maximal free untouched run. */
3448 35193 : run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages <<
3449 : pagesize_2pow));
3450 70386 : for (i = 0; i < arena_chunk_header_npages; i++)
3451 35193 : chunk->map[i].bits = 0;
3452 35193 : chunk->map[i].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3453 8939022 : for (i++; i < chunk_npages-1; i++) {
3454 8903829 : chunk->map[i].bits = CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3455 : }
3456 35193 : chunk->map[chunk_npages-1].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
3457 :
3458 : #ifdef MALLOC_DECOMMIT
3459 : /*
3460 : * Start out decommitted, in order to force a closer correspondence
3461 : * between dirty pages and committed untouched pages.
3462 : */
3463 : pages_decommit(run, arena_maxclass);
3464 : # ifdef MALLOC_STATS
3465 : arena->stats.ndecommit++;
3466 : arena->stats.decommitted += (chunk_npages - arena_chunk_header_npages);
3467 : # endif
3468 : #endif
3469 : #ifdef MALLOC_STATS
3470 35193 : arena->stats.committed += arena_chunk_header_npages;
3471 : #endif
3472 :
3473 : /* Insert the run into the runs_avail tree. */
3474 35193 : arena_avail_tree_insert(&arena->runs_avail,
3475 35193 : &chunk->map[arena_chunk_header_npages]);
3476 :
3477 : #ifdef MALLOC_DOUBLE_PURGE
3478 : LinkedList_Init(&chunk->chunks_madvised_elem);
3479 : #endif
3480 35193 : }
3481 :
3482 : static void
3483 37150 : arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
3484 : {
3485 :
3486 37150 : if (arena->spare != NULL) {
3487 2438 : if (arena->spare->ndirty > 0) {
3488 4008 : arena_chunk_tree_dirty_remove(
3489 2004 : &chunk->arena->chunks_dirty, arena->spare);
3490 2004 : arena->ndirty -= arena->spare->ndirty;
3491 : #ifdef MALLOC_STATS
3492 2004 : arena->stats.committed -= arena->spare->ndirty;
3493 : #endif
3494 : }
3495 :
3496 : #ifdef MALLOC_DOUBLE_PURGE
3497 : /* This is safe to do even if arena->spare is not in the list. */
3498 : LinkedList_Remove(&arena->spare->chunks_madvised_elem);
3499 : #endif
3500 :
3501 : VALGRIND_FREELIKE_BLOCK(arena->spare, 0);
3502 2438 : chunk_dealloc((void *)arena->spare, chunksize);
3503 : #ifdef MALLOC_STATS
3504 2438 : arena->stats.mapped -= chunksize;
3505 2438 : arena->stats.committed -= arena_chunk_header_npages;
3506 : #endif
3507 : }
3508 :
3509 : /*
3510 : * Remove run from runs_avail, so that the arena does not use it.
3511 : * Dirty page flushing only uses the chunks_dirty tree, so leaving this
3512 : * chunk in the chunks_* trees is sufficient for that purpose.
3513 : */
3514 37150 : arena_avail_tree_remove(&arena->runs_avail,
3515 37150 : &chunk->map[arena_chunk_header_npages]);
3516 :
3517 37150 : arena->spare = chunk;
3518 37150 : }
3519 :
3520 : static arena_run_t *
3521 6621364 : arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large,
3522 : bool zero)
3523 : {
3524 : arena_run_t *run;
3525 : arena_chunk_map_t *mapelm, key;
3526 :
3527 6621364 : assert(size <= arena_maxclass);
3528 6621364 : assert((size & pagesize_mask) == 0);
3529 :
3530 : /* Search the arena's chunks for the lowest best fit. */
3531 6621364 : key.bits = size | CHUNK_MAP_KEY;
3532 6621364 : mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
3533 6621364 : if (mapelm != NULL) {
3534 6554745 : arena_chunk_t *chunk =
3535 6554745 : (arena_chunk_t*)CHUNK_ADDR2BASE(mapelm);
3536 13109490 : size_t pageind = ((uintptr_t)mapelm -
3537 6554745 : (uintptr_t)chunk->map) /
3538 : sizeof(arena_chunk_map_t);
3539 :
3540 6554745 : run = (arena_run_t *)((uintptr_t)chunk + (pageind
3541 6554745 : << pagesize_2pow));
3542 6554745 : arena_run_split(arena, run, size, large, zero);
3543 6554745 : return (run);
3544 : }
3545 :
3546 66619 : if (arena->spare != NULL) {
3547 : /* Use the spare. */
3548 31426 : arena_chunk_t *chunk = arena->spare;
3549 31426 : arena->spare = NULL;
3550 31426 : run = (arena_run_t *)((uintptr_t)chunk +
3551 31426 : (arena_chunk_header_npages << pagesize_2pow));
3552 : /* Insert the run into the runs_avail tree. */
3553 31426 : arena_avail_tree_insert(&arena->runs_avail,
3554 31426 : &chunk->map[arena_chunk_header_npages]);
3555 31426 : arena_run_split(arena, run, size, large, zero);
3556 31426 : return (run);
3557 : }
3558 :
3559 : /*
3560 : * No usable runs. Create a new chunk from which to allocate
3561 : * the run.
3562 : */
3563 : {
3564 35193 : arena_chunk_t *chunk = (arena_chunk_t *)
3565 35193 : chunk_alloc(chunksize, true, true);
3566 35193 : if (chunk == NULL)
3567 0 : return (NULL);
3568 :
3569 35193 : arena_chunk_init(arena, chunk);
3570 35193 : run = (arena_run_t *)((uintptr_t)chunk +
3571 35193 : (arena_chunk_header_npages << pagesize_2pow));
3572 : }
3573 : /* Update page map. */
3574 35193 : arena_run_split(arena, run, size, large, zero);
3575 35193 : return (run);
3576 : }
3577 :
3578 : static void
3579 3981 : arena_purge(arena_t *arena)
3580 : {
3581 : arena_chunk_t *chunk;
3582 : size_t i, npages;
3583 : #ifdef MALLOC_DEBUG
3584 3981 : size_t ndirty = 0;
3585 59413 : rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty,
3586 : chunk) {
3587 51451 : ndirty += chunk->ndirty;
3588 51451 : } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk)
3589 3981 : assert(ndirty == arena->ndirty);
3590 : #endif
3591 3981 : assert(arena->ndirty > opt_dirty_max);
3592 :
3593 : #ifdef MALLOC_STATS
3594 3981 : arena->stats.npurge++;
3595 : #endif
3596 :
3597 : /*
3598 : * Iterate downward through chunks until enough dirty memory has been
3599 : * purged. Terminate as soon as possible in order to minimize the
3600 : * number of system calls, even if a chunk has only been partially
3601 : * purged.
3602 : */
3603 42742 : while (arena->ndirty > (opt_dirty_max >> 1)) {
3604 : #ifdef MALLOC_DOUBLE_PURGE
3605 : bool madvised = false;
3606 : #endif
3607 34780 : chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty);
3608 34780 : assert(chunk != NULL);
3609 :
3610 4675669 : for (i = chunk_npages - 1; chunk->ndirty > 0; i--) {
3611 4644870 : assert(i >= arena_chunk_header_npages);
3612 :
3613 4644870 : if (chunk->map[i].bits & CHUNK_MAP_DIRTY) {
3614 : #ifdef MALLOC_DECOMMIT
3615 : const size_t free_operation = CHUNK_MAP_DECOMMITTED;
3616 : #else
3617 162169 : const size_t free_operation = CHUNK_MAP_MADVISED;
3618 : #endif
3619 162169 : assert((chunk->map[i].bits &
3620 : CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
3621 162169 : chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
3622 : /* Find adjacent dirty run(s). */
3623 2366910 : for (npages = 1;
3624 4400528 : i > arena_chunk_header_npages &&
3625 2195787 : (chunk->map[i - 1].bits & CHUNK_MAP_DIRTY);
3626 2042572 : npages++) {
3627 2042572 : i--;
3628 2042572 : assert((chunk->map[i].bits &
3629 : CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
3630 2042572 : chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
3631 : }
3632 162169 : chunk->ndirty -= npages;
3633 162169 : arena->ndirty -= npages;
3634 :
3635 : #ifdef MALLOC_DECOMMIT
3636 : pages_decommit((void *)((uintptr_t)
3637 : chunk + (i << pagesize_2pow)),
3638 : (npages << pagesize_2pow));
3639 : # ifdef MALLOC_STATS
3640 : arena->stats.ndecommit++;
3641 : arena->stats.decommitted += npages;
3642 : # endif
3643 : #endif
3644 : #ifdef MALLOC_STATS
3645 162169 : arena->stats.committed -= npages;
3646 : #endif
3647 :
3648 : #ifndef MALLOC_DECOMMIT
3649 162169 : madvise((void *)((uintptr_t)chunk + (i <<
3650 : pagesize_2pow)), (npages << pagesize_2pow),
3651 : MADV_FREE);
3652 : # ifdef MALLOC_DOUBLE_PURGE
3653 : madvised = true;
3654 : # endif
3655 : #endif
3656 : #ifdef MALLOC_STATS
3657 162169 : arena->stats.nmadvise++;
3658 162169 : arena->stats.purged += npages;
3659 : #endif
3660 162169 : if (arena->ndirty <= (opt_dirty_max >> 1))
3661 3981 : break;
3662 : }
3663 : }
3664 :
3665 34780 : if (chunk->ndirty == 0) {
3666 32535 : arena_chunk_tree_dirty_remove(&arena->chunks_dirty,
3667 : chunk);
3668 : }
3669 : #ifdef MALLOC_DOUBLE_PURGE
3670 : if (madvised) {
3671 : /* The chunk might already be in the list, but this
3672 : * makes sure it's at the front. */
3673 : LinkedList_Remove(&chunk->chunks_madvised_elem);
3674 : LinkedList_InsertHead(&arena->chunks_madvised, &chunk->chunks_madvised_elem);
3675 : }
3676 : #endif
3677 : }
3678 3981 : }
3679 :
3680 : static void
3681 6178032 : arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
3682 : {
3683 : arena_chunk_t *chunk;
3684 : size_t size, run_ind, run_pages;
3685 :
3686 6178032 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
3687 6178032 : run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk)
3688 : >> pagesize_2pow);
3689 6178032 : assert(run_ind >= arena_chunk_header_npages);
3690 6178032 : assert(run_ind < chunk_npages);
3691 6178032 : if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0)
3692 2276421 : size = chunk->map[run_ind].bits & ~pagesize_mask;
3693 : else
3694 3901611 : size = run->bin->run_size;
3695 6178032 : run_pages = (size >> pagesize_2pow);
3696 :
3697 : /* Mark pages as unallocated in the chunk map. */
3698 6178032 : if (dirty) {
3699 : size_t i;
3700 :
3701 34732483 : for (i = 0; i < run_pages; i++) {
3702 28554451 : assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY)
3703 : == 0);
3704 28554451 : chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY;
3705 : }
3706 :
3707 6178032 : if (chunk->ndirty == 0) {
3708 556607 : arena_chunk_tree_dirty_insert(&arena->chunks_dirty,
3709 : chunk);
3710 : }
3711 6178032 : chunk->ndirty += run_pages;
3712 6178032 : arena->ndirty += run_pages;
3713 : } else {
3714 : size_t i;
3715 :
3716 0 : for (i = 0; i < run_pages; i++) {
3717 0 : chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE |
3718 : CHUNK_MAP_ALLOCATED);
3719 : }
3720 : }
3721 6178032 : chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3722 : pagesize_mask);
3723 12356064 : chunk->map[run_ind+run_pages-1].bits = size |
3724 6178032 : (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3725 :
3726 : /* Try to coalesce forward. */
3727 12305487 : if (run_ind + run_pages < chunk_npages &&
3728 6127455 : (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) {
3729 3799839 : size_t nrun_size = chunk->map[run_ind+run_pages].bits &
3730 : ~pagesize_mask;
3731 :
3732 : /*
3733 : * Remove successor from runs_avail; the coalesced run is
3734 : * inserted later.
3735 : */
3736 3799839 : arena_avail_tree_remove(&arena->runs_avail,
3737 3799839 : &chunk->map[run_ind+run_pages]);
3738 :
3739 3799839 : size += nrun_size;
3740 3799839 : run_pages = size >> pagesize_2pow;
3741 :
3742 3799839 : assert((chunk->map[run_ind+run_pages-1].bits & ~pagesize_mask)
3743 : == nrun_size);
3744 3799839 : chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3745 : pagesize_mask);
3746 7599678 : chunk->map[run_ind+run_pages-1].bits = size |
3747 3799839 : (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3748 : }
3749 :
3750 : /* Try to coalesce backward. */
3751 6178032 : if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits &
3752 : CHUNK_MAP_ALLOCATED) == 0) {
3753 1025273 : size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask;
3754 :
3755 1025273 : run_ind -= prun_size >> pagesize_2pow;
3756 :
3757 : /*
3758 : * Remove predecessor from runs_avail; the coalesced run is
3759 : * inserted later.
3760 : */
3761 1025273 : arena_avail_tree_remove(&arena->runs_avail,
3762 1025273 : &chunk->map[run_ind]);
3763 :
3764 1025273 : size += prun_size;
3765 1025273 : run_pages = size >> pagesize_2pow;
3766 :
3767 1025273 : assert((chunk->map[run_ind].bits & ~pagesize_mask) ==
3768 : prun_size);
3769 1025273 : chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
3770 : pagesize_mask);
3771 2050546 : chunk->map[run_ind+run_pages-1].bits = size |
3772 1025273 : (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
3773 : }
3774 :
3775 : /* Insert into runs_avail, now that coalescing is complete. */
3776 6178032 : arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]);
3777 :
3778 : /* Deallocate chunk if it is now completely unused. */
3779 12356064 : if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask |
3780 6178032 : CHUNK_MAP_ALLOCATED)) == arena_maxclass)
3781 37150 : arena_chunk_dealloc(arena, chunk);
3782 :
3783 : /* Enforce opt_dirty_max. */
3784 6178032 : if (arena->ndirty > opt_dirty_max)
3785 3981 : arena_purge(arena);
3786 6178032 : }
3787 :
3788 : static void
3789 0 : arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
3790 : size_t oldsize, size_t newsize)
3791 : {
3792 0 : size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
3793 0 : size_t head_npages = (oldsize - newsize) >> pagesize_2pow;
3794 :
3795 0 : assert(oldsize > newsize);
3796 :
3797 : /*
3798 : * Update the chunk map so that arena_run_dalloc() can treat the
3799 : * leading run as separately allocated.
3800 : */
3801 0 : chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE |
3802 : CHUNK_MAP_ALLOCATED;
3803 0 : chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE |
3804 : CHUNK_MAP_ALLOCATED;
3805 :
3806 0 : arena_run_dalloc(arena, run, false);
3807 0 : }
3808 :
3809 : static void
3810 702 : arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
3811 : size_t oldsize, size_t newsize, bool dirty)
3812 : {
3813 702 : size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
3814 702 : size_t npages = newsize >> pagesize_2pow;
3815 :
3816 702 : assert(oldsize > newsize);
3817 :
3818 : /*
3819 : * Update the chunk map so that arena_run_dalloc() can treat the
3820 : * trailing run as separately allocated.
3821 : */
3822 702 : chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE |
3823 : CHUNK_MAP_ALLOCATED;
3824 1404 : chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE
3825 702 : | CHUNK_MAP_ALLOCATED;
3826 :
3827 702 : arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
3828 : dirty);
3829 702 : }
3830 :
3831 : static arena_run_t *
3832 6129663 : arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
3833 : {
3834 : arena_chunk_map_t *mapelm;
3835 : arena_run_t *run;
3836 : unsigned i, remainder;
3837 :
3838 : /* Look for a usable run. */
3839 6129663 : mapelm = arena_run_tree_first(&bin->runs);
3840 6129663 : if (mapelm != NULL) {
3841 : /* run is guaranteed to have available space. */
3842 1790091 : arena_run_tree_remove(&bin->runs, mapelm);
3843 1790091 : run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
3844 : #ifdef MALLOC_STATS
3845 1790091 : bin->stats.reruns++;
3846 : #endif
3847 1790091 : return (run);
3848 : }
3849 : /* No existing runs have any space available. */
3850 :
3851 : /* Allocate a new run. */
3852 4339572 : run = arena_run_alloc(arena, bin, bin->run_size, false, false);
3853 4339572 : if (run == NULL)
3854 0 : return (NULL);
3855 : /*
3856 : * Don't initialize if a race in arena_run_alloc() allowed an existing
3857 : * run to become usable.
3858 : */
3859 4339572 : if (run == bin->runcur)
3860 0 : return (run);
3861 :
3862 : VALGRIND_MALLOCLIKE_BLOCK(run, sizeof(arena_run_t) + (sizeof(unsigned) *
3863 : (bin->regs_mask_nelms - 1)), 0, false);
3864 :
3865 : /* Initialize run internals. */
3866 4339572 : run->bin = bin;
3867 :
3868 10790691 : for (i = 0; i < bin->regs_mask_nelms - 1; i++)
3869 6451119 : run->regs_mask[i] = UINT_MAX;
3870 4339572 : remainder = bin->nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1);
3871 4339572 : if (remainder == 0)
3872 0 : run->regs_mask[i] = UINT_MAX;
3873 : else {
3874 : /* The last element has spare bits that need to be unset. */
3875 4339572 : run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3))
3876 4339572 : - remainder));
3877 : }
3878 :
3879 4339572 : run->regs_minelm = 0;
3880 :
3881 4339572 : run->nfree = bin->nregs;
3882 : #ifdef MALLOC_DEBUG
3883 4339572 : run->magic = ARENA_RUN_MAGIC;
3884 : #endif
3885 :
3886 : #ifdef MALLOC_STATS
3887 4339572 : bin->stats.nruns++;
3888 4339572 : bin->stats.curruns++;
3889 4339572 : if (bin->stats.curruns > bin->stats.highruns)
3890 1127454 : bin->stats.highruns = bin->stats.curruns;
3891 : #endif
3892 4339572 : return (run);
3893 : }
3894 :
3895 : /* bin->runcur must have space available before this function is called. */
3896 : static inline void *
3897 123481815 : arena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run)
3898 : {
3899 : void *ret;
3900 :
3901 123481815 : assert(run->magic == ARENA_RUN_MAGIC);
3902 123481815 : assert(run->nfree > 0);
3903 :
3904 123481815 : ret = arena_run_reg_alloc(run, bin);
3905 123481815 : assert(ret != NULL);
3906 123481815 : run->nfree--;
3907 :
3908 123481815 : return (ret);
3909 : }
3910 :
3911 : /* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */
3912 : static void *
3913 6129663 : arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
3914 : {
3915 :
3916 6129663 : bin->runcur = arena_bin_nonfull_run_get(arena, bin);
3917 6129663 : if (bin->runcur == NULL)
3918 0 : return (NULL);
3919 6129663 : assert(bin->runcur->magic == ARENA_RUN_MAGIC);
3920 6129663 : assert(bin->runcur->nfree > 0);
3921 :
3922 6129663 : return (arena_bin_malloc_easy(arena, bin, bin->runcur));
3923 : }
3924 :
3925 : /*
3926 : * Calculate bin->run_size such that it meets the following constraints:
3927 : *
3928 : * *) bin->run_size >= min_run_size
3929 : * *) bin->run_size <= arena_maxclass
3930 : * *) bin->run_size <= RUN_MAX_SMALL
3931 : * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
3932 : *
3933 : * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are
3934 : * also calculated here, since these settings are all interdependent.
3935 : */
3936 : static size_t
3937 716436 : arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
3938 : {
3939 : size_t try_run_size, good_run_size;
3940 : unsigned good_nregs, good_mask_nelms, good_reg0_offset;
3941 : unsigned try_nregs, try_mask_nelms, try_reg0_offset;
3942 :
3943 716436 : assert(min_run_size >= pagesize);
3944 716436 : assert(min_run_size <= arena_maxclass);
3945 716436 : assert(min_run_size <= RUN_MAX_SMALL);
3946 :
3947 : /*
3948 : * Calculate known-valid settings before entering the run_size
3949 : * expansion loop, so that the first part of the loop always copies
3950 : * valid settings.
3951 : *
3952 : * The do..while loop iteratively reduces the number of regions until
3953 : * the run header and the regions no longer overlap. A closed formula
3954 : * would be quite messy, since there is an interdependency between the
3955 : * header's mask length and the number of regions.
3956 : */
3957 716436 : try_run_size = min_run_size;
3958 716436 : try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size)
3959 : + 1; /* Counter-act try_nregs-- in loop. */
3960 : do {
3961 1472674 : try_nregs--;
3962 2945348 : try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
3963 1472674 : ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0);
3964 1472674 : try_reg0_offset = try_run_size - (try_nregs * bin->reg_size);
3965 1472674 : } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))
3966 1472674 : > try_reg0_offset);
3967 :
3968 : /* run_size expansion loop. */
3969 : do {
3970 : /*
3971 : * Copy valid settings before trying more aggressive settings.
3972 : */
3973 855743 : good_run_size = try_run_size;
3974 855743 : good_nregs = try_nregs;
3975 855743 : good_mask_nelms = try_mask_nelms;
3976 855743 : good_reg0_offset = try_reg0_offset;
3977 :
3978 : /* Try more aggressive settings. */
3979 855743 : try_run_size += pagesize;
3980 1711486 : try_nregs = ((try_run_size - sizeof(arena_run_t)) /
3981 855743 : bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */
3982 : do {
3983 2467724 : try_nregs--;
3984 4935448 : try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
3985 2467724 : ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ?
3986 : 1 : 0);
3987 2467724 : try_reg0_offset = try_run_size - (try_nregs *
3988 2467724 : bin->reg_size);
3989 2467724 : } while (sizeof(arena_run_t) + (sizeof(unsigned) *
3990 2467724 : (try_mask_nelms - 1)) > try_reg0_offset);
3991 1711486 : } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL
3992 796040 : && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
3993 1611981 : && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
3994 :
3995 716436 : assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1))
3996 : <= good_reg0_offset);
3997 716436 : assert((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs);
3998 :
3999 : /* Copy final settings. */
4000 716436 : bin->run_size = good_run_size;
4001 716436 : bin->nregs = good_nregs;
4002 716436 : bin->regs_mask_nelms = good_mask_nelms;
4003 716436 : bin->reg0_offset = good_reg0_offset;
4004 :
4005 716436 : return (good_run_size);
4006 : }
4007 :
4008 : #ifdef MALLOC_BALANCE
4009 : static inline void
4010 : arena_lock_balance(arena_t *arena)
4011 : {
4012 : unsigned contention;
4013 :
4014 : contention = malloc_spin_lock(&arena->lock);
4015 : if (narenas > 1) {
4016 : /*
4017 : * Calculate the exponentially averaged contention for this
4018 : * arena. Due to integer math always rounding down, this value
4019 : * decays somewhat faster then normal.
4020 : */
4021 : arena->contention = (((uint64_t)arena->contention
4022 : * (uint64_t)((1U << BALANCE_ALPHA_INV_2POW)-1))
4023 : + (uint64_t)contention) >> BALANCE_ALPHA_INV_2POW;
4024 : if (arena->contention >= opt_balance_threshold)
4025 : arena_lock_balance_hard(arena);
4026 : }
4027 : }
4028 :
4029 : static void
4030 : arena_lock_balance_hard(arena_t *arena)
4031 : {
4032 : uint32_t ind;
4033 :
4034 : arena->contention = 0;
4035 : #ifdef MALLOC_STATS
4036 : arena->stats.nbalance++;
4037 : #endif
4038 : ind = PRN(balance, narenas_2pow);
4039 : if (arenas[ind] != NULL) {
4040 : #ifdef MOZ_MEMORY_WINDOWS
4041 : TlsSetValue(tlsIndex, arenas[ind]);
4042 : #else
4043 : arenas_map = arenas[ind];
4044 : #endif
4045 : } else {
4046 : malloc_spin_lock(&arenas_lock);
4047 : if (arenas[ind] != NULL) {
4048 : #ifdef MOZ_MEMORY_WINDOWS
4049 : TlsSetValue(tlsIndex, arenas[ind]);
4050 : #else
4051 : arenas_map = arenas[ind];
4052 : #endif
4053 : } else {
4054 : #ifdef MOZ_MEMORY_WINDOWS
4055 : TlsSetValue(tlsIndex, arenas_extend(ind));
4056 : #else
4057 : arenas_map = arenas_extend(ind);
4058 : #endif
4059 : }
4060 : malloc_spin_unlock(&arenas_lock);
4061 : }
4062 : }
4063 : #endif
4064 :
4065 : static inline void *
4066 123481101 : arena_malloc_small(arena_t *arena, size_t size, bool zero)
4067 : {
4068 : void *ret;
4069 : arena_bin_t *bin;
4070 : arena_run_t *run;
4071 :
4072 123481101 : if (size < small_min) {
4073 : /* Tiny. */
4074 17536923 : size = pow2_ceil(size);
4075 17536740 : bin = &arena->bins[ffs((int)(size >> (TINY_MIN_2POW +
4076 : 1)))];
4077 : #if (!defined(NDEBUG) || defined(MALLOC_STATS))
4078 : /*
4079 : * Bin calculation is always correct, but we may need
4080 : * to fix size for the purposes of assertions and/or
4081 : * stats accuracy.
4082 : */
4083 17536740 : if (size < (1U << TINY_MIN_2POW))
4084 1194758 : size = (1U << TINY_MIN_2POW);
4085 : #endif
4086 105944178 : } else if (size <= small_max) {
4087 : /* Quantum-spaced. */
4088 95606285 : size = QUANTUM_CEILING(size);
4089 95606285 : bin = &arena->bins[ntbins + (size >> opt_quantum_2pow)
4090 : - 1];
4091 : } else {
4092 : /* Sub-page. */
4093 10337893 : size = pow2_ceil(size);
4094 20675786 : bin = &arena->bins[ntbins + nqbins
4095 10337893 : + (ffs((int)(size >> opt_small_max_2pow)) - 2)];
4096 : }
4097 123480918 : assert(size == bin->reg_size);
4098 :
4099 : #ifdef MALLOC_BALANCE
4100 : arena_lock_balance(arena);
4101 : #else
4102 123480918 : malloc_spin_lock(&arena->lock);
4103 : #endif
4104 123481815 : if ((run = bin->runcur) != NULL && run->nfree > 0)
4105 117352152 : ret = arena_bin_malloc_easy(arena, bin, run);
4106 : else
4107 6129663 : ret = arena_bin_malloc_hard(arena, bin);
4108 :
4109 123481815 : if (ret == NULL) {
4110 0 : malloc_spin_unlock(&arena->lock);
4111 0 : return (NULL);
4112 : }
4113 :
4114 : #ifdef MALLOC_STATS
4115 123481815 : bin->stats.nrequests++;
4116 123481815 : arena->stats.nmalloc_small++;
4117 123481815 : arena->stats.allocated_small += size;
4118 : #endif
4119 123481815 : malloc_spin_unlock(&arena->lock);
4120 :
4121 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
4122 123481435 : if (zero == false) {
4123 : #ifdef MALLOC_FILL
4124 116047151 : if (opt_junk)
4125 116047153 : memset(ret, 0xa5, size);
4126 0 : else if (opt_zero)
4127 0 : memset(ret, 0, size);
4128 : #endif
4129 : } else
4130 7434284 : memset(ret, 0, size);
4131 :
4132 123481435 : return (ret);
4133 : }
4134 :
4135 : static void *
4136 2281792 : arena_malloc_large(arena_t *arena, size_t size, bool zero)
4137 : {
4138 : void *ret;
4139 :
4140 : /* Large allocation. */
4141 2281792 : size = PAGE_CEILING(size);
4142 : #ifdef MALLOC_BALANCE
4143 : arena_lock_balance(arena);
4144 : #else
4145 2281792 : malloc_spin_lock(&arena->lock);
4146 : #endif
4147 2281792 : ret = (void *)arena_run_alloc(arena, NULL, size, true, zero);
4148 2281792 : if (ret == NULL) {
4149 0 : malloc_spin_unlock(&arena->lock);
4150 0 : return (NULL);
4151 : }
4152 : #ifdef MALLOC_STATS
4153 2281792 : arena->stats.nmalloc_large++;
4154 2281792 : arena->stats.allocated_large += size;
4155 : #endif
4156 2281792 : malloc_spin_unlock(&arena->lock);
4157 :
4158 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
4159 2281791 : if (zero == false) {
4160 : #ifdef MALLOC_FILL
4161 1729696 : if (opt_junk)
4162 1729696 : memset(ret, 0xa5, size);
4163 0 : else if (opt_zero)
4164 0 : memset(ret, 0, size);
4165 : #endif
4166 : }
4167 :
4168 2281791 : return (ret);
4169 : }
4170 :
4171 : static inline void *
4172 125762809 : arena_malloc(arena_t *arena, size_t size, bool zero)
4173 : {
4174 :
4175 125762809 : assert(arena != NULL);
4176 125762809 : assert(arena->magic == ARENA_MAGIC);
4177 125762809 : assert(size != 0);
4178 125762809 : assert(QUANTUM_CEILING(size) <= arena_maxclass);
4179 :
4180 125762809 : if (size <= bin_maxclass) {
4181 123481017 : return (arena_malloc_small(arena, size, zero));
4182 : } else
4183 2281792 : return (arena_malloc_large(arena, size, zero));
4184 : }
4185 :
4186 : static inline void *
4187 113343083 : imalloc(size_t size)
4188 : {
4189 :
4190 113343083 : assert(size != 0);
4191 :
4192 113343083 : if (size <= arena_maxclass)
4193 113339406 : return (arena_malloc(choose_arena(), size, false));
4194 : else
4195 3677 : return (huge_malloc(size, false));
4196 : }
4197 :
4198 : static inline void *
4199 7986415 : icalloc(size_t size)
4200 : {
4201 :
4202 7986415 : if (size <= arena_maxclass)
4203 7986379 : return (arena_malloc(choose_arena(), size, true));
4204 : else
4205 36 : return (huge_malloc(size, true));
4206 : }
4207 :
4208 : /* Only handles large allocations that require more than page alignment. */
4209 : static void *
4210 0 : arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
4211 : {
4212 : void *ret;
4213 : size_t offset;
4214 : arena_chunk_t *chunk;
4215 :
4216 0 : assert((size & pagesize_mask) == 0);
4217 0 : assert((alignment & pagesize_mask) == 0);
4218 :
4219 : #ifdef MALLOC_BALANCE
4220 : arena_lock_balance(arena);
4221 : #else
4222 0 : malloc_spin_lock(&arena->lock);
4223 : #endif
4224 0 : ret = (void *)arena_run_alloc(arena, NULL, alloc_size, true, false);
4225 0 : if (ret == NULL) {
4226 0 : malloc_spin_unlock(&arena->lock);
4227 0 : return (NULL);
4228 : }
4229 :
4230 0 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
4231 :
4232 0 : offset = (uintptr_t)ret & (alignment - 1);
4233 0 : assert((offset & pagesize_mask) == 0);
4234 0 : assert(offset < alloc_size);
4235 0 : if (offset == 0)
4236 0 : arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, alloc_size, size, false);
4237 : else {
4238 : size_t leadsize, trailsize;
4239 :
4240 0 : leadsize = alignment - offset;
4241 0 : if (leadsize > 0) {
4242 0 : arena_run_trim_head(arena, chunk, (arena_run_t*)ret, alloc_size,
4243 : alloc_size - leadsize);
4244 0 : ret = (void *)((uintptr_t)ret + leadsize);
4245 : }
4246 :
4247 0 : trailsize = alloc_size - leadsize - size;
4248 0 : if (trailsize != 0) {
4249 : /* Trim trailing space. */
4250 0 : assert(trailsize < alloc_size);
4251 0 : arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, size + trailsize,
4252 : size, false);
4253 : }
4254 : }
4255 :
4256 : #ifdef MALLOC_STATS
4257 0 : arena->stats.nmalloc_large++;
4258 0 : arena->stats.allocated_large += size;
4259 : #endif
4260 0 : malloc_spin_unlock(&arena->lock);
4261 :
4262 : VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
4263 : #ifdef MALLOC_FILL
4264 0 : if (opt_junk)
4265 0 : memset(ret, 0xa5, size);
4266 0 : else if (opt_zero)
4267 0 : memset(ret, 0, size);
4268 : #endif
4269 0 : return (ret);
4270 : }
4271 :
4272 : static inline void *
4273 30860 : ipalloc(size_t alignment, size_t size)
4274 : {
4275 : void *ret;
4276 : size_t ceil_size;
4277 :
4278 : /*
4279 : * Round size up to the nearest multiple of alignment.
4280 : *
4281 : * This done, we can take advantage of the fact that for each small
4282 : * size class, every object is aligned at the smallest power of two
4283 : * that is non-zero in the base two representation of the size. For
4284 : * example:
4285 : *
4286 : * Size | Base 2 | Minimum alignment
4287 : * -----+----------+------------------
4288 : * 96 | 1100000 | 32
4289 : * 144 | 10100000 | 32
4290 : * 192 | 11000000 | 64
4291 : *
4292 : * Depending on runtime settings, it is possible that arena_malloc()
4293 : * will further round up to a power of two, but that never causes
4294 : * correctness issues.
4295 : */
4296 30860 : ceil_size = (size + (alignment - 1)) & (-alignment);
4297 : /*
4298 : * (ceil_size < size) protects against the combination of maximal
4299 : * alignment and size greater than maximal alignment.
4300 : */
4301 30860 : if (ceil_size < size) {
4302 : /* size_t overflow. */
4303 0 : return (NULL);
4304 : }
4305 :
4306 30860 : if (ceil_size <= pagesize || (alignment <= pagesize
4307 12 : && ceil_size <= arena_maxclass))
4308 30860 : ret = arena_malloc(choose_arena(), ceil_size, false);
4309 : else {
4310 : size_t run_size;
4311 :
4312 : /*
4313 : * We can't achieve sub-page alignment, so round up alignment
4314 : * permanently; it makes later calculations simpler.
4315 : */
4316 0 : alignment = PAGE_CEILING(alignment);
4317 0 : ceil_size = PAGE_CEILING(size);
4318 : /*
4319 : * (ceil_size < size) protects against very large sizes within
4320 : * pagesize of SIZE_T_MAX.
4321 : *
4322 : * (ceil_size + alignment < ceil_size) protects against the
4323 : * combination of maximal alignment and ceil_size large enough
4324 : * to cause overflow. This is similar to the first overflow
4325 : * check above, but it needs to be repeated due to the new
4326 : * ceil_size value, which may now be *equal* to maximal
4327 : * alignment, whereas before we only detected overflow if the
4328 : * original size was *greater* than maximal alignment.
4329 : */
4330 0 : if (ceil_size < size || ceil_size + alignment < ceil_size) {
4331 : /* size_t overflow. */
4332 0 : return (NULL);
4333 : }
4334 :
4335 : /*
4336 : * Calculate the size of the over-size run that arena_palloc()
4337 : * would need to allocate in order to guarantee the alignment.
4338 : */
4339 0 : if (ceil_size >= alignment)
4340 0 : run_size = ceil_size + alignment - pagesize;
4341 : else {
4342 : /*
4343 : * It is possible that (alignment << 1) will cause
4344 : * overflow, but it doesn't matter because we also
4345 : * subtract pagesize, which in the case of overflow
4346 : * leaves us with a very large run_size. That causes
4347 : * the first conditional below to fail, which means
4348 : * that the bogus run_size value never gets used for
4349 : * anything important.
4350 : */
4351 0 : run_size = (alignment << 1) - pagesize;
4352 : }
4353 :
4354 0 : if (run_size <= arena_maxclass) {
4355 0 : ret = arena_palloc(choose_arena(), alignment, ceil_size,
4356 : run_size);
4357 0 : } else if (alignment <= chunksize)
4358 0 : ret = huge_malloc(ceil_size, false);
4359 : else
4360 0 : ret = huge_palloc(alignment, ceil_size);
4361 : }
4362 :
4363 30860 : assert(((uintptr_t)ret & (alignment - 1)) == 0);
4364 30860 : return (ret);
4365 : }
4366 :
4367 : /* Return the size of the allocation pointed to by ptr. */
4368 : static size_t
4369 18939099 : arena_salloc(const void *ptr)
4370 : {
4371 : size_t ret;
4372 : arena_chunk_t *chunk;
4373 : size_t pageind, mapbits;
4374 :
4375 18939099 : assert(ptr != NULL);
4376 18939099 : assert(CHUNK_ADDR2BASE(ptr) != ptr);
4377 :
4378 18939099 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4379 18939099 : pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
4380 18939099 : mapbits = chunk->map[pageind].bits;
4381 18939099 : assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
4382 18939099 : if ((mapbits & CHUNK_MAP_LARGE) == 0) {
4383 18304078 : arena_run_t *run = (arena_run_t *)(mapbits & ~pagesize_mask);
4384 18304078 : assert(run->magic == ARENA_RUN_MAGIC);
4385 18304078 : ret = run->bin->reg_size;
4386 : } else {
4387 635021 : ret = mapbits & ~pagesize_mask;
4388 635021 : assert(ret != 0);
4389 : }
4390 :
4391 18939099 : return (ret);
4392 : }
4393 :
4394 : #if (defined(MALLOC_VALIDATE) || defined(MOZ_MEMORY_DARWIN))
4395 : /*
4396 : * Validate ptr before assuming that it points to an allocation. Currently,
4397 : * the following validation is performed:
4398 : *
4399 : * + Check that ptr is not NULL.
4400 : *
4401 : * + Check that ptr lies within a mapped chunk.
4402 : */
4403 : static inline size_t
4404 14218183 : isalloc_validate(const void *ptr)
4405 : {
4406 : arena_chunk_t *chunk;
4407 :
4408 14218183 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4409 14218183 : if (chunk == NULL)
4410 0 : return (0);
4411 :
4412 14218183 : if (malloc_rtree_get(chunk_rtree, (uintptr_t)chunk) == NULL)
4413 0 : return (0);
4414 :
4415 14218192 : if (chunk != ptr) {
4416 14218023 : assert(chunk->arena->magic == ARENA_MAGIC);
4417 14218023 : return (arena_salloc(ptr));
4418 : } else {
4419 : size_t ret;
4420 : extent_node_t *node;
4421 : extent_node_t key;
4422 :
4423 : /* Chunk. */
4424 169 : key.addr = (void *)chunk;
4425 169 : malloc_mutex_lock(&huge_mtx);
4426 169 : node = extent_tree_ad_search(&huge, &key);
4427 169 : if (node != NULL)
4428 169 : ret = node->size;
4429 : else
4430 0 : ret = 0;
4431 169 : malloc_mutex_unlock(&huge_mtx);
4432 169 : return (ret);
4433 : }
4434 : }
4435 : #endif
4436 :
4437 : static inline size_t
4438 4723563 : isalloc(const void *ptr)
4439 : {
4440 : size_t ret;
4441 : arena_chunk_t *chunk;
4442 :
4443 4723563 : assert(ptr != NULL);
4444 :
4445 4723563 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4446 4723563 : if (chunk != ptr) {
4447 : /* Region. */
4448 4721119 : assert(chunk->arena->magic == ARENA_MAGIC);
4449 :
4450 4721119 : ret = arena_salloc(ptr);
4451 : } else {
4452 : extent_node_t *node, key;
4453 :
4454 : /* Chunk (huge allocation). */
4455 :
4456 2444 : malloc_mutex_lock(&huge_mtx);
4457 :
4458 : /* Extract from tree of huge allocations. */
4459 2444 : key.addr = __DECONST(void *, ptr);
4460 2444 : node = extent_tree_ad_search(&huge, &key);
4461 2444 : assert(node != NULL);
4462 :
4463 2444 : ret = node->size;
4464 :
4465 2444 : malloc_mutex_unlock(&huge_mtx);
4466 : }
4467 :
4468 4723563 : return (ret);
4469 : }
4470 :
4471 : static inline void
4472 119028910 : arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4473 : arena_chunk_map_t *mapelm)
4474 : {
4475 : arena_run_t *run;
4476 : arena_bin_t *bin;
4477 : size_t size;
4478 :
4479 119028910 : run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
4480 119028910 : assert(run->magic == ARENA_RUN_MAGIC);
4481 119028910 : bin = run->bin;
4482 119028910 : size = bin->reg_size;
4483 :
4484 : #ifdef MALLOC_FILL
4485 119028910 : if (opt_junk)
4486 119028910 : memset(ptr, 0x5a, size);
4487 : #endif
4488 :
4489 119028910 : arena_run_reg_dalloc(run, bin, ptr, size);
4490 119028910 : run->nfree++;
4491 :
4492 119028910 : if (run->nfree == bin->nregs) {
4493 : /* Deallocate run. */
4494 3901611 : if (run == bin->runcur)
4495 3346264 : bin->runcur = NULL;
4496 555347 : else if (bin->nregs != 1) {
4497 1110694 : size_t run_pageind = (((uintptr_t)run -
4498 555347 : (uintptr_t)chunk)) >> pagesize_2pow;
4499 555347 : arena_chunk_map_t *run_mapelm =
4500 555347 : &chunk->map[run_pageind];
4501 : /*
4502 : * This block's conditional is necessary because if the
4503 : * run only contains one region, then it never gets
4504 : * inserted into the non-full runs tree.
4505 : */
4506 555347 : assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
4507 : run_mapelm);
4508 555347 : arena_run_tree_remove(&bin->runs, run_mapelm);
4509 : }
4510 : #ifdef MALLOC_DEBUG
4511 3901611 : run->magic = 0;
4512 : #endif
4513 : VALGRIND_FREELIKE_BLOCK(run, 0);
4514 3901611 : arena_run_dalloc(arena, run, true);
4515 : #ifdef MALLOC_STATS
4516 3901611 : bin->stats.curruns--;
4517 : #endif
4518 115127299 : } else if (run->nfree == 1 && run != bin->runcur) {
4519 : /*
4520 : * Make sure that bin->runcur always refers to the lowest
4521 : * non-full run, if one exists.
4522 : */
4523 2555071 : if (bin->runcur == NULL)
4524 67990 : bin->runcur = run;
4525 2487081 : else if ((uintptr_t)run < (uintptr_t)bin->runcur) {
4526 : /* Switch runcur. */
4527 1678159 : if (bin->runcur->nfree > 0) {
4528 1623386 : arena_chunk_t *runcur_chunk =
4529 1623386 : (arena_chunk_t*)CHUNK_ADDR2BASE(bin->runcur);
4530 1623386 : size_t runcur_pageind =
4531 3246772 : (((uintptr_t)bin->runcur -
4532 1623386 : (uintptr_t)runcur_chunk)) >> pagesize_2pow;
4533 1623386 : arena_chunk_map_t *runcur_mapelm =
4534 1623386 : &runcur_chunk->map[runcur_pageind];
4535 :
4536 : /* Insert runcur. */
4537 1623386 : assert(arena_run_tree_search(&bin->runs,
4538 : runcur_mapelm) == NULL);
4539 1623386 : arena_run_tree_insert(&bin->runs,
4540 : runcur_mapelm);
4541 : }
4542 1678159 : bin->runcur = run;
4543 : } else {
4544 1617844 : size_t run_pageind = (((uintptr_t)run -
4545 808922 : (uintptr_t)chunk)) >> pagesize_2pow;
4546 808922 : arena_chunk_map_t *run_mapelm =
4547 808922 : &chunk->map[run_pageind];
4548 :
4549 808922 : assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
4550 : NULL);
4551 808922 : arena_run_tree_insert(&bin->runs, run_mapelm);
4552 : }
4553 : }
4554 : #ifdef MALLOC_STATS
4555 119028910 : arena->stats.allocated_small -= size;
4556 119028910 : arena->stats.ndalloc_small++;
4557 : #endif
4558 119028910 : }
4559 :
4560 : static void
4561 2275716 : arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
4562 : {
4563 : /* Large allocation. */
4564 2275716 : malloc_spin_lock(&arena->lock);
4565 :
4566 : #ifdef MALLOC_FILL
4567 : #ifndef MALLOC_STATS
4568 : if (opt_junk)
4569 : #endif
4570 : #endif
4571 : {
4572 2275719 : size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
4573 : pagesize_2pow;
4574 2275719 : size_t size = chunk->map[pageind].bits & ~pagesize_mask;
4575 :
4576 : #ifdef MALLOC_FILL
4577 : #ifdef MALLOC_STATS
4578 2275719 : if (opt_junk)
4579 : #endif
4580 2275719 : memset(ptr, 0x5a, size);
4581 : #endif
4582 : #ifdef MALLOC_STATS
4583 2275719 : arena->stats.allocated_large -= size;
4584 : #endif
4585 : }
4586 : #ifdef MALLOC_STATS
4587 2275719 : arena->stats.ndalloc_large++;
4588 : #endif
4589 :
4590 2275719 : arena_run_dalloc(arena, (arena_run_t *)ptr, true);
4591 2275719 : malloc_spin_unlock(&arena->lock);
4592 2275719 : }
4593 :
4594 : static inline void
4595 121279191 : arena_dalloc(void *ptr, size_t offset)
4596 : {
4597 : arena_chunk_t *chunk;
4598 : arena_t *arena;
4599 : size_t pageind;
4600 : arena_chunk_map_t *mapelm;
4601 :
4602 121279191 : assert(ptr != NULL);
4603 121279191 : assert(offset != 0);
4604 121279191 : assert(CHUNK_ADDR2OFFSET(ptr) == offset);
4605 :
4606 121279191 : chunk = (arena_chunk_t *) ((uintptr_t)ptr - offset);
4607 121279191 : arena = chunk->arena;
4608 121279191 : assert(arena != NULL);
4609 121279191 : assert(arena->magic == ARENA_MAGIC);
4610 :
4611 121279191 : pageind = offset >> pagesize_2pow;
4612 121279191 : mapelm = &chunk->map[pageind];
4613 121279191 : assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
4614 121279191 : if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
4615 : /* Small allocation. */
4616 119003475 : malloc_spin_lock(&arena->lock);
4617 119028910 : arena_dalloc_small(arena, chunk, ptr, mapelm);
4618 119028910 : malloc_spin_unlock(&arena->lock);
4619 : } else
4620 2275716 : arena_dalloc_large(arena, chunk, ptr);
4621 : VALGRIND_FREELIKE_BLOCK(ptr, 0);
4622 121303509 : }
4623 :
4624 : static inline void
4625 4408108 : idalloc(void *ptr)
4626 : {
4627 : size_t offset;
4628 :
4629 4408108 : assert(ptr != NULL);
4630 :
4631 4408108 : offset = CHUNK_ADDR2OFFSET(ptr);
4632 4408108 : if (offset != 0)
4633 4406384 : arena_dalloc(ptr, offset);
4634 : else
4635 1724 : huge_dalloc(ptr);
4636 4408108 : }
4637 :
4638 : static void
4639 702 : arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4640 : size_t size, size_t oldsize)
4641 : {
4642 :
4643 702 : assert(size < oldsize);
4644 :
4645 : /*
4646 : * Shrink the run, and make trailing pages available for other
4647 : * allocations.
4648 : */
4649 : #ifdef MALLOC_BALANCE
4650 : arena_lock_balance(arena);
4651 : #else
4652 702 : malloc_spin_lock(&arena->lock);
4653 : #endif
4654 702 : arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size,
4655 : true);
4656 : #ifdef MALLOC_STATS
4657 702 : arena->stats.allocated_large -= oldsize - size;
4658 : #endif
4659 702 : malloc_spin_unlock(&arena->lock);
4660 702 : }
4661 :
4662 : static bool
4663 181846 : arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
4664 : size_t size, size_t oldsize)
4665 : {
4666 181846 : size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow;
4667 181846 : size_t npages = oldsize >> pagesize_2pow;
4668 :
4669 181846 : assert(oldsize == (chunk->map[pageind].bits & ~pagesize_mask));
4670 :
4671 : /* Try to extend the run. */
4672 181846 : assert(size > oldsize);
4673 : #ifdef MALLOC_BALANCE
4674 : arena_lock_balance(arena);
4675 : #else
4676 181846 : malloc_spin_lock(&arena->lock);
4677 : #endif
4678 357265 : if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits
4679 420097 : & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits &
4680 122339 : ~pagesize_mask) >= size - oldsize) {
4681 : /*
4682 : * The next run is available and sufficiently large. Split the
4683 : * following run, then merge the first part with the existing
4684 : * allocation.
4685 : */
4686 233348 : arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
4687 116674 : ((pageind+npages) << pagesize_2pow)), size - oldsize, true,
4688 : false);
4689 :
4690 116674 : chunk->map[pageind].bits = size | CHUNK_MAP_LARGE |
4691 : CHUNK_MAP_ALLOCATED;
4692 116674 : chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE |
4693 : CHUNK_MAP_ALLOCATED;
4694 :
4695 : #ifdef MALLOC_STATS
4696 116674 : arena->stats.allocated_large += size - oldsize;
4697 : #endif
4698 116674 : malloc_spin_unlock(&arena->lock);
4699 116674 : return (false);
4700 : }
4701 65172 : malloc_spin_unlock(&arena->lock);
4702 :
4703 65172 : return (true);
4704 : }
4705 :
4706 : /*
4707 : * Try to resize a large allocation, in order to avoid copying. This will
4708 : * always fail if growing an object, and the following run is already in use.
4709 : */
4710 : static bool
4711 191562 : arena_ralloc_large(void *ptr, size_t size, size_t oldsize)
4712 : {
4713 : size_t psize;
4714 :
4715 191562 : psize = PAGE_CEILING(size);
4716 191562 : if (psize == oldsize) {
4717 : /* Same size class. */
4718 : #ifdef MALLOC_FILL
4719 9014 : if (opt_junk && size < oldsize) {
4720 9010 : memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize -
4721 : size);
4722 : }
4723 : #endif
4724 9014 : return (false);
4725 : } else {
4726 : arena_chunk_t *chunk;
4727 : arena_t *arena;
4728 :
4729 182548 : chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
4730 182548 : arena = chunk->arena;
4731 182548 : assert(arena->magic == ARENA_MAGIC);
4732 :
4733 182548 : if (psize < oldsize) {
4734 : #ifdef MALLOC_FILL
4735 : /* Fill before shrinking in order avoid a race. */
4736 702 : if (opt_junk) {
4737 702 : memset((void *)((uintptr_t)ptr + size), 0x5a,
4738 : oldsize - size);
4739 : }
4740 : #endif
4741 702 : arena_ralloc_large_shrink(arena, chunk, ptr, psize,
4742 : oldsize);
4743 702 : return (false);
4744 : } else {
4745 181846 : bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
4746 : psize, oldsize);
4747 : #ifdef MALLOC_FILL
4748 181846 : if (ret == false && opt_zero) {
4749 0 : memset((void *)((uintptr_t)ptr + oldsize), 0,
4750 : size - oldsize);
4751 : }
4752 : #endif
4753 181846 : return (ret);
4754 : }
4755 : }
4756 : }
4757 :
4758 : static void *
4759 4721020 : arena_ralloc(void *ptr, size_t size, size_t oldsize)
4760 : {
4761 : void *ret;
4762 : size_t copysize;
4763 :
4764 : /* Try to avoid moving the allocation. */
4765 4721020 : if (size < small_min) {
4766 391966 : if (oldsize < small_min &&
4767 111301 : ffs((int)(pow2_ceil(size) >> (TINY_MIN_2POW + 1)))
4768 111301 : == ffs((int)(pow2_ceil(oldsize) >> (TINY_MIN_2POW + 1))))
4769 24671 : goto IN_PLACE; /* Same size class. */
4770 4440355 : } else if (size <= small_max) {
4771 6989528 : if (oldsize >= small_min && oldsize <= small_max &&
4772 3462597 : (QUANTUM_CEILING(size) >> opt_quantum_2pow)
4773 3462597 : == (QUANTUM_CEILING(oldsize) >> opt_quantum_2pow))
4774 99224 : goto IN_PLACE; /* Same size class. */
4775 913424 : } else if (size <= bin_maxclass) {
4776 912451 : if (oldsize > small_max && oldsize <= bin_maxclass &&
4777 296660 : pow2_ceil(size) == pow2_ceil(oldsize))
4778 64452 : goto IN_PLACE; /* Same size class. */
4779 297633 : } else if (oldsize > bin_maxclass && oldsize <= arena_maxclass) {
4780 191562 : assert(size > bin_maxclass);
4781 191562 : if (arena_ralloc_large(ptr, size, oldsize) == false)
4782 126390 : return (ptr);
4783 : }
4784 :
4785 : /*
4786 : * If we get here, then size and oldsize are different enough that we
4787 : * need to move the object. In that case, fall back to allocating new
4788 : * space and copying.
4789 : */
4790 4406283 : ret = arena_malloc(choose_arena(), size, false);
4791 4406282 : if (ret == NULL)
4792 0 : return (NULL);
4793 :
4794 : /* Junk/zero-filling were already done by arena_malloc(). */
4795 4406282 : copysize = (size < oldsize) ? size : oldsize;
4796 : #ifdef VM_COPY_MIN
4797 : if (copysize >= VM_COPY_MIN)
4798 : pages_copy(ret, ptr, copysize);
4799 : else
4800 : #endif
4801 4406282 : memcpy(ret, ptr, copysize);
4802 4406282 : idalloc(ptr);
4803 4406283 : return (ret);
4804 : IN_PLACE:
4805 : #ifdef MALLOC_FILL
4806 188347 : if (opt_junk && size < oldsize)
4807 98059 : memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size);
4808 90288 : else if (opt_zero && size > oldsize)
4809 0 : memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
4810 : #endif
4811 188347 : return (ptr);
4812 : }
4813 :
4814 : static inline void *
4815 4723564 : iralloc(void *ptr, size_t size)
4816 : {
4817 : size_t oldsize;
4818 :
4819 4723564 : assert(ptr != NULL);
4820 4723564 : assert(size != 0);
4821 :
4822 4723564 : oldsize = isalloc(ptr);
4823 :
4824 : #ifndef MALLOC_VALGRIND
4825 4723563 : if (size <= arena_maxclass)
4826 4721018 : return (arena_ralloc(ptr, size, oldsize));
4827 : else
4828 2545 : return (huge_ralloc(ptr, size, oldsize));
4829 : #else
4830 : /*
4831 : * Valgrind does not provide a public interface for modifying an
4832 : * existing allocation, so use malloc/memcpy/free instead.
4833 : */
4834 : {
4835 : void *ret = imalloc(size);
4836 : if (ret != NULL) {
4837 : if (oldsize < size)
4838 : memcpy(ret, ptr, oldsize);
4839 : else
4840 : memcpy(ret, ptr, size);
4841 : idalloc(ptr);
4842 : }
4843 : return (ret);
4844 : }
4845 : #endif
4846 : }
4847 :
4848 : static bool
4849 19901 : arena_new(arena_t *arena)
4850 : {
4851 : unsigned i;
4852 : arena_bin_t *bin;
4853 : size_t pow2_size, prev_run_size;
4854 :
4855 19901 : if (malloc_spin_init(&arena->lock))
4856 0 : return (true);
4857 :
4858 : #ifdef MALLOC_STATS
4859 19901 : memset(&arena->stats, 0, sizeof(arena_stats_t));
4860 : #endif
4861 :
4862 : /* Initialize chunks. */
4863 19901 : arena_chunk_tree_dirty_new(&arena->chunks_dirty);
4864 : #ifdef MALLOC_DOUBLE_PURGE
4865 : LinkedList_Init(&arena->chunks_madvised);
4866 : #endif
4867 19901 : arena->spare = NULL;
4868 :
4869 19901 : arena->ndirty = 0;
4870 :
4871 19901 : arena_avail_tree_new(&arena->runs_avail);
4872 :
4873 : #ifdef MALLOC_BALANCE
4874 : arena->contention = 0;
4875 : #endif
4876 :
4877 : /* Initialize bins. */
4878 19901 : prev_run_size = pagesize;
4879 :
4880 : /* (2^n)-spaced tiny bins. */
4881 59703 : for (i = 0; i < ntbins; i++) {
4882 39802 : bin = &arena->bins[i];
4883 39802 : bin->runcur = NULL;
4884 39802 : arena_run_tree_new(&bin->runs);
4885 :
4886 39802 : bin->reg_size = (1U << (TINY_MIN_2POW + i));
4887 :
4888 39802 : prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
4889 :
4890 : #ifdef MALLOC_STATS
4891 39802 : memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
4892 : #endif
4893 : }
4894 :
4895 : /* Quantum-spaced bins. */
4896 656733 : for (; i < ntbins + nqbins; i++) {
4897 636832 : bin = &arena->bins[i];
4898 636832 : bin->runcur = NULL;
4899 636832 : arena_run_tree_new(&bin->runs);
4900 :
4901 636832 : bin->reg_size = quantum * (i - ntbins + 1);
4902 :
4903 636832 : pow2_size = pow2_ceil(quantum * (i - ntbins + 1));
4904 636832 : prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
4905 :
4906 : #ifdef MALLOC_STATS
4907 636832 : memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
4908 : #endif
4909 : }
4910 :
4911 : /* (2^n)-spaced sub-page bins. */
4912 59703 : for (; i < ntbins + nqbins + nsbins; i++) {
4913 39802 : bin = &arena->bins[i];
4914 39802 : bin->runcur = NULL;
4915 39802 : arena_run_tree_new(&bin->runs);
4916 :
4917 39802 : bin->reg_size = (small_max << (i - (ntbins + nqbins) + 1));
4918 :
4919 39802 : prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
4920 :
4921 : #ifdef MALLOC_STATS
4922 39802 : memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
4923 : #endif
4924 : }
4925 :
4926 : #ifdef MALLOC_DEBUG
4927 19901 : arena->magic = ARENA_MAGIC;
4928 : #endif
4929 :
4930 19901 : return (false);
4931 : }
4932 :
4933 : /* Create a new arena and insert it into the arenas array at index ind. */
4934 : static arena_t *
4935 19901 : arenas_extend(unsigned ind)
4936 : {
4937 : arena_t *ret;
4938 :
4939 : /* Allocate enough space for trailing bins. */
4940 19901 : ret = (arena_t *)base_alloc(sizeof(arena_t)
4941 19901 : + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
4942 19901 : if (ret != NULL && arena_new(ret) == false) {
4943 19901 : arenas[ind] = ret;
4944 19901 : return (ret);
4945 : }
4946 : /* Only reached if there is an OOM error. */
4947 :
4948 : /*
4949 : * OOM here is quite inconvenient to propagate, since dealing with it
4950 : * would require a check for failure in the fast path. Instead, punt
4951 : * by using arenas[0]. In practice, this is an extremely unlikely
4952 : * failure.
4953 : */
4954 0 : _malloc_message(_getprogname(),
4955 : ": (malloc) Error initializing arena\n", "", "");
4956 0 : if (opt_abort)
4957 0 : abort();
4958 :
4959 0 : return (arenas[0]);
4960 : }
4961 :
4962 : /*
4963 : * End arena.
4964 : */
4965 : /******************************************************************************/
4966 : /*
4967 : * Begin general internal functions.
4968 : */
4969 :
4970 : static void *
4971 5538 : huge_malloc(size_t size, bool zero)
4972 : {
4973 : void *ret;
4974 : size_t csize;
4975 : size_t psize;
4976 : extent_node_t *node;
4977 :
4978 : /* Allocate one or more contiguous chunks for this request. */
4979 :
4980 5538 : csize = CHUNK_CEILING(size);
4981 5538 : if (csize == 0) {
4982 : /* size is large enough to cause size_t wrap-around. */
4983 0 : return (NULL);
4984 : }
4985 :
4986 : /* Allocate an extent node with which to track the chunk. */
4987 5538 : node = base_node_alloc();
4988 5538 : if (node == NULL)
4989 0 : return (NULL);
4990 :
4991 5538 : ret = chunk_alloc(csize, zero, true);
4992 5538 : if (ret == NULL) {
4993 19 : base_node_dealloc(node);
4994 19 : return (NULL);
4995 : }
4996 :
4997 : /* Insert node into huge. */
4998 5519 : node->addr = ret;
4999 5519 : psize = PAGE_CEILING(size);
5000 5519 : node->size = psize;
5001 :
5002 5519 : malloc_mutex_lock(&huge_mtx);
5003 5519 : extent_tree_ad_insert(&huge, node);
5004 : #ifdef MALLOC_STATS
5005 5519 : huge_nmalloc++;
5006 :
5007 : /* Although we allocated space for csize bytes, we indicate that we've
5008 : * allocated only psize bytes.
5009 : *
5010 : * If DECOMMIT is defined, this is a reasonable thing to do, since
5011 : * we'll explicitly decommit the bytes in excess of psize.
5012 : *
5013 : * If DECOMMIT is not defined, then we're relying on the OS to be lazy
5014 : * about how it allocates physical pages to mappings. If we never
5015 : * touch the pages in excess of psize, the OS won't allocate a physical
5016 : * page, and we won't use more than psize bytes of physical memory.
5017 : *
5018 : * A correct program will only touch memory in excess of how much it
5019 : * requested if it first calls malloc_usable_size and finds out how
5020 : * much space it has to play with. But because we set node->size =
5021 : * psize above, malloc_usable_size will return psize, not csize, and
5022 : * the program will (hopefully) never touch bytes in excess of psize.
5023 : * Thus those bytes won't take up space in physical memory, and we can
5024 : * reasonably claim we never "allocated" them in the first place. */
5025 5519 : huge_allocated += psize;
5026 : #endif
5027 5519 : malloc_mutex_unlock(&huge_mtx);
5028 :
5029 : #ifdef MALLOC_DECOMMIT
5030 : if (csize - psize > 0)
5031 : pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
5032 : #endif
5033 :
5034 : #ifdef MALLOC_DECOMMIT
5035 : VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, zero);
5036 : #else
5037 : VALGRIND_MALLOCLIKE_BLOCK(ret, csize, 0, zero);
5038 : #endif
5039 :
5040 : #ifdef MALLOC_FILL
5041 5519 : if (zero == false) {
5042 5501 : if (opt_junk)
5043 : # ifdef MALLOC_DECOMMIT
5044 : memset(ret, 0xa5, psize);
5045 : # else
5046 5501 : memset(ret, 0xa5, csize);
5047 : # endif
5048 0 : else if (opt_zero)
5049 : # ifdef MALLOC_DECOMMIT
5050 : memset(ret, 0, psize);
5051 : # else
5052 0 : memset(ret, 0, csize);
5053 : # endif
5054 : }
5055 : #endif
5056 :
5057 5519 : return (ret);
5058 : }
5059 :
5060 : /* Only handles large allocations that require more than chunk alignment. */
5061 : static void *
5062 0 : huge_palloc(size_t alignment, size_t size)
5063 : {
5064 : void *ret;
5065 : size_t alloc_size, chunk_size, offset;
5066 : size_t psize;
5067 : extent_node_t *node;
5068 : int pfd;
5069 :
5070 : /*
5071 : * This allocation requires alignment that is even larger than chunk
5072 : * alignment. This means that huge_malloc() isn't good enough.
5073 : *
5074 : * Allocate almost twice as many chunks as are demanded by the size or
5075 : * alignment, in order to assure the alignment can be achieved, then
5076 : * unmap leading and trailing chunks.
5077 : */
5078 0 : assert(alignment >= chunksize);
5079 :
5080 0 : chunk_size = CHUNK_CEILING(size);
5081 :
5082 0 : if (size >= alignment)
5083 0 : alloc_size = chunk_size + alignment - chunksize;
5084 : else
5085 0 : alloc_size = (alignment << 1) - chunksize;
5086 :
5087 : /* Allocate an extent node with which to track the chunk. */
5088 0 : node = base_node_alloc();
5089 0 : if (node == NULL)
5090 0 : return (NULL);
5091 :
5092 : /*
5093 : * Windows requires that there be a 1:1 mapping between VM
5094 : * allocation/deallocation operations. Therefore, take care here to
5095 : * acquire the final result via one mapping operation.
5096 : *
5097 : * The MALLOC_PAGEFILE code also benefits from this mapping algorithm,
5098 : * since it reduces the number of page files.
5099 : */
5100 : #ifdef MALLOC_PAGEFILE
5101 : if (opt_pagefile) {
5102 : pfd = pagefile_init(size);
5103 : if (pfd == -1)
5104 : return (NULL);
5105 : } else
5106 : #endif
5107 0 : pfd = -1;
5108 : #ifdef JEMALLOC_USES_MAP_ALIGN
5109 : ret = pages_map_align(chunk_size, pfd, alignment);
5110 : #else
5111 : do {
5112 : void *over;
5113 :
5114 0 : over = chunk_alloc(alloc_size, false, false);
5115 0 : if (over == NULL) {
5116 0 : base_node_dealloc(node);
5117 0 : ret = NULL;
5118 0 : goto RETURN;
5119 : }
5120 :
5121 0 : offset = (uintptr_t)over & (alignment - 1);
5122 0 : assert((offset & chunksize_mask) == 0);
5123 0 : assert(offset < alloc_size);
5124 0 : ret = (void *)((uintptr_t)over + offset);
5125 0 : chunk_dealloc(over, alloc_size);
5126 0 : ret = pages_map(ret, chunk_size, pfd);
5127 : /*
5128 : * Failure here indicates a race with another thread, so try
5129 : * again.
5130 : */
5131 0 : } while (ret == NULL);
5132 : #endif
5133 : /* Insert node into huge. */
5134 0 : node->addr = ret;
5135 0 : psize = PAGE_CEILING(size);
5136 0 : node->size = psize;
5137 :
5138 0 : malloc_mutex_lock(&huge_mtx);
5139 0 : extent_tree_ad_insert(&huge, node);
5140 : #ifdef MALLOC_STATS
5141 0 : huge_nmalloc++;
5142 : /* See note in huge_alloc() for why huge_allocated += psize is correct
5143 : * here even when DECOMMIT is not defined. */
5144 0 : huge_allocated += psize;
5145 : #endif
5146 0 : malloc_mutex_unlock(&huge_mtx);
5147 :
5148 : #ifdef MALLOC_DECOMMIT
5149 : if (chunk_size - psize > 0) {
5150 : pages_decommit((void *)((uintptr_t)ret + psize),
5151 : chunk_size - psize);
5152 : }
5153 : #endif
5154 :
5155 : #ifdef MALLOC_DECOMMIT
5156 : VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, false);
5157 : #else
5158 : VALGRIND_MALLOCLIKE_BLOCK(ret, chunk_size, 0, false);
5159 : #endif
5160 :
5161 : #ifdef MALLOC_FILL
5162 0 : if (opt_junk)
5163 : # ifdef MALLOC_DECOMMIT
5164 : memset(ret, 0xa5, psize);
5165 : # else
5166 0 : memset(ret, 0xa5, chunk_size);
5167 : # endif
5168 0 : else if (opt_zero)
5169 : # ifdef MALLOC_DECOMMIT
5170 : memset(ret, 0, psize);
5171 : # else
5172 0 : memset(ret, 0, chunk_size);
5173 : # endif
5174 : #endif
5175 :
5176 : RETURN:
5177 : #ifdef MALLOC_PAGEFILE
5178 : if (pfd != -1)
5179 : pagefile_close(pfd);
5180 : #endif
5181 0 : return (ret);
5182 : }
5183 :
5184 : static void *
5185 2545 : huge_ralloc(void *ptr, size_t size, size_t oldsize)
5186 : {
5187 : void *ret;
5188 : size_t copysize;
5189 :
5190 : /* Avoid moving the allocation if the size class would not change. */
5191 :
5192 4871 : if (oldsize > arena_maxclass &&
5193 2326 : CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
5194 720 : size_t psize = PAGE_CEILING(size);
5195 : #ifdef MALLOC_FILL
5196 720 : if (opt_junk && size < oldsize) {
5197 0 : memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
5198 : - size);
5199 : }
5200 : #endif
5201 : #ifdef MALLOC_DECOMMIT
5202 : if (psize < oldsize) {
5203 : extent_node_t *node, key;
5204 :
5205 : pages_decommit((void *)((uintptr_t)ptr + psize),
5206 : oldsize - psize);
5207 :
5208 : /* Update recorded size. */
5209 : malloc_mutex_lock(&huge_mtx);
5210 : key.addr = __DECONST(void *, ptr);
5211 : node = extent_tree_ad_search(&huge, &key);
5212 : assert(node != NULL);
5213 : assert(node->size == oldsize);
5214 : # ifdef MALLOC_STATS
5215 : huge_allocated -= oldsize - psize;
5216 : # endif
5217 : node->size = psize;
5218 : malloc_mutex_unlock(&huge_mtx);
5219 : } else if (psize > oldsize) {
5220 : pages_commit((void *)((uintptr_t)ptr + oldsize),
5221 : psize - oldsize);
5222 : }
5223 : #endif
5224 :
5225 : /* Although we don't have to commit or decommit anything if
5226 : * DECOMMIT is not defined and the size class didn't change, we
5227 : * do need to update the recorded size if the size increased,
5228 : * so malloc_usable_size doesn't return a value smaller than
5229 : * what was requested via realloc(). */
5230 :
5231 720 : if (psize > oldsize) {
5232 : /* Update recorded size. */
5233 : extent_node_t *node, key;
5234 720 : malloc_mutex_lock(&huge_mtx);
5235 720 : key.addr = __DECONST(void *, ptr);
5236 720 : node = extent_tree_ad_search(&huge, &key);
5237 720 : assert(node != NULL);
5238 720 : assert(node->size == oldsize);
5239 : # ifdef MALLOC_STATS
5240 720 : huge_allocated += psize - oldsize;
5241 : # endif
5242 720 : node->size = psize;
5243 720 : malloc_mutex_unlock(&huge_mtx);
5244 : }
5245 :
5246 : #ifdef MALLOC_FILL
5247 720 : if (opt_zero && size > oldsize) {
5248 0 : memset((void *)((uintptr_t)ptr + oldsize), 0, size
5249 : - oldsize);
5250 : }
5251 : #endif
5252 720 : return (ptr);
5253 : }
5254 :
5255 : /*
5256 : * If we get here, then size and oldsize are different enough that we
5257 : * need to use a different size class. In that case, fall back to
5258 : * allocating new space and copying.
5259 : */
5260 1825 : ret = huge_malloc(size, false);
5261 1825 : if (ret == NULL)
5262 0 : return (NULL);
5263 :
5264 1825 : copysize = (size < oldsize) ? size : oldsize;
5265 : #ifdef VM_COPY_MIN
5266 : if (copysize >= VM_COPY_MIN)
5267 : pages_copy(ret, ptr, copysize);
5268 : else
5269 : #endif
5270 1825 : memcpy(ret, ptr, copysize);
5271 1825 : idalloc(ptr);
5272 1825 : return (ret);
5273 : }
5274 :
5275 : static void
5276 5519 : huge_dalloc(void *ptr)
5277 : {
5278 : extent_node_t *node, key;
5279 :
5280 5519 : malloc_mutex_lock(&huge_mtx);
5281 :
5282 : /* Extract from tree of huge allocations. */
5283 5519 : key.addr = ptr;
5284 5519 : node = extent_tree_ad_search(&huge, &key);
5285 5519 : assert(node != NULL);
5286 5519 : assert(node->addr == ptr);
5287 5519 : extent_tree_ad_remove(&huge, node);
5288 :
5289 : #ifdef MALLOC_STATS
5290 5519 : huge_ndalloc++;
5291 5519 : huge_allocated -= node->size;
5292 : #endif
5293 :
5294 5519 : malloc_mutex_unlock(&huge_mtx);
5295 :
5296 : /* Unmap chunk. */
5297 : #ifdef MALLOC_FILL
5298 5519 : if (opt_junk)
5299 5519 : memset(node->addr, 0x5a, node->size);
5300 : #endif
5301 5519 : chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
5302 : VALGRIND_FREELIKE_BLOCK(node->addr, 0);
5303 :
5304 5519 : base_node_dealloc(node);
5305 5519 : }
5306 :
5307 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5308 : #ifdef MOZ_MEMORY_BSD
5309 : static inline unsigned
5310 : malloc_ncpus(void)
5311 : {
5312 : unsigned ret;
5313 : int mib[2];
5314 : size_t len;
5315 :
5316 : mib[0] = CTL_HW;
5317 : mib[1] = HW_NCPU;
5318 : len = sizeof(ret);
5319 : if (sysctl(mib, 2, &ret, &len, (void *) 0, 0) == -1) {
5320 : /* Error. */
5321 : return (1);
5322 : }
5323 :
5324 : return (ret);
5325 : }
5326 : #elif (defined(MOZ_MEMORY_LINUX))
5327 : #include <fcntl.h>
5328 :
5329 : static inline unsigned
5330 : malloc_ncpus(void)
5331 : {
5332 : unsigned ret;
5333 : int fd, nread, column;
5334 : char buf[1024];
5335 : static const char matchstr[] = "processor\t:";
5336 : int i;
5337 :
5338 : /*
5339 : * sysconf(3) would be the preferred method for determining the number
5340 : * of CPUs, but it uses malloc internally, which causes untennable
5341 : * recursion during malloc initialization.
5342 : */
5343 : fd = open("/proc/cpuinfo", O_RDONLY);
5344 : if (fd == -1)
5345 : return (1); /* Error. */
5346 : /*
5347 : * Count the number of occurrences of matchstr at the beginnings of
5348 : * lines. This treats hyperthreaded CPUs as multiple processors.
5349 : */
5350 : column = 0;
5351 : ret = 0;
5352 : while (true) {
5353 : nread = read(fd, &buf, sizeof(buf));
5354 : if (nread <= 0)
5355 : break; /* EOF or error. */
5356 : for (i = 0;i < nread;i++) {
5357 : char c = buf[i];
5358 : if (c == '\n')
5359 : column = 0;
5360 : else if (column != -1) {
5361 : if (c == matchstr[column]) {
5362 : column++;
5363 : if (column == sizeof(matchstr) - 1) {
5364 : column = -1;
5365 : ret++;
5366 : }
5367 : } else
5368 : column = -1;
5369 : }
5370 : }
5371 : }
5372 :
5373 : if (ret == 0)
5374 : ret = 1; /* Something went wrong in the parser. */
5375 : close(fd);
5376 :
5377 : return (ret);
5378 : }
5379 : #elif (defined(MOZ_MEMORY_DARWIN))
5380 : #include <mach/mach_init.h>
5381 : #include <mach/mach_host.h>
5382 :
5383 : static inline unsigned
5384 : malloc_ncpus(void)
5385 : {
5386 : kern_return_t error;
5387 : natural_t n;
5388 : processor_info_array_t pinfo;
5389 : mach_msg_type_number_t pinfocnt;
5390 :
5391 : error = host_processor_info(mach_host_self(), PROCESSOR_BASIC_INFO,
5392 : &n, &pinfo, &pinfocnt);
5393 : if (error != KERN_SUCCESS)
5394 : return (1); /* Error. */
5395 : else
5396 : return (n);
5397 : }
5398 : #elif (defined(MOZ_MEMORY_SOLARIS))
5399 :
5400 : static inline unsigned
5401 : malloc_ncpus(void)
5402 : {
5403 : return sysconf(_SC_NPROCESSORS_ONLN);
5404 : }
5405 : #else
5406 : static inline unsigned
5407 : malloc_ncpus(void)
5408 : {
5409 :
5410 : /*
5411 : * We lack a way to determine the number of CPUs on this platform, so
5412 : * assume 1 CPU.
5413 : */
5414 : return (1);
5415 : }
5416 : #endif
5417 : #endif
5418 :
5419 : static void
5420 0 : malloc_print_stats(void)
5421 : {
5422 :
5423 0 : if (opt_print_stats) {
5424 : char s[UMAX2S_BUFSIZE];
5425 0 : _malloc_message("___ Begin malloc statistics ___\n", "", "",
5426 : "");
5427 0 : _malloc_message("Assertions ",
5428 : #ifdef NDEBUG
5429 : "disabled",
5430 : #else
5431 : "enabled",
5432 : #endif
5433 : "\n", "");
5434 0 : _malloc_message("Boolean MALLOC_OPTIONS: ",
5435 : opt_abort ? "A" : "a", "", "");
5436 : #ifdef MALLOC_FILL
5437 0 : _malloc_message(opt_junk ? "J" : "j", "", "", "");
5438 : #endif
5439 : #ifdef MALLOC_PAGEFILE
5440 : _malloc_message(opt_pagefile ? "o" : "O", "", "", "");
5441 : #endif
5442 0 : _malloc_message("P", "", "", "");
5443 : #ifdef MALLOC_UTRACE
5444 0 : _malloc_message(opt_utrace ? "U" : "u", "", "", "");
5445 : #endif
5446 : #ifdef MALLOC_SYSV
5447 0 : _malloc_message(opt_sysv ? "V" : "v", "", "", "");
5448 : #endif
5449 : #ifdef MALLOC_XMALLOC
5450 0 : _malloc_message(opt_xmalloc ? "X" : "x", "", "", "");
5451 : #endif
5452 : #ifdef MALLOC_FILL
5453 0 : _malloc_message(opt_zero ? "Z" : "z", "", "", "");
5454 : #endif
5455 0 : _malloc_message("\n", "", "", "");
5456 :
5457 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5458 : _malloc_message("CPUs: ", umax2s(ncpus, 10, s), "\n", "");
5459 : #endif
5460 0 : _malloc_message("Max arenas: ", umax2s(narenas, 10, s), "\n",
5461 : "");
5462 : #ifdef MALLOC_BALANCE
5463 : _malloc_message("Arena balance threshold: ",
5464 : umax2s(opt_balance_threshold, 10, s), "\n", "");
5465 : #endif
5466 0 : _malloc_message("Pointer size: ", umax2s(sizeof(void *), 10, s),
5467 : "\n", "");
5468 0 : _malloc_message("Quantum size: ", umax2s(quantum, 10, s), "\n",
5469 : "");
5470 0 : _malloc_message("Max small size: ", umax2s(small_max, 10, s),
5471 : "\n", "");
5472 0 : _malloc_message("Max dirty pages per arena: ",
5473 0 : umax2s(opt_dirty_max, 10, s), "\n", "");
5474 :
5475 0 : _malloc_message("Chunk size: ", umax2s(chunksize, 10, s), "",
5476 : "");
5477 0 : _malloc_message(" (2^", umax2s(opt_chunk_2pow, 10, s), ")\n",
5478 : "");
5479 :
5480 : #ifdef MALLOC_STATS
5481 : {
5482 : size_t allocated, mapped;
5483 : #ifdef MALLOC_BALANCE
5484 : uint64_t nbalance = 0;
5485 : #endif
5486 : unsigned i;
5487 : arena_t *arena;
5488 :
5489 : /* Calculate and print allocated/mapped stats. */
5490 :
5491 : /* arenas. */
5492 0 : for (i = 0, allocated = 0; i < narenas; i++) {
5493 0 : if (arenas[i] != NULL) {
5494 0 : malloc_spin_lock(&arenas[i]->lock);
5495 0 : allocated +=
5496 0 : arenas[i]->stats.allocated_small;
5497 0 : allocated +=
5498 0 : arenas[i]->stats.allocated_large;
5499 : #ifdef MALLOC_BALANCE
5500 : nbalance += arenas[i]->stats.nbalance;
5501 : #endif
5502 0 : malloc_spin_unlock(&arenas[i]->lock);
5503 : }
5504 : }
5505 :
5506 : /* huge/base. */
5507 0 : malloc_mutex_lock(&huge_mtx);
5508 0 : allocated += huge_allocated;
5509 0 : mapped = stats_chunks.curchunks * chunksize;
5510 0 : malloc_mutex_unlock(&huge_mtx);
5511 :
5512 0 : malloc_mutex_lock(&base_mtx);
5513 0 : mapped += base_mapped;
5514 0 : malloc_mutex_unlock(&base_mtx);
5515 :
5516 : #ifdef MOZ_MEMORY_WINDOWS
5517 : malloc_printf("Allocated: %lu, mapped: %lu\n",
5518 : allocated, mapped);
5519 : #else
5520 0 : malloc_printf("Allocated: %zu, mapped: %zu\n",
5521 : allocated, mapped);
5522 : #endif
5523 :
5524 : #ifdef MALLOC_BALANCE
5525 : malloc_printf("Arena balance reassignments: %llu\n",
5526 : nbalance);
5527 : #endif
5528 :
5529 : /* Print chunk stats. */
5530 : {
5531 : chunk_stats_t chunks_stats;
5532 :
5533 0 : malloc_mutex_lock(&huge_mtx);
5534 0 : chunks_stats = stats_chunks;
5535 0 : malloc_mutex_unlock(&huge_mtx);
5536 :
5537 0 : malloc_printf("chunks: nchunks "
5538 : "highchunks curchunks\n");
5539 0 : malloc_printf(" %13llu%13lu%13lu\n",
5540 : chunks_stats.nchunks,
5541 : chunks_stats.highchunks,
5542 : chunks_stats.curchunks);
5543 : }
5544 :
5545 : /* Print chunk stats. */
5546 0 : malloc_printf(
5547 : "huge: nmalloc ndalloc allocated\n");
5548 : #ifdef MOZ_MEMORY_WINDOWS
5549 : malloc_printf(" %12llu %12llu %12lu\n",
5550 : huge_nmalloc, huge_ndalloc, huge_allocated);
5551 : #else
5552 0 : malloc_printf(" %12llu %12llu %12zu\n",
5553 : huge_nmalloc, huge_ndalloc, huge_allocated);
5554 : #endif
5555 : /* Print stats for each arena. */
5556 0 : for (i = 0; i < narenas; i++) {
5557 0 : arena = arenas[i];
5558 0 : if (arena != NULL) {
5559 0 : malloc_printf(
5560 : "\narenas[%u]:\n", i);
5561 0 : malloc_spin_lock(&arena->lock);
5562 0 : stats_print(arena);
5563 0 : malloc_spin_unlock(&arena->lock);
5564 : }
5565 : }
5566 : }
5567 : #endif /* #ifdef MALLOC_STATS */
5568 0 : _malloc_message("--- End malloc statistics ---\n", "", "", "");
5569 : }
5570 0 : }
5571 :
5572 : /*
5573 : * FreeBSD's pthreads implementation calls malloc(3), so the malloc
5574 : * implementation has to take pains to avoid infinite recursion during
5575 : * initialization.
5576 : */
5577 : #if (defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_DARWIN))
5578 : #define malloc_init() false
5579 : #else
5580 : static inline bool
5581 121360462 : malloc_init(void)
5582 : {
5583 :
5584 121360462 : if (malloc_initialized == false)
5585 19901 : return (malloc_init_hard());
5586 :
5587 121340561 : return (false);
5588 : }
5589 : #endif
5590 :
5591 : #if !defined(MOZ_MEMORY_WINDOWS)
5592 : static
5593 : #endif
5594 : bool
5595 19901 : malloc_init_hard(void)
5596 : {
5597 : unsigned i;
5598 : char buf[PATH_MAX + 1];
5599 : const char *opts;
5600 : long result;
5601 : #ifndef MOZ_MEMORY_WINDOWS
5602 : int linklen;
5603 : #endif
5604 : #ifdef MOZ_MEMORY_DARWIN
5605 : malloc_zone_t* default_zone;
5606 : #endif
5607 :
5608 : #ifndef MOZ_MEMORY_WINDOWS
5609 19901 : malloc_mutex_lock(&init_lock);
5610 : #endif
5611 :
5612 19901 : if (malloc_initialized) {
5613 : /*
5614 : * Another thread initialized the allocator before this one
5615 : * acquired init_lock.
5616 : */
5617 : #ifndef MOZ_MEMORY_WINDOWS
5618 0 : malloc_mutex_unlock(&init_lock);
5619 : #endif
5620 0 : return (false);
5621 : }
5622 :
5623 : #ifdef MOZ_MEMORY_WINDOWS
5624 : /* get a thread local storage index */
5625 : tlsIndex = TlsAlloc();
5626 : #endif
5627 :
5628 : /* Get page size and number of CPUs */
5629 : #ifdef MOZ_MEMORY_WINDOWS
5630 : {
5631 : SYSTEM_INFO info;
5632 :
5633 : GetSystemInfo(&info);
5634 : result = info.dwPageSize;
5635 :
5636 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5637 : ncpus = info.dwNumberOfProcessors;
5638 : #endif
5639 : }
5640 : #else
5641 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
5642 : ncpus = malloc_ncpus();
5643 : #endif
5644 :
5645 19901 : result = sysconf(_SC_PAGESIZE);
5646 19901 : assert(result != -1);
5647 : #endif
5648 :
5649 : /* We assume that the page size is a power of 2. */
5650 19901 : assert(((result - 1) & result) == 0);
5651 : #ifdef MALLOC_STATIC_SIZES
5652 19901 : if (pagesize % (size_t) result) {
5653 0 : _malloc_message(_getprogname(),
5654 : "Compile-time page size does not divide the runtime one.\n",
5655 : "", "");
5656 0 : abort();
5657 : }
5658 : #else
5659 : pagesize = (size_t) result;
5660 : pagesize_mask = (size_t) result - 1;
5661 : pagesize_2pow = ffs((int)result) - 1;
5662 : #endif
5663 :
5664 : #ifdef MALLOC_PAGEFILE
5665 : /*
5666 : * Determine where to create page files. It is insufficient to
5667 : * unconditionally use P_tmpdir (typically "/tmp"), since for some
5668 : * operating systems /tmp is a separate filesystem that is rather small.
5669 : * Therefore prefer, in order, the following locations:
5670 : *
5671 : * 1) MALLOC_TMPDIR
5672 : * 2) TMPDIR
5673 : * 3) P_tmpdir
5674 : */
5675 : {
5676 : char *s;
5677 : size_t slen;
5678 : static const char suffix[] = "/jemalloc.XXXXXX";
5679 :
5680 : if ((s = getenv("MALLOC_TMPDIR")) == NULL && (s =
5681 : getenv("TMPDIR")) == NULL)
5682 : s = P_tmpdir;
5683 : slen = strlen(s);
5684 : if (slen + sizeof(suffix) > sizeof(pagefile_templ)) {
5685 : _malloc_message(_getprogname(),
5686 : ": (malloc) Page file path too long\n",
5687 : "", "");
5688 : abort();
5689 : }
5690 : memcpy(pagefile_templ, s, slen);
5691 : memcpy(&pagefile_templ[slen], suffix, sizeof(suffix));
5692 : }
5693 : #endif
5694 :
5695 79604 : for (i = 0; i < 3; i++) {
5696 : unsigned j;
5697 :
5698 : /* Get runtime configuration. */
5699 59703 : switch (i) {
5700 : case 0:
5701 : #ifndef MOZ_MEMORY_WINDOWS
5702 19901 : if ((linklen = readlink("/etc/malloc.conf", buf,
5703 : sizeof(buf) - 1)) != -1) {
5704 : /*
5705 : * Use the contents of the "/etc/malloc.conf"
5706 : * symbolic link's name.
5707 : */
5708 0 : buf[linklen] = '\0';
5709 0 : opts = buf;
5710 : } else
5711 : #endif
5712 : {
5713 : /* No configuration specified. */
5714 19901 : buf[0] = '\0';
5715 19901 : opts = buf;
5716 : }
5717 19901 : break;
5718 : case 1:
5719 19901 : if (issetugid() == 0 && (opts =
5720 : getenv("MALLOC_OPTIONS")) != NULL) {
5721 : /*
5722 : * Do nothing; opts is already initialized to
5723 : * the value of the MALLOC_OPTIONS environment
5724 : * variable.
5725 : */
5726 : } else {
5727 : /* No configuration specified. */
5728 19901 : buf[0] = '\0';
5729 19901 : opts = buf;
5730 : }
5731 19901 : break;
5732 : case 2:
5733 19901 : if (_malloc_options != NULL) {
5734 : /*
5735 : * Use options that were compiled into the
5736 : * program.
5737 : */
5738 0 : opts = _malloc_options;
5739 : } else {
5740 : /* No configuration specified. */
5741 19901 : buf[0] = '\0';
5742 19901 : opts = buf;
5743 : }
5744 19901 : break;
5745 : default:
5746 : /* NOTREACHED */
5747 0 : buf[0] = '\0';
5748 0 : opts = buf;
5749 0 : assert(false);
5750 : }
5751 :
5752 59703 : for (j = 0; opts[j] != '\0'; j++) {
5753 : unsigned k, nreps;
5754 : bool nseen;
5755 :
5756 : /* Parse repetition count, if any. */
5757 0 : for (nreps = 0, nseen = false;; j++, nseen = true) {
5758 0 : switch (opts[j]) {
5759 : case '0': case '1': case '2': case '3':
5760 : case '4': case '5': case '6': case '7':
5761 : case '8': case '9':
5762 0 : nreps *= 10;
5763 0 : nreps += opts[j] - '0';
5764 : break;
5765 : default:
5766 0 : goto MALLOC_OUT;
5767 : }
5768 0 : }
5769 : MALLOC_OUT:
5770 0 : if (nseen == false)
5771 0 : nreps = 1;
5772 :
5773 0 : for (k = 0; k < nreps; k++) {
5774 0 : switch (opts[j]) {
5775 : case 'a':
5776 0 : opt_abort = false;
5777 0 : break;
5778 : case 'A':
5779 0 : opt_abort = true;
5780 0 : break;
5781 : case 'b':
5782 : #ifdef MALLOC_BALANCE
5783 : opt_balance_threshold >>= 1;
5784 : #endif
5785 0 : break;
5786 : case 'B':
5787 : #ifdef MALLOC_BALANCE
5788 : if (opt_balance_threshold == 0)
5789 : opt_balance_threshold = 1;
5790 : else if ((opt_balance_threshold << 1)
5791 : > opt_balance_threshold)
5792 : opt_balance_threshold <<= 1;
5793 : #endif
5794 0 : break;
5795 : case 'f':
5796 0 : opt_dirty_max >>= 1;
5797 0 : break;
5798 : case 'F':
5799 0 : if (opt_dirty_max == 0)
5800 0 : opt_dirty_max = 1;
5801 0 : else if ((opt_dirty_max << 1) != 0)
5802 0 : opt_dirty_max <<= 1;
5803 0 : break;
5804 : #ifdef MALLOC_FILL
5805 : case 'j':
5806 0 : opt_junk = false;
5807 0 : break;
5808 : case 'J':
5809 0 : opt_junk = true;
5810 0 : break;
5811 : #endif
5812 : #ifndef MALLOC_STATIC_SIZES
5813 : case 'k':
5814 : /*
5815 : * Chunks always require at least one
5816 : * header page, so chunks can never be
5817 : * smaller than two pages.
5818 : */
5819 : if (opt_chunk_2pow > pagesize_2pow + 1)
5820 : opt_chunk_2pow--;
5821 : break;
5822 : case 'K':
5823 : if (opt_chunk_2pow + 1 <
5824 : (sizeof(size_t) << 3))
5825 : opt_chunk_2pow++;
5826 : break;
5827 : #endif
5828 : case 'n':
5829 0 : opt_narenas_lshift--;
5830 0 : break;
5831 : case 'N':
5832 0 : opt_narenas_lshift++;
5833 0 : break;
5834 : #ifdef MALLOC_PAGEFILE
5835 : case 'o':
5836 : /* Do not over-commit. */
5837 : opt_pagefile = true;
5838 : break;
5839 : case 'O':
5840 : /* Allow over-commit. */
5841 : opt_pagefile = false;
5842 : break;
5843 : #endif
5844 : case 'p':
5845 0 : opt_print_stats = false;
5846 0 : break;
5847 : case 'P':
5848 0 : opt_print_stats = true;
5849 0 : break;
5850 : #ifndef MALLOC_STATIC_SIZES
5851 : case 'q':
5852 : if (opt_quantum_2pow > QUANTUM_2POW_MIN)
5853 : opt_quantum_2pow--;
5854 : break;
5855 : case 'Q':
5856 : if (opt_quantum_2pow < pagesize_2pow -
5857 : 1)
5858 : opt_quantum_2pow++;
5859 : break;
5860 : case 's':
5861 : if (opt_small_max_2pow >
5862 : QUANTUM_2POW_MIN)
5863 : opt_small_max_2pow--;
5864 : break;
5865 : case 'S':
5866 : if (opt_small_max_2pow < pagesize_2pow
5867 : - 1)
5868 : opt_small_max_2pow++;
5869 : break;
5870 : #endif
5871 : #ifdef MALLOC_UTRACE
5872 : case 'u':
5873 0 : opt_utrace = false;
5874 0 : break;
5875 : case 'U':
5876 0 : opt_utrace = true;
5877 0 : break;
5878 : #endif
5879 : #ifdef MALLOC_SYSV
5880 : case 'v':
5881 0 : opt_sysv = false;
5882 0 : break;
5883 : case 'V':
5884 0 : opt_sysv = true;
5885 0 : break;
5886 : #endif
5887 : #ifdef MALLOC_XMALLOC
5888 : case 'x':
5889 0 : opt_xmalloc = false;
5890 0 : break;
5891 : case 'X':
5892 0 : opt_xmalloc = true;
5893 0 : break;
5894 : #endif
5895 : #ifdef MALLOC_FILL
5896 : case 'z':
5897 0 : opt_zero = false;
5898 0 : break;
5899 : case 'Z':
5900 0 : opt_zero = true;
5901 0 : break;
5902 : #endif
5903 : default: {
5904 : char cbuf[2];
5905 :
5906 0 : cbuf[0] = opts[j];
5907 0 : cbuf[1] = '\0';
5908 0 : _malloc_message(_getprogname(),
5909 : ": (malloc) Unsupported character "
5910 : "in malloc options: '", cbuf,
5911 : "'\n");
5912 : }
5913 : }
5914 : }
5915 : }
5916 : }
5917 :
5918 : /* Take care to call atexit() only once. */
5919 19901 : if (opt_print_stats) {
5920 : #ifndef MOZ_MEMORY_WINDOWS
5921 : /* Print statistics at exit. */
5922 0 : atexit(malloc_print_stats);
5923 : #endif
5924 : }
5925 :
5926 : #if (!defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN) && !defined(MOZ_MEMORY_ANDROID))
5927 : /* Prevent potential deadlock on malloc locks after fork. */
5928 : /* XXX on Android there is no pthread_atfork, so we specifically
5929 : call _malloc_prefork and _malloc_postfork in process_util_linux.cc */
5930 19901 : pthread_atfork(_malloc_prefork, _malloc_postfork, _malloc_postfork);
5931 : #endif
5932 :
5933 : #ifndef MALLOC_STATIC_SIZES
5934 : /* Set variables according to the value of opt_small_max_2pow. */
5935 : if (opt_small_max_2pow < opt_quantum_2pow)
5936 : opt_small_max_2pow = opt_quantum_2pow;
5937 : small_max = (1U << opt_small_max_2pow);
5938 :
5939 : /* Set bin-related variables. */
5940 : bin_maxclass = (pagesize >> 1);
5941 : assert(opt_quantum_2pow >= TINY_MIN_2POW);
5942 : ntbins = opt_quantum_2pow - TINY_MIN_2POW;
5943 : assert(ntbins <= opt_quantum_2pow);
5944 : nqbins = (small_max >> opt_quantum_2pow);
5945 : nsbins = pagesize_2pow - opt_small_max_2pow - 1;
5946 :
5947 : /* Set variables according to the value of opt_quantum_2pow. */
5948 : quantum = (1U << opt_quantum_2pow);
5949 : quantum_mask = quantum - 1;
5950 : if (ntbins > 0)
5951 : small_min = (quantum >> 1) + 1;
5952 : else
5953 : small_min = 1;
5954 : assert(small_min <= quantum);
5955 :
5956 : /* Set variables according to the value of opt_chunk_2pow. */
5957 : chunksize = (1LU << opt_chunk_2pow);
5958 : chunksize_mask = chunksize - 1;
5959 : chunk_npages = (chunksize >> pagesize_2pow);
5960 :
5961 : arena_chunk_header_npages = calculate_arena_header_pages();
5962 : arena_maxclass = calculate_arena_maxclass();
5963 : #endif
5964 :
5965 : #ifdef JEMALLOC_USES_MAP_ALIGN
5966 : /*
5967 : * When using MAP_ALIGN, the alignment parameter must be a power of two
5968 : * multiple of the system pagesize, or mmap will fail.
5969 : */
5970 : assert((chunksize % pagesize) == 0);
5971 : assert((1 << (ffs(chunksize / pagesize) - 1)) == (chunksize/pagesize));
5972 : #endif
5973 :
5974 19901 : UTRACE(0, 0, 0);
5975 :
5976 : #ifdef MALLOC_STATS
5977 19901 : memset(&stats_chunks, 0, sizeof(chunk_stats_t));
5978 : #endif
5979 :
5980 : /* Various sanity checks that regard configuration. */
5981 19901 : assert(quantum >= sizeof(void *));
5982 19901 : assert(quantum <= pagesize);
5983 19901 : assert(chunksize >= pagesize);
5984 19901 : assert(quantum * 4 <= chunksize);
5985 :
5986 : /* Initialize chunks data. */
5987 19901 : malloc_mutex_init(&huge_mtx);
5988 19901 : extent_tree_ad_new(&huge);
5989 : #ifdef MALLOC_STATS
5990 19901 : huge_nmalloc = 0;
5991 19901 : huge_ndalloc = 0;
5992 19901 : huge_allocated = 0;
5993 : #endif
5994 :
5995 : /* Initialize base allocation data structures. */
5996 : #ifdef MALLOC_STATS
5997 19901 : base_mapped = 0;
5998 19901 : base_committed = 0;
5999 : #endif
6000 19901 : base_nodes = NULL;
6001 19901 : malloc_mutex_init(&base_mtx);
6002 :
6003 : #ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE
6004 19901 : narenas = 1;
6005 : #else
6006 : if (ncpus > 1) {
6007 : /*
6008 : * For SMP systems, create four times as many arenas as there
6009 : * are CPUs by default.
6010 : */
6011 : opt_narenas_lshift += 2;
6012 : }
6013 :
6014 : /* Determine how many arenas to use. */
6015 : narenas = ncpus;
6016 : #endif
6017 19901 : if (opt_narenas_lshift > 0) {
6018 0 : if ((narenas << opt_narenas_lshift) > narenas)
6019 0 : narenas <<= opt_narenas_lshift;
6020 : /*
6021 : * Make sure not to exceed the limits of what base_alloc() can
6022 : * handle.
6023 : */
6024 0 : if (narenas * sizeof(arena_t *) > chunksize)
6025 0 : narenas = chunksize / sizeof(arena_t *);
6026 19901 : } else if (opt_narenas_lshift < 0) {
6027 0 : if ((narenas >> -opt_narenas_lshift) < narenas)
6028 0 : narenas >>= -opt_narenas_lshift;
6029 : /* Make sure there is at least one arena. */
6030 0 : if (narenas == 0)
6031 0 : narenas = 1;
6032 : }
6033 : #ifdef MALLOC_BALANCE
6034 : assert(narenas != 0);
6035 : for (narenas_2pow = 0;
6036 : (narenas >> (narenas_2pow + 1)) != 0;
6037 : narenas_2pow++);
6038 : #endif
6039 :
6040 : #ifdef NO_TLS
6041 : if (narenas > 1) {
6042 : static const unsigned primes[] = {1, 3, 5, 7, 11, 13, 17, 19,
6043 : 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
6044 : 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149,
6045 : 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
6046 : 223, 227, 229, 233, 239, 241, 251, 257, 263};
6047 : unsigned nprimes, parenas;
6048 :
6049 : /*
6050 : * Pick a prime number of hash arenas that is more than narenas
6051 : * so that direct hashing of pthread_self() pointers tends to
6052 : * spread allocations evenly among the arenas.
6053 : */
6054 : assert((narenas & 1) == 0); /* narenas must be even. */
6055 : nprimes = (sizeof(primes) >> SIZEOF_INT_2POW);
6056 : parenas = primes[nprimes - 1]; /* In case not enough primes. */
6057 : for (i = 1; i < nprimes; i++) {
6058 : if (primes[i] > narenas) {
6059 : parenas = primes[i];
6060 : break;
6061 : }
6062 : }
6063 : narenas = parenas;
6064 : }
6065 : #endif
6066 :
6067 : #ifndef NO_TLS
6068 : # ifndef MALLOC_BALANCE
6069 19901 : next_arena = 0;
6070 : # endif
6071 : #endif
6072 :
6073 : /* Allocate and initialize arenas. */
6074 19901 : arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
6075 19901 : if (arenas == NULL) {
6076 : #ifndef MOZ_MEMORY_WINDOWS
6077 0 : malloc_mutex_unlock(&init_lock);
6078 : #endif
6079 0 : return (true);
6080 : }
6081 : /*
6082 : * Zero the array. In practice, this should always be pre-zeroed,
6083 : * since it was just mmap()ed, but let's be sure.
6084 : */
6085 19901 : memset(arenas, 0, sizeof(arena_t *) * narenas);
6086 :
6087 : /*
6088 : * Initialize one arena here. The rest are lazily created in
6089 : * choose_arena_hard().
6090 : */
6091 19901 : arenas_extend(0);
6092 19901 : if (arenas[0] == NULL) {
6093 : #ifndef MOZ_MEMORY_WINDOWS
6094 0 : malloc_mutex_unlock(&init_lock);
6095 : #endif
6096 0 : return (true);
6097 : }
6098 : #ifndef NO_TLS
6099 : /*
6100 : * Assign the initial arena to the initial thread, in order to avoid
6101 : * spurious creation of an extra arena if the application switches to
6102 : * threaded mode.
6103 : */
6104 : #ifdef MOZ_MEMORY_WINDOWS
6105 : TlsSetValue(tlsIndex, arenas[0]);
6106 : #else
6107 19901 : arenas_map = arenas[0];
6108 : #endif
6109 : #endif
6110 :
6111 : /*
6112 : * Seed here for the initial thread, since choose_arena_hard() is only
6113 : * called for other threads. The seed value doesn't really matter.
6114 : */
6115 : #ifdef MALLOC_BALANCE
6116 : SPRN(balance, 42);
6117 : #endif
6118 :
6119 19901 : malloc_spin_init(&arenas_lock);
6120 :
6121 : #ifdef MALLOC_VALIDATE
6122 19901 : chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
6123 19901 : if (chunk_rtree == NULL)
6124 0 : return (true);
6125 : #endif
6126 :
6127 19901 : malloc_initialized = true;
6128 :
6129 : #if defined(NEEDS_PTHREAD_MMAP_UNALIGNED_TSD)
6130 : if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
6131 : malloc_printf("<jemalloc>: Error in pthread_key_create()\n");
6132 : }
6133 : #endif
6134 :
6135 : #ifdef MOZ_MEMORY_DARWIN
6136 : /*
6137 : * Overwrite the default memory allocator to use jemalloc everywhere.
6138 : */
6139 : default_zone = malloc_default_zone();
6140 :
6141 : /*
6142 : * We only use jemalloc with MacOS 10.6 and 10.7. jemalloc is disabled
6143 : * on 32-bit builds (10.5 and 32-bit 10.6) due to bug 702250, an
6144 : * apparent MacOS bug. In fact, this code isn't even compiled on
6145 : * 32-bit builds.
6146 : *
6147 : * We'll have to update our code to work with newer versions, because
6148 : * the malloc zone layout is likely to change.
6149 : */
6150 :
6151 : osx_use_jemalloc = (default_zone->version == SNOW_LEOPARD_MALLOC_ZONE_T_VERSION ||
6152 : default_zone->version == LION_MALLOC_ZONE_T_VERSION);
6153 :
6154 : /* Allow us dynamically turn off jemalloc for testing. */
6155 : if (getenv("NO_MAC_JEMALLOC")) {
6156 : osx_use_jemalloc = false;
6157 : #ifdef __i386__
6158 : malloc_printf("Warning: NO_MAC_JEMALLOC has no effect on "
6159 : "i386 machines (such as this one).\n");
6160 : #endif
6161 : }
6162 :
6163 : if (osx_use_jemalloc) {
6164 : /*
6165 : * Convert the default szone to an "overlay zone" that is capable
6166 : * of deallocating szone-allocated objects, but allocating new
6167 : * objects from jemalloc.
6168 : */
6169 : size_t size = zone_version_size(default_zone->version);
6170 : szone2ozone(default_zone, size);
6171 : }
6172 : else {
6173 : szone = default_zone;
6174 : }
6175 : #endif
6176 :
6177 : #ifndef MOZ_MEMORY_WINDOWS
6178 19901 : malloc_mutex_unlock(&init_lock);
6179 : #endif
6180 19901 : return (false);
6181 : }
6182 :
6183 : /* XXX Why not just expose malloc_print_stats()? */
6184 : #ifdef MOZ_MEMORY_WINDOWS
6185 : void
6186 : malloc_shutdown()
6187 : {
6188 :
6189 : malloc_print_stats();
6190 : }
6191 : #endif
6192 :
6193 : /*
6194 : * End general internal functions.
6195 : */
6196 : /******************************************************************************/
6197 : /*
6198 : * Begin malloc(3)-compatible functions.
6199 : */
6200 :
6201 : /*
6202 : * Mangle standard interfaces, in order to avoid linking problems.
6203 : */
6204 : #if defined(MOZ_MEMORY_DARWIN) || defined(MOZ_MEMORY_WINDOWS) || \
6205 : defined(MOZ_MEMORY_ANDROID)
6206 :
6207 : #ifdef MOZ_MEMORY_ANDROID
6208 : /*
6209 : * On Android, we use __wrap_* instead of je_* to accomodate with the
6210 : * linker's --wrap option we use. That option prefixes the function
6211 : * names it is given with __wrap_.
6212 : */
6213 : #define wrap(a) __wrap_ ## a
6214 :
6215 : /* Extra wrappers for NSPR alloc functions */
6216 : void *
6217 : __wrap_PR_Malloc(size_t size) __attribute__((alias("__wrap_malloc")));
6218 : void *
6219 : __wrap_PR_Calloc(size_t num, size_t size) __attribute__((alias("__wrap_calloc")));
6220 : void *
6221 : __wrap_PR_Realloc(void *ptr, size_t size) __attribute__((alias("__wrap_realloc")));
6222 : void
6223 : __wrap_PR_Free(void *ptr) __attribute__((alias("__wrap_free")));
6224 :
6225 : #else
6226 : #define wrap(a) je_ ## a
6227 : #endif
6228 :
6229 : #define malloc(a) wrap(malloc)(a)
6230 : #define memalign(a, b) wrap(memalign)(a, b)
6231 : #define posix_memalign(a, b, c) wrap(posix_memalign)(a, b, c)
6232 : #define valloc(a) wrap(valloc)(a)
6233 : #define calloc(a, b) wrap(calloc)(a, b)
6234 : #define realloc(a, b) wrap(realloc)(a, b)
6235 : #define free(a) wrap(free)(a)
6236 : #define malloc_usable_size(a) wrap(malloc_usable_size)(a)
6237 :
6238 : void *malloc(size_t size);
6239 :
6240 : char *
6241 : wrap(strndup)(const char *src, size_t len) {
6242 : char* dst = (char*) malloc(len + 1);
6243 : if (dst)
6244 : strncpy(dst, src, len + 1);
6245 : return dst;
6246 : }
6247 :
6248 : char *
6249 : wrap(strdup)(const char *src) {
6250 : size_t len = strlen(src);
6251 : return wrap(strndup)(src, len);
6252 : }
6253 : #endif
6254 :
6255 : /*
6256 : * Even though we compile with MOZ_MEMORY, we may have to dynamically decide
6257 : * not to use jemalloc, as discussed above. However, we call jemalloc
6258 : * functions directly from mozalloc. Since it's pretty dangerous to mix the
6259 : * allocators, we need to call the OSX allocators from the functions below,
6260 : * when osx_use_jemalloc is not (dynamically) set.
6261 : *
6262 : * Note that we assume jemalloc is enabled on i386. This is safe because the
6263 : * only i386 versions of MacOS are 10.5 and 10.6, which we support. We have to
6264 : * do this because madvise isn't in the malloc zone struct for 10.5.
6265 : *
6266 : * This means that NO_MAC_JEMALLOC doesn't work on i386.
6267 : */
6268 : #if defined(MOZ_MEMORY_DARWIN) && !defined(__i386__)
6269 : #define DARWIN_ONLY(A) if (!osx_use_jemalloc) { A; }
6270 : #else
6271 : #define DARWIN_ONLY(A)
6272 : #endif
6273 :
6274 : void *
6275 : malloc(size_t size)
6276 : {
6277 : void *ret;
6278 :
6279 : DARWIN_ONLY(return (szone->malloc)(szone, size));
6280 :
6281 108483841 : if (malloc_init()) {
6282 0 : ret = NULL;
6283 0 : goto RETURN;
6284 : }
6285 :
6286 108483676 : if (size == 0) {
6287 : #ifdef MALLOC_SYSV
6288 72 : if (opt_sysv == false)
6289 : #endif
6290 72 : size = 1;
6291 : #ifdef MALLOC_SYSV
6292 : else {
6293 0 : ret = NULL;
6294 0 : goto RETURN;
6295 : }
6296 : #endif
6297 : }
6298 :
6299 108483676 : ret = imalloc(size);
6300 :
6301 : RETURN:
6302 108484005 : if (ret == NULL) {
6303 : #ifdef MALLOC_XMALLOC
6304 1 : if (opt_xmalloc) {
6305 0 : _malloc_message(_getprogname(),
6306 : ": (malloc) Error in malloc(): out of memory\n", "",
6307 : "");
6308 0 : abort();
6309 : }
6310 : #endif
6311 1 : errno = ENOMEM;
6312 : }
6313 :
6314 108484005 : UTRACE(0, size, ret);
6315 108484005 : return (ret);
6316 : }
6317 :
6318 : /*
6319 : * In ELF systems the default visibility allows symbols to be preempted at
6320 : * runtime. This in turn prevents the uses of memalign in this file from being
6321 : * optimized. What we do in here is define two aliasing symbols (they point to
6322 : * the same code): memalign and memalign_internal. The internal version has
6323 : * hidden visibility and is used in every reference from this file.
6324 : *
6325 : * For more information on this technique, see section 2.2.7 (Avoid Using
6326 : * Exported Symbols) in http://www.akkadia.org/drepper/dsohowto.pdf.
6327 : */
6328 :
6329 : #if defined(__GNUC__) && !defined(MOZ_MEMORY_DARWIN)
6330 : #define MOZ_MEMORY_ELF
6331 : #endif
6332 :
6333 : #ifdef MOZ_MEMORY_SOLARIS
6334 : # ifdef __SUNPRO_C
6335 : void *
6336 : memalign(size_t alignment, size_t size);
6337 : #pragma no_inline(memalign)
6338 : # elif (defined(__GNUC__))
6339 : __attribute__((noinline))
6340 : # endif
6341 : #else
6342 : #if (defined(MOZ_MEMORY_ELF))
6343 : __attribute__((visibility ("hidden")))
6344 : #endif
6345 : #endif
6346 :
6347 :
6348 : #ifdef MOZ_MEMORY_ELF
6349 : #define MEMALIGN memalign_internal
6350 : #else
6351 : #define MEMALIGN memalign
6352 : #endif
6353 :
6354 : void *
6355 30860 : MEMALIGN(size_t alignment, size_t size)
6356 : {
6357 : void *ret;
6358 :
6359 : DARWIN_ONLY(return (szone->memalign)(szone, alignment, size));
6360 :
6361 30860 : assert(((alignment - 1) & alignment) == 0);
6362 :
6363 30860 : if (malloc_init()) {
6364 0 : ret = NULL;
6365 0 : goto RETURN;
6366 : }
6367 :
6368 30860 : if (size == 0) {
6369 : #ifdef MALLOC_SYSV
6370 0 : if (opt_sysv == false)
6371 : #endif
6372 0 : size = 1;
6373 : #ifdef MALLOC_SYSV
6374 : else {
6375 0 : ret = NULL;
6376 0 : goto RETURN;
6377 : }
6378 : #endif
6379 : }
6380 :
6381 30860 : alignment = alignment < sizeof(void*) ? sizeof(void*) : alignment;
6382 30860 : ret = ipalloc(alignment, size);
6383 :
6384 : RETURN:
6385 : #ifdef MALLOC_XMALLOC
6386 30860 : if (opt_xmalloc && ret == NULL) {
6387 0 : _malloc_message(_getprogname(),
6388 : ": (malloc) Error in memalign(): out of memory\n", "", "");
6389 0 : abort();
6390 : }
6391 : #endif
6392 30860 : UTRACE(0, size, ret);
6393 30860 : return (ret);
6394 : }
6395 :
6396 : #ifdef MOZ_MEMORY_ELF
6397 : extern void *
6398 : memalign(size_t alignment, size_t size) __attribute__((alias ("memalign_internal"), visibility ("default")));
6399 : #endif
6400 :
6401 : int
6402 : posix_memalign(void **memptr, size_t alignment, size_t size)
6403 : {
6404 : void *result;
6405 :
6406 : /* Make sure that alignment is a large enough power of 2. */
6407 30837 : if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
6408 : #ifdef MALLOC_XMALLOC
6409 0 : if (opt_xmalloc) {
6410 0 : _malloc_message(_getprogname(),
6411 : ": (malloc) Error in posix_memalign(): "
6412 : "invalid alignment\n", "", "");
6413 0 : abort();
6414 : }
6415 : #endif
6416 0 : return (EINVAL);
6417 : }
6418 :
6419 : /* The 0-->1 size promotion is done in the memalign() call below */
6420 :
6421 30837 : result = MEMALIGN(alignment, size);
6422 :
6423 30837 : if (result == NULL)
6424 0 : return (ENOMEM);
6425 :
6426 30837 : *memptr = result;
6427 30837 : return (0);
6428 : }
6429 :
6430 : void *
6431 : valloc(size_t size)
6432 : {
6433 0 : return (MEMALIGN(pagesize, size));
6434 : }
6435 :
6436 : void *
6437 : calloc(size_t num, size_t size)
6438 : {
6439 : void *ret;
6440 : size_t num_size;
6441 :
6442 : DARWIN_ONLY(return (szone->calloc)(szone, num, size));
6443 :
6444 7986415 : if (malloc_init()) {
6445 0 : num_size = 0;
6446 0 : ret = NULL;
6447 0 : goto RETURN;
6448 : }
6449 :
6450 7986414 : num_size = num * size;
6451 7986414 : if (num_size == 0) {
6452 : #ifdef MALLOC_SYSV
6453 0 : if ((opt_sysv == false) && ((num == 0) || (size == 0)))
6454 : #endif
6455 0 : num_size = 1;
6456 : #ifdef MALLOC_SYSV
6457 : else {
6458 0 : ret = NULL;
6459 0 : goto RETURN;
6460 : }
6461 : #endif
6462 : /*
6463 : * Try to avoid division here. We know that it isn't possible to
6464 : * overflow during multiplication if neither operand uses any of the
6465 : * most significant half of the bits in a size_t.
6466 : */
6467 7986414 : } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
6468 963 : && (num_size / size != num)) {
6469 : /* size_t overflow. */
6470 0 : ret = NULL;
6471 0 : goto RETURN;
6472 : }
6473 :
6474 7986414 : ret = icalloc(num_size);
6475 :
6476 : RETURN:
6477 7986415 : if (ret == NULL) {
6478 : #ifdef MALLOC_XMALLOC
6479 18 : if (opt_xmalloc) {
6480 0 : _malloc_message(_getprogname(),
6481 : ": (malloc) Error in calloc(): out of memory\n", "",
6482 : "");
6483 0 : abort();
6484 : }
6485 : #endif
6486 18 : errno = ENOMEM;
6487 : }
6488 :
6489 7986415 : UTRACE(0, num_size, ret);
6490 7986415 : return (ret);
6491 : }
6492 :
6493 : void *
6494 : realloc(void *ptr, size_t size)
6495 : {
6496 : void *ret;
6497 :
6498 : DARWIN_ONLY(return (szone->realloc)(szone, ptr, size));
6499 :
6500 9582979 : if (size == 0) {
6501 : #ifdef MALLOC_SYSV
6502 0 : if (opt_sysv == false)
6503 : #endif
6504 0 : size = 1;
6505 : #ifdef MALLOC_SYSV
6506 : else {
6507 0 : if (ptr != NULL)
6508 0 : idalloc(ptr);
6509 0 : ret = NULL;
6510 0 : goto RETURN;
6511 : }
6512 : #endif
6513 : }
6514 :
6515 9582979 : if (ptr != NULL) {
6516 4723565 : assert(malloc_initialized);
6517 :
6518 4723565 : ret = iralloc(ptr, size);
6519 :
6520 4723565 : if (ret == NULL) {
6521 : #ifdef MALLOC_XMALLOC
6522 0 : if (opt_xmalloc) {
6523 0 : _malloc_message(_getprogname(),
6524 : ": (malloc) Error in realloc(): out of "
6525 : "memory\n", "", "");
6526 0 : abort();
6527 : }
6528 : #endif
6529 0 : errno = ENOMEM;
6530 : }
6531 : } else {
6532 4859414 : if (malloc_init())
6533 0 : ret = NULL;
6534 : else
6535 4859414 : ret = imalloc(size);
6536 :
6537 4859414 : if (ret == NULL) {
6538 : #ifdef MALLOC_XMALLOC
6539 0 : if (opt_xmalloc) {
6540 0 : _malloc_message(_getprogname(),
6541 : ": (malloc) Error in realloc(): out of "
6542 : "memory\n", "", "");
6543 0 : abort();
6544 : }
6545 : #endif
6546 0 : errno = ENOMEM;
6547 : }
6548 : }
6549 :
6550 : #ifdef MALLOC_SYSV
6551 : RETURN:
6552 : #endif
6553 9582979 : UTRACE(ptr, size, ret);
6554 9582979 : return (ret);
6555 : }
6556 :
6557 : void
6558 : free(void *ptr)
6559 : {
6560 : size_t offset;
6561 :
6562 : DARWIN_ONLY((szone->free)(szone, ptr); return);
6563 :
6564 128640171 : UTRACE(ptr, 0, 0);
6565 :
6566 : /*
6567 : * A version of idalloc that checks for NULL pointer but only for
6568 : * huge allocations assuming that CHUNK_ADDR2OFFSET(NULL) == 0.
6569 : */
6570 : assert(CHUNK_ADDR2OFFSET(NULL) == 0);
6571 128640889 : offset = CHUNK_ADDR2OFFSET(ptr);
6572 128640889 : if (offset != 0)
6573 116879861 : arena_dalloc(ptr, offset);
6574 11761028 : else if (ptr != NULL)
6575 3795 : huge_dalloc(ptr);
6576 128658207 : }
6577 :
6578 : /*
6579 : * End malloc(3)-compatible functions.
6580 : */
6581 : /******************************************************************************/
6582 : /*
6583 : * Begin non-standard functions.
6584 : */
6585 :
6586 : /* This was added by Mozilla for use by SQLite. */
6587 : size_t
6588 7134088 : je_malloc_usable_size_in_advance(size_t size)
6589 : {
6590 : /*
6591 : * This duplicates the logic in imalloc(), arena_malloc() and
6592 : * arena_malloc_small().
6593 : */
6594 7134088 : if (size < small_min) {
6595 : /* Small (tiny). */
6596 544582 : size = pow2_ceil(size);
6597 : /*
6598 : * We omit the #ifdefs from arena_malloc_small() --
6599 : * it can be inaccurate with its size in some cases, but this
6600 : * function must be accurate.
6601 : */
6602 544582 : if (size < (1U << TINY_MIN_2POW))
6603 34768 : size = (1U << TINY_MIN_2POW);
6604 6589506 : } else if (size <= small_max) {
6605 : /* Small (quantum-spaced). */
6606 4271051 : size = QUANTUM_CEILING(size);
6607 2318455 : } else if (size <= bin_maxclass) {
6608 : /* Small (sub-page). */
6609 2190858 : size = pow2_ceil(size);
6610 127597 : } else if (size <= arena_maxclass) {
6611 : /* Large. */
6612 127428 : size = PAGE_CEILING(size);
6613 : } else {
6614 : /*
6615 : * Huge. We use PAGE_CEILING to get psize, instead of using
6616 : * CHUNK_CEILING to get csize. This ensures that this
6617 : * malloc_usable_size(malloc(n)) always matches
6618 : * je_malloc_usable_size_in_advance(n).
6619 : */
6620 169 : size = PAGE_CEILING(size);
6621 : }
6622 7134088 : return size;
6623 : }
6624 :
6625 :
6626 : #ifdef MOZ_MEMORY_ANDROID
6627 : size_t
6628 : malloc_usable_size(void *ptr)
6629 : #else
6630 : size_t
6631 14218181 : malloc_usable_size(const void *ptr)
6632 : #endif
6633 : {
6634 : DARWIN_ONLY(return (szone->size)(szone, ptr));
6635 :
6636 : #ifdef MALLOC_VALIDATE
6637 14218181 : return (isalloc_validate(ptr));
6638 : #else
6639 : assert(ptr != NULL);
6640 :
6641 : return (isalloc(ptr));
6642 : #endif
6643 : }
6644 :
6645 : void
6646 6 : jemalloc_stats(jemalloc_stats_t *stats)
6647 : {
6648 : size_t i;
6649 :
6650 6 : assert(stats != NULL);
6651 :
6652 : /*
6653 : * Gather runtime settings.
6654 : */
6655 6 : stats->opt_abort = opt_abort;
6656 6 : stats->opt_junk =
6657 : #ifdef MALLOC_FILL
6658 : opt_junk ? true :
6659 : #endif
6660 : false;
6661 6 : stats->opt_utrace =
6662 : #ifdef MALLOC_UTRACE
6663 : opt_utrace ? true :
6664 : #endif
6665 : false;
6666 6 : stats->opt_sysv =
6667 : #ifdef MALLOC_SYSV
6668 : opt_sysv ? true :
6669 : #endif
6670 : false;
6671 6 : stats->opt_xmalloc =
6672 : #ifdef MALLOC_XMALLOC
6673 : opt_xmalloc ? true :
6674 : #endif
6675 : false;
6676 6 : stats->opt_zero =
6677 : #ifdef MALLOC_FILL
6678 : opt_zero ? true :
6679 : #endif
6680 : false;
6681 6 : stats->narenas = narenas;
6682 6 : stats->balance_threshold =
6683 : #ifdef MALLOC_BALANCE
6684 : opt_balance_threshold
6685 : #else
6686 : SIZE_T_MAX
6687 : #endif
6688 : ;
6689 6 : stats->quantum = quantum;
6690 6 : stats->small_max = small_max;
6691 6 : stats->large_max = arena_maxclass;
6692 6 : stats->chunksize = chunksize;
6693 6 : stats->dirty_max = opt_dirty_max;
6694 :
6695 : /*
6696 : * Gather current memory usage statistics.
6697 : */
6698 6 : stats->mapped = 0;
6699 6 : stats->committed = 0;
6700 6 : stats->allocated = 0;
6701 6 : stats->dirty = 0;
6702 :
6703 : /* Get huge mapped/allocated. */
6704 6 : malloc_mutex_lock(&huge_mtx);
6705 6 : stats->mapped += stats_chunks.curchunks * chunksize;
6706 6 : stats->committed += huge_allocated;
6707 6 : stats->allocated += huge_allocated;
6708 6 : malloc_mutex_unlock(&huge_mtx);
6709 :
6710 : /* Get base mapped. */
6711 6 : malloc_mutex_lock(&base_mtx);
6712 6 : stats->mapped += base_mapped;
6713 6 : assert(base_committed <= base_mapped);
6714 6 : stats->committed += base_committed;
6715 6 : malloc_mutex_unlock(&base_mtx);
6716 :
6717 : /* Iterate over arenas and their chunks. */
6718 12 : for (i = 0; i < narenas; i++) {
6719 6 : arena_t *arena = arenas[i];
6720 6 : if (arena != NULL) {
6721 6 : malloc_spin_lock(&arena->lock);
6722 6 : stats->allocated += arena->stats.allocated_small;
6723 6 : stats->allocated += arena->stats.allocated_large;
6724 6 : stats->committed += (arena->stats.committed <<
6725 : pagesize_2pow);
6726 6 : stats->dirty += (arena->ndirty << pagesize_2pow);
6727 6 : malloc_spin_unlock(&arena->lock);
6728 : }
6729 : }
6730 :
6731 6 : assert(stats->mapped >= stats->committed);
6732 6 : assert(stats->committed >= stats->allocated);
6733 6 : }
6734 :
6735 : #ifdef MALLOC_DOUBLE_PURGE
6736 :
6737 : /* Explicitly remove all of this chunk's MADV_FREE'd pages from memory. */
6738 : static void
6739 : hard_purge_chunk(arena_chunk_t *chunk)
6740 : {
6741 : /* See similar logic in arena_purge(). */
6742 :
6743 : size_t i;
6744 : for (i = arena_chunk_header_npages; i < chunk_npages; i++) {
6745 : /* Find all adjacent pages with CHUNK_MAP_MADVISED set. */
6746 : size_t npages;
6747 : for (npages = 0;
6748 : chunk->map[i + npages].bits & CHUNK_MAP_MADVISED && i + npages < chunk_npages;
6749 : npages++) {
6750 : /* Turn off the chunk's MADV_FREED bit and turn on its
6751 : * DECOMMITTED bit. */
6752 : assert(!(chunk->map[i + npages].bits & CHUNK_MAP_DECOMMITTED));
6753 : chunk->map[i + npages].bits ^= CHUNK_MAP_MADVISED_OR_DECOMMITTED;
6754 : }
6755 :
6756 : /* We could use mincore to find out which pages are actually
6757 : * present, but it's not clear that's better. */
6758 : if (npages > 0) {
6759 : pages_decommit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
6760 : pages_commit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
6761 : }
6762 : i += npages;
6763 : }
6764 : }
6765 :
6766 : /* Explicitly remove all of this arena's MADV_FREE'd pages from memory. */
6767 : static void
6768 : hard_purge_arena(arena_t *arena)
6769 : {
6770 : malloc_spin_lock(&arena->lock);
6771 :
6772 : while (!LinkedList_IsEmpty(&arena->chunks_madvised)) {
6773 : LinkedList* next = arena->chunks_madvised.next;
6774 : arena_chunk_t *chunk =
6775 : LinkedList_Get(arena->chunks_madvised.next,
6776 : arena_chunk_t, chunks_madvised_elem);
6777 : hard_purge_chunk(chunk);
6778 : LinkedList_Remove(&chunk->chunks_madvised_elem);
6779 : }
6780 :
6781 : malloc_spin_unlock(&arena->lock);
6782 : }
6783 :
6784 : void
6785 : jemalloc_purge_freed_pages()
6786 : {
6787 : size_t i;
6788 : for (i = 0; i < narenas; i++) {
6789 : arena_t *arena = arenas[i];
6790 : if (arena != NULL)
6791 : hard_purge_arena(arena);
6792 : }
6793 : }
6794 :
6795 : #else /* !defined MALLOC_DOUBLE_PURGE */
6796 :
6797 : void
6798 0 : jemalloc_purge_freed_pages()
6799 : {
6800 : /* Do nothing. */
6801 0 : }
6802 :
6803 : #endif /* defined MALLOC_DOUBLE_PURGE */
6804 :
6805 :
6806 :
6807 : #ifdef MOZ_MEMORY_WINDOWS
6808 : void*
6809 : _recalloc(void *ptr, size_t count, size_t size)
6810 : {
6811 : size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
6812 : size_t newsize = count * size;
6813 :
6814 : /*
6815 : * In order for all trailing bytes to be zeroed, the caller needs to
6816 : * use calloc(), followed by recalloc(). However, the current calloc()
6817 : * implementation only zeros the bytes requested, so if recalloc() is
6818 : * to work 100% correctly, calloc() will need to change to zero
6819 : * trailing bytes.
6820 : */
6821 :
6822 : ptr = realloc(ptr, newsize);
6823 : if (ptr != NULL && oldsize < newsize) {
6824 : memset((void *)((uintptr_t)ptr + oldsize), 0, newsize -
6825 : oldsize);
6826 : }
6827 :
6828 : return ptr;
6829 : }
6830 :
6831 : /*
6832 : * This impl of _expand doesn't ever actually expand or shrink blocks: it
6833 : * simply replies that you may continue using a shrunk block.
6834 : */
6835 : void*
6836 : _expand(void *ptr, size_t newsize)
6837 : {
6838 : if (isalloc(ptr) >= newsize)
6839 : return ptr;
6840 :
6841 : return NULL;
6842 : }
6843 :
6844 : size_t
6845 : _msize(const void *ptr)
6846 : {
6847 :
6848 : return malloc_usable_size(ptr);
6849 : }
6850 : #endif
6851 :
6852 : /*
6853 : * End non-standard functions.
6854 : */
6855 : /******************************************************************************/
6856 : /*
6857 : * Begin library-private functions, used by threading libraries for protection
6858 : * of malloc during fork(). These functions are only called if the program is
6859 : * running in threaded mode, so there is no need to check whether the program
6860 : * is threaded here.
6861 : */
6862 :
6863 : #ifdef MOZ_MEMORY_ANDROID
6864 : void
6865 : #else
6866 : static void
6867 : #endif
6868 1097 : _malloc_prefork(void)
6869 : {
6870 : unsigned i;
6871 :
6872 : /* Acquire all mutexes in a safe order. */
6873 :
6874 1097 : malloc_spin_lock(&arenas_lock);
6875 2194 : for (i = 0; i < narenas; i++) {
6876 1097 : if (arenas[i] != NULL)
6877 1097 : malloc_spin_lock(&arenas[i]->lock);
6878 : }
6879 :
6880 1097 : malloc_mutex_lock(&base_mtx);
6881 :
6882 1097 : malloc_mutex_lock(&huge_mtx);
6883 1097 : }
6884 :
6885 : #ifdef MOZ_MEMORY_ANDROID
6886 : void
6887 : #else
6888 : static void
6889 : #endif
6890 1097 : _malloc_postfork(void)
6891 : {
6892 : unsigned i;
6893 :
6894 : /* Release all mutexes, now that fork() has completed. */
6895 :
6896 1097 : malloc_mutex_unlock(&huge_mtx);
6897 :
6898 1097 : malloc_mutex_unlock(&base_mtx);
6899 :
6900 2194 : for (i = 0; i < narenas; i++) {
6901 1097 : if (arenas[i] != NULL)
6902 1097 : malloc_spin_unlock(&arenas[i]->lock);
6903 : }
6904 1097 : malloc_spin_unlock(&arenas_lock);
6905 1097 : }
6906 :
6907 : /*
6908 : * End library-private functions.
6909 : */
6910 : /******************************************************************************/
6911 :
6912 : #ifdef HAVE_DLOPEN
6913 : # include <dlfcn.h>
6914 : #endif
6915 :
6916 : #ifdef MOZ_MEMORY_DARWIN
6917 :
6918 : static void *
6919 : zone_malloc(malloc_zone_t *zone, size_t size)
6920 : {
6921 :
6922 : return (malloc(size));
6923 : }
6924 :
6925 : static void *
6926 : zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
6927 : {
6928 :
6929 : return (calloc(num, size));
6930 : }
6931 :
6932 : static void *
6933 : zone_valloc(malloc_zone_t *zone, size_t size)
6934 : {
6935 : void *ret = NULL; /* Assignment avoids useless compiler warning. */
6936 :
6937 : posix_memalign(&ret, pagesize, size);
6938 :
6939 : return (ret);
6940 : }
6941 :
6942 : static void *
6943 : zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
6944 : {
6945 : return (memalign(alignment, size));
6946 : }
6947 :
6948 : static void *
6949 : zone_destroy(malloc_zone_t *zone)
6950 : {
6951 :
6952 : /* This function should never be called. */
6953 : assert(false);
6954 : return (NULL);
6955 : }
6956 :
6957 : static size_t
6958 : zone_good_size(malloc_zone_t *zone, size_t size)
6959 : {
6960 : return je_malloc_usable_size_in_advance(size);
6961 : }
6962 :
6963 : static size_t
6964 : ozone_size(malloc_zone_t *zone, void *ptr)
6965 : {
6966 : size_t ret = isalloc_validate(ptr);
6967 : if (ret == 0)
6968 : ret = szone->size(zone, ptr);
6969 :
6970 : return ret;
6971 : }
6972 :
6973 : static void
6974 : ozone_free(malloc_zone_t *zone, void *ptr)
6975 : {
6976 : if (isalloc_validate(ptr) != 0)
6977 : free(ptr);
6978 : else {
6979 : size_t size = szone->size(zone, ptr);
6980 : if (size != 0)
6981 : (szone->free)(zone, ptr);
6982 : /* Otherwise we leak. */
6983 : }
6984 : }
6985 :
6986 : static void *
6987 : ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
6988 : {
6989 : size_t oldsize;
6990 : if (ptr == NULL)
6991 : return (malloc(size));
6992 :
6993 : oldsize = isalloc_validate(ptr);
6994 : if (oldsize != 0)
6995 : return (realloc(ptr, size));
6996 : else {
6997 : oldsize = szone->size(zone, ptr);
6998 : if (oldsize == 0)
6999 : return (malloc(size));
7000 : else {
7001 : void *ret = malloc(size);
7002 : if (ret != NULL) {
7003 : memcpy(ret, ptr, (oldsize < size) ? oldsize :
7004 : size);
7005 : (szone->free)(zone, ptr);
7006 : }
7007 : return (ret);
7008 : }
7009 : }
7010 : }
7011 :
7012 : static unsigned
7013 : ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
7014 : unsigned num_requested)
7015 : {
7016 : /* Don't bother implementing this interface, since it isn't required. */
7017 : return 0;
7018 : }
7019 :
7020 : static void
7021 : ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
7022 : {
7023 : unsigned i;
7024 :
7025 : for (i = 0; i < num; i++)
7026 : ozone_free(zone, to_be_freed[i]);
7027 : }
7028 :
7029 : static void
7030 : ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
7031 : {
7032 : if (isalloc_validate(ptr) != 0) {
7033 : assert(isalloc_validate(ptr) == size);
7034 : free(ptr);
7035 : } else {
7036 : assert(size == szone->size(zone, ptr));
7037 : l_szone.m16(zone, ptr, size);
7038 : }
7039 : }
7040 :
7041 : static void
7042 : ozone_force_lock(malloc_zone_t *zone)
7043 : {
7044 : /* jemalloc locking is taken care of by the normal jemalloc zone. */
7045 : szone->introspect->force_lock(zone);
7046 : }
7047 :
7048 : static void
7049 : ozone_force_unlock(malloc_zone_t *zone)
7050 : {
7051 : /* jemalloc locking is taken care of by the normal jemalloc zone. */
7052 : szone->introspect->force_unlock(zone);
7053 : }
7054 :
7055 : static size_t
7056 : zone_version_size(int version)
7057 : {
7058 : switch (version)
7059 : {
7060 : case SNOW_LEOPARD_MALLOC_ZONE_T_VERSION:
7061 : return sizeof(snow_leopard_malloc_zone);
7062 : case LEOPARD_MALLOC_ZONE_T_VERSION:
7063 : return sizeof(leopard_malloc_zone);
7064 : default:
7065 : case LION_MALLOC_ZONE_T_VERSION:
7066 : return sizeof(lion_malloc_zone);
7067 : }
7068 : }
7069 :
7070 : /*
7071 : * Overlay the default scalable zone (szone) such that existing allocations are
7072 : * drained, and further allocations come from jemalloc. This is necessary
7073 : * because Core Foundation directly accesses and uses the szone before the
7074 : * jemalloc library is even loaded.
7075 : */
7076 : static void
7077 : szone2ozone(malloc_zone_t *default_zone, size_t size)
7078 : {
7079 : lion_malloc_zone *l_zone;
7080 : assert(malloc_initialized);
7081 :
7082 : /*
7083 : * Stash a copy of the original szone so that we can call its
7084 : * functions as needed. Note that internally, the szone stores its
7085 : * bookkeeping data structures immediately following the malloc_zone_t
7086 : * header, so when calling szone functions, we need to pass a pointer to
7087 : * the original zone structure.
7088 : */
7089 : memcpy(szone, default_zone, size);
7090 :
7091 : /* OSX 10.7 allocates the default zone in protected memory. */
7092 : if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
7093 : void* start_of_page = (void*)((size_t)(default_zone) & ~pagesize_mask);
7094 : mprotect (start_of_page, size, PROT_READ | PROT_WRITE);
7095 : }
7096 :
7097 : default_zone->size = (void *)ozone_size;
7098 : default_zone->malloc = (void *)zone_malloc;
7099 : default_zone->calloc = (void *)zone_calloc;
7100 : default_zone->valloc = (void *)zone_valloc;
7101 : default_zone->free = (void *)ozone_free;
7102 : default_zone->realloc = (void *)ozone_realloc;
7103 : default_zone->destroy = (void *)zone_destroy;
7104 : default_zone->batch_malloc = NULL;
7105 : default_zone->batch_free = ozone_batch_free;
7106 : default_zone->introspect = ozone_introspect;
7107 :
7108 : /* Don't modify default_zone->zone_name; Mac libc may rely on the name
7109 : * being unchanged. See Mozilla bug 694896. */
7110 :
7111 : ozone_introspect->enumerator = NULL;
7112 : ozone_introspect->good_size = (void *)zone_good_size;
7113 : ozone_introspect->check = NULL;
7114 : ozone_introspect->print = NULL;
7115 : ozone_introspect->log = NULL;
7116 : ozone_introspect->force_lock = (void *)ozone_force_lock;
7117 : ozone_introspect->force_unlock = (void *)ozone_force_unlock;
7118 : ozone_introspect->statistics = NULL;
7119 :
7120 : /* Platform-dependent structs */
7121 : l_zone = (lion_malloc_zone*)(default_zone);
7122 :
7123 : if (default_zone->version >= SNOW_LEOPARD_MALLOC_ZONE_T_VERSION) {
7124 : l_zone->m15 = (void (*)())zone_memalign;
7125 : l_zone->m16 = (void (*)())ozone_free_definite_size;
7126 : l_ozone_introspect.m9 = NULL;
7127 : }
7128 :
7129 : if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
7130 : l_zone->m17 = NULL;
7131 : l_ozone_introspect.m10 = NULL;
7132 : l_ozone_introspect.m11 = NULL;
7133 : l_ozone_introspect.m12 = NULL;
7134 : l_ozone_introspect.m13 = NULL;
7135 : }
7136 : }
7137 :
7138 : __attribute__((constructor))
7139 : void
7140 : jemalloc_darwin_init(void)
7141 : {
7142 : if (malloc_init_hard())
7143 : abort();
7144 : }
7145 :
7146 : #elif defined(__GLIBC__) && !defined(__UCLIBC__)
7147 : /*
7148 : * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
7149 : * to inconsistently reference libc's malloc(3)-compatible functions
7150 : * (bug 493541).
7151 : *
7152 : * These definitions interpose hooks in glibc. The functions are actually
7153 : * passed an extra argument for the caller return address, which will be
7154 : * ignored.
7155 : */
7156 : void (*__free_hook)(void *ptr) = free;
7157 : void *(*__malloc_hook)(size_t size) = malloc;
7158 : void *(*__realloc_hook)(void *ptr, size_t size) = realloc;
7159 : void *(*__memalign_hook)(size_t alignment, size_t size) = MEMALIGN;
7160 :
7161 : #elif defined(RTLD_DEEPBIND)
7162 : /*
7163 : * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
7164 : * implementations permit similar inconsistencies? Should STV_SINGLETON
7165 : * visibility be used for interposition where available?
7166 : */
7167 : # error "Interposing malloc is unsafe on this system without libc malloc hooks."
7168 : #endif
7169 :
7170 : #ifdef MOZ_MEMORY_WINDOWS
7171 : /*
7172 : * In the new style jemalloc integration jemalloc is built as a separate
7173 : * shared library. Since we're no longer hooking into the CRT binary,
7174 : * we need to initialize the heap at the first opportunity we get.
7175 : * DLL_PROCESS_ATTACH in DllMain is that opportunity.
7176 : */
7177 : BOOL APIENTRY DllMain(HINSTANCE hModule,
7178 : DWORD reason,
7179 : LPVOID lpReserved)
7180 : {
7181 : switch (reason) {
7182 : case DLL_PROCESS_ATTACH:
7183 : /* Don't force the system to page DllMain back in every time
7184 : * we create/destroy a thread */
7185 : DisableThreadLibraryCalls(hModule);
7186 : /* Initialize the heap */
7187 : malloc_init_hard();
7188 : break;
7189 :
7190 : case DLL_PROCESS_DETACH:
7191 : break;
7192 :
7193 : }
7194 :
7195 : return TRUE;
7196 : }
7197 :
7198 : /*
7199 : * There's a fun allocator mismatch in (at least) the VS 2010 CRT
7200 : * (see the giant comment in this directory's Makefile.in
7201 : * that gets redirected here to avoid a crash on shutdown.
7202 : */
7203 : void
7204 : je_dumb_free_thunk(void *ptr)
7205 : {
7206 : return; /* shutdown leaks that we don't care about */
7207 : }
7208 :
7209 : #include <wchar.h>
7210 :
7211 : /*
7212 : * We also need to provide our own impl of wcsdup so that we don't ask
7213 : * the CRT for memory from its heap (which will then be unfreeable).
7214 : */
7215 : wchar_t *je_wcsdup(const wchar_t *src)
7216 : {
7217 : size_t len = wcslen(src);
7218 : wchar_t* dst = (wchar_t*)je_malloc((len + 1) * sizeof(wchar_t));
7219 : if(dst)
7220 : wcsncpy(dst, src, len + 1);
7221 : return dst;
7222 : }
7223 :
7224 : #endif
|