LCOV - code coverage report
Current view: directory - js/src - jsgc.h (source / functions) Found Hit Coverage
Test: app.info Lines: 534 491 91.9 %
Date: 2012-06-02 Functions: 127 119 93.7 %

       1                 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
       2                 :  *
       3                 :  * ***** BEGIN LICENSE BLOCK *****
       4                 :  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
       5                 :  *
       6                 :  * The contents of this file are subject to the Mozilla Public License Version
       7                 :  * 1.1 (the "License"); you may not use this file except in compliance with
       8                 :  * the License. You may obtain a copy of the License at
       9                 :  * http://www.mozilla.org/MPL/
      10                 :  *
      11                 :  * Software distributed under the License is distributed on an "AS IS" basis,
      12                 :  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
      13                 :  * for the specific language governing rights and limitations under the
      14                 :  * License.
      15                 :  *
      16                 :  * The Original Code is Mozilla Communicator client code, released
      17                 :  * March 31, 1998.
      18                 :  *
      19                 :  * The Initial Developer of the Original Code is
      20                 :  * Netscape Communications Corporation.
      21                 :  * Portions created by the Initial Developer are Copyright (C) 1998
      22                 :  * the Initial Developer. All Rights Reserved.
      23                 :  *
      24                 :  * Contributor(s):
      25                 :  *
      26                 :  * Alternatively, the contents of this file may be used under the terms of
      27                 :  * either of the GNU General Public License Version 2 or later (the "GPL"),
      28                 :  * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
      29                 :  * in which case the provisions of the GPL or the LGPL are applicable instead
      30                 :  * of those above. If you wish to allow use of your version of this file only
      31                 :  * under the terms of either the GPL or the LGPL, and not to allow others to
      32                 :  * use your version of this file under the terms of the MPL, indicate your
      33                 :  * decision by deleting the provisions above and replace them with the notice
      34                 :  * and other provisions required by the GPL or the LGPL. If you do not delete
      35                 :  * the provisions above, a recipient may use your version of this file under
      36                 :  * the terms of any one of the MPL, the GPL or the LGPL.
      37                 :  *
      38                 :  * ***** END LICENSE BLOCK ***** */
      39                 : 
      40                 : #ifndef jsgc_h___
      41                 : #define jsgc_h___
      42                 : 
      43                 : /*
      44                 :  * JS Garbage Collector.
      45                 :  */
      46                 : #include <setjmp.h>
      47                 : 
      48                 : #include "mozilla/Util.h"
      49                 : 
      50                 : #include "jsalloc.h"
      51                 : #include "jstypes.h"
      52                 : #include "jsprvtd.h"
      53                 : #include "jspubtd.h"
      54                 : #include "jsdhash.h"
      55                 : #include "jslock.h"
      56                 : #include "jsutil.h"
      57                 : #include "jsversion.h"
      58                 : #include "jscell.h"
      59                 : 
      60                 : #include "ds/BitArray.h"
      61                 : #include "gc/Statistics.h"
      62                 : #include "js/HashTable.h"
      63                 : #include "js/Vector.h"
      64                 : #include "js/TemplateLib.h"
      65                 : 
      66                 : struct JSCompartment;
      67                 : 
      68                 : extern "C" void
      69                 : js_TraceXML(JSTracer *trc, JSXML* thing);
      70                 : 
      71                 : #if JS_STACK_GROWTH_DIRECTION > 0
      72                 : # define JS_CHECK_STACK_SIZE(limit, lval)  ((uintptr_t)(lval) < limit)
      73                 : #else
      74                 : # define JS_CHECK_STACK_SIZE(limit, lval)  ((uintptr_t)(lval) > limit)
      75                 : #endif
      76                 : 
      77                 : namespace js {
      78                 : 
      79                 : class GCHelperThread;
      80                 : struct Shape;
      81                 : 
      82                 : namespace gc {
      83                 : 
      84                 : enum State {
      85                 :     NO_INCREMENTAL,
      86                 :     MARK_ROOTS,
      87                 :     MARK,
      88                 :     SWEEP,
      89                 :     INVALID
      90                 : };
      91                 : 
      92                 : struct Arena;
      93                 : 
      94                 : /*
      95                 :  * This must be an upper bound, but we do not need the least upper bound, so
      96                 :  * we just exclude non-background objects.
      97                 :  */
      98                 : const size_t MAX_BACKGROUND_FINALIZE_KINDS = FINALIZE_LIMIT - FINALIZE_OBJECT_LIMIT / 2;
      99                 : 
     100                 : /*
     101                 :  * Page size is 4096 by default, except for SPARC, where it is 8192.
     102                 :  * Note: Do not use JS_CPU_SPARC here, this header is used outside JS.
     103                 :  * Bug 692267: Move page size definition to gc/Memory.h and include it
     104                 :  *             directly once jsgc.h is no longer an installed header.
     105                 :  */
     106                 : #if defined(SOLARIS) && (defined(__sparc) || defined(__sparcv9))
     107                 : const size_t PageShift = 13;
     108                 : #else
     109                 : const size_t PageShift = 12;
     110                 : #endif
     111                 : const size_t PageSize = size_t(1) << PageShift;
     112                 : 
     113                 : const size_t ChunkShift = 20;
     114                 : const size_t ChunkSize = size_t(1) << ChunkShift;
     115                 : const size_t ChunkMask = ChunkSize - 1;
     116                 : 
     117                 : const size_t ArenaShift = PageShift;
     118                 : const size_t ArenaSize = PageSize;
     119                 : const size_t ArenaMask = ArenaSize - 1;
     120                 : 
     121                 : /*
     122                 :  * This is the maximum number of arenas we allow in the FreeCommitted state
     123                 :  * before we trigger a GC_SHRINK to release free arenas to the OS.
     124                 :  */
     125                 : const static uint32_t FreeCommittedArenasThreshold = (32 << 20) / ArenaSize;
     126                 : 
     127                 : /*
     128                 :  * The mark bitmap has one bit per each GC cell. For multi-cell GC things this
     129                 :  * wastes space but allows to avoid expensive devisions by thing's size when
     130                 :  * accessing the bitmap. In addition this allows to use some bits for colored
     131                 :  * marking during the cycle GC.
     132                 :  */
     133                 : const size_t ArenaCellCount = size_t(1) << (ArenaShift - Cell::CellShift);
     134                 : const size_t ArenaBitmapBits = ArenaCellCount;
     135                 : const size_t ArenaBitmapBytes = ArenaBitmapBits / 8;
     136                 : const size_t ArenaBitmapWords = ArenaBitmapBits / JS_BITS_PER_WORD;
     137                 : 
     138                 : /*
     139                 :  * A FreeSpan represents a contiguous sequence of free cells in an Arena.
     140                 :  * |first| is the address of the first free cell in the span. |last| is the
     141                 :  * address of the last free cell in the span. This last cell holds a FreeSpan
     142                 :  * data structure for the next span unless this is the last span on the list
     143                 :  * of spans in the arena. For this last span |last| points to the last byte of
     144                 :  * the last thing in the arena and no linkage is stored there, so
     145                 :  * |last| == arenaStart + ArenaSize - 1. If the space at the arena end is
     146                 :  * fully used this last span is empty and |first| == |last + 1|.
     147                 :  *
     148                 :  * Thus |first| < |last| implies that we have either the last span with at least
     149                 :  * one element or that the span is not the last and contains at least 2
     150                 :  * elements. In both cases to allocate a thing from this span we need simply
     151                 :  * to increment |first| by the allocation size.
     152                 :  *
     153                 :  * |first| == |last| implies that we have a one element span that records the
     154                 :  * next span. So to allocate from it we need to update the span list head
     155                 :  * with a copy of the span stored at |last| address so the following
     156                 :  * allocations will use that span.
     157                 :  *
     158                 :  * |first| > |last| implies that we have an empty last span and the arena is
     159                 :  * fully used.
     160                 :  *
     161                 :  * Also only for the last span (|last| & 1)! = 0 as all allocation sizes are
     162                 :  * multiples of Cell::CellSize.
     163                 :  */
     164                 : struct FreeSpan {
     165                 :     uintptr_t   first;
     166                 :     uintptr_t   last;
     167                 : 
     168                 :   public:
     169         6814712 :     FreeSpan() {}
     170                 : 
     171        30307367 :     FreeSpan(uintptr_t first, uintptr_t last)
     172        30307367 :       : first(first), last(last) {
     173        30307367 :         checkSpan();
     174        30307335 :     }
     175                 : 
     176                 :     /*
     177                 :      * To minimize the size of the arena header the first span is encoded
     178                 :      * there as offsets from the arena start.
     179                 :      */
     180         5331337 :     static size_t encodeOffsets(size_t firstOffset, size_t lastOffset) {
     181                 :         /* Check that we can pack the offsets into uint16. */
     182                 :         JS_STATIC_ASSERT(ArenaShift < 16);
     183         5331337 :         JS_ASSERT(firstOffset <= ArenaSize);
     184         5331336 :         JS_ASSERT(lastOffset < ArenaSize);
     185         5331336 :         JS_ASSERT(firstOffset <= ((lastOffset + 1) & ~size_t(1)));
     186         5331336 :         return firstOffset | (lastOffset << 16);
     187                 :     }
     188                 : 
     189                 :     /*
     190                 :      * Encoded offsets for a full arena when its first span is the last one
     191                 :      * and empty.
     192                 :      */
     193                 :     static const size_t FullArenaOffsets = ArenaSize | ((ArenaSize - 1) << 16);
     194                 : 
     195        30307420 :     static FreeSpan decodeOffsets(uintptr_t arenaAddr, size_t offsets) {
     196        30307420 :         JS_ASSERT(!(arenaAddr & ArenaMask));
     197                 : 
     198        30307412 :         size_t firstOffset = offsets & 0xFFFF;
     199        30307412 :         size_t lastOffset = offsets >> 16;
     200        30307412 :         JS_ASSERT(firstOffset <= ArenaSize);
     201        30307412 :         JS_ASSERT(lastOffset < ArenaSize);
     202                 : 
     203                 :         /*
     204                 :          * We must not use | when calculating first as firstOffset is
     205                 :          * ArenaMask + 1 for the empty span.
     206                 :          */
     207        30307412 :         return FreeSpan(arenaAddr + firstOffset, arenaAddr | lastOffset);
     208                 :     }
     209                 : 
     210         1785856 :     void initAsEmpty(uintptr_t arenaAddr = 0) {
     211         1785856 :         JS_ASSERT(!(arenaAddr & ArenaMask));
     212         1785856 :         first = arenaAddr + ArenaSize;
     213         1785856 :         last = arenaAddr | (ArenaSize  - 1);
     214         1785856 :         JS_ASSERT(isEmpty());
     215         1785856 :     }
     216                 : 
     217        40205068 :     bool isEmpty() const {
     218        40205068 :         checkSpan();
     219        40205068 :         return first > last;
     220                 :     }
     221                 : 
     222        36429818 :     bool hasNext() const {
     223        36429818 :         checkSpan();
     224        36429812 :         return !(last & uintptr_t(1));
     225                 :     }
     226                 : 
     227        33846341 :     const FreeSpan *nextSpan() const {
     228        33846341 :         JS_ASSERT(hasNext());
     229        33846342 :         return reinterpret_cast<FreeSpan *>(last);
     230                 :     }
     231                 : 
     232        13528507 :     FreeSpan *nextSpanUnchecked(size_t thingSize) const {
     233                 : #ifdef DEBUG
     234        13528507 :         uintptr_t lastOffset = last & ArenaMask;
     235        13528507 :         JS_ASSERT(!(lastOffset & 1));
     236        13528507 :         JS_ASSERT((ArenaSize - lastOffset) % thingSize == 0);
     237                 : #endif
     238        13528507 :         return reinterpret_cast<FreeSpan *>(last);
     239                 :     }
     240                 : 
     241       812783034 :     uintptr_t arenaAddressUnchecked() const {
     242       812783034 :         return last & ~ArenaMask;
     243                 :     }
     244                 : 
     245        14307409 :     uintptr_t arenaAddress() const {
     246        14307409 :         checkSpan();
     247        14307409 :         return arenaAddressUnchecked();
     248                 :     }
     249                 : 
     250         1451057 :     ArenaHeader *arenaHeader() const {
     251         1451057 :         return reinterpret_cast<ArenaHeader *>(arenaAddress());
     252                 :     }
     253                 : 
     254         1204023 :     bool isSameNonEmptySpan(const FreeSpan *another) const {
     255         1204023 :         JS_ASSERT(!isEmpty());
     256         1204023 :         JS_ASSERT(!another->isEmpty());
     257         1204023 :         return first == another->first && last == another->last;
     258                 :     }
     259                 : 
     260         5058885 :     bool isWithinArena(uintptr_t arenaAddr) const {
     261         5058885 :         JS_ASSERT(!(arenaAddr & ArenaMask));
     262                 : 
     263                 :         /* Return true for the last empty span as well. */
     264         5058885 :         return arenaAddress() == arenaAddr;
     265                 :     }
     266                 : 
     267         5058885 :     size_t encodeAsOffsets() const {
     268                 :         /*
     269                 :          * We must use first - arenaAddress(), not first & ArenaMask as
     270                 :          * first == ArenaMask + 1 for an empty span.
     271                 :          */
     272         5058885 :         uintptr_t arenaAddr = arenaAddress();
     273         5058885 :         return encodeOffsets(first - arenaAddr, last & ArenaMask);
     274                 :     }
     275                 : 
     276                 :     /* See comments before FreeSpan for details. */
     277       270373605 :     JS_ALWAYS_INLINE void *allocate(size_t thingSize) {
     278       270373605 :         JS_ASSERT(thingSize % Cell::CellSize == 0);
     279       270373605 :         checkSpan();
     280       270373598 :         uintptr_t thing = first;
     281       270373598 :         if (thing < last) {
     282                 :             /* Bump-allocate from the current span. */
     283       266128987 :             first = thing + thingSize;
     284         4244611 :         } else if (JS_LIKELY(thing == last)) {
     285                 :             /*
     286                 :              * Move to the next span. We use JS_LIKELY as without PGO
     287                 :              * compilers mis-predict == here as unlikely to succeed.
     288                 :              */
     289         1538535 :             *this = *reinterpret_cast<FreeSpan *>(thing);
     290                 :         } else {
     291         2706076 :             return NULL;
     292                 :         }
     293       267667522 :         checkSpan();
     294       267667515 :         return reinterpret_cast<void *>(thing);
     295                 :     }
     296                 : 
     297                 :     /* A version of allocate when we know that the span is not empty. */
     298          272469 :     JS_ALWAYS_INLINE void *infallibleAllocate(size_t thingSize) {
     299          272469 :         JS_ASSERT(thingSize % Cell::CellSize == 0);
     300          272469 :         checkSpan();
     301          272469 :         uintptr_t thing = first;
     302          272469 :         if (thing < last) {
     303          167390 :             first = thing + thingSize;
     304                 :         } else {
     305          105079 :             JS_ASSERT(thing == last);
     306          105079 :             *this = *reinterpret_cast<FreeSpan *>(thing);
     307                 :         }
     308          272469 :         checkSpan();
     309          272469 :         return reinterpret_cast<void *>(thing);
     310                 :     }
     311                 : 
     312                 :     /*
     313                 :      * Allocate from a newly allocated arena. We do not move the free list
     314                 :      * from the arena. Rather we set the arena up as fully used during the
     315                 :      * initialization so to allocate we simply return the first thing in the
     316                 :      * arena and set the free list to point to the second.
     317                 :      */
     318         2067427 :     JS_ALWAYS_INLINE void *allocateFromNewArena(uintptr_t arenaAddr, size_t firstThingOffset,
     319                 :                                                 size_t thingSize) {
     320         2067427 :         JS_ASSERT(!(arenaAddr & ArenaMask));
     321         2067427 :         uintptr_t thing = arenaAddr | firstThingOffset;
     322         2067427 :         first = thing + thingSize;
     323         2067427 :         last = arenaAddr | ArenaMask;
     324         2067427 :         checkSpan();
     325         2067427 :         return reinterpret_cast<void *>(thing);
     326                 :     }
     327                 : 
     328       692673797 :     void checkSpan() const {
     329                 : #ifdef DEBUG
     330                 :         /* We do not allow spans at the end of the address space. */
     331       692673797 :         JS_ASSERT(last != uintptr_t(-1));
     332       692673636 :         JS_ASSERT(first);
     333       692673636 :         JS_ASSERT(last);
     334       692673636 :         JS_ASSERT(first - 1 <= last);
     335       692673636 :         uintptr_t arenaAddr = arenaAddressUnchecked();
     336       692681895 :         if (last & 1) {
     337                 :             /* The span is the last. */
     338       586868786 :             JS_ASSERT((last & ArenaMask) == ArenaMask);
     339                 : 
     340       586872170 :             if (first - 1 == last) {
     341                 :                 /* The span is last and empty. The above start != 0 check
     342                 :                  * implies that we are not at the end of the address space.
     343                 :                  */
     344        43332544 :                 return;
     345                 :             }
     346       543539626 :             size_t spanLength = last - first + 1;
     347       543539626 :             JS_ASSERT(spanLength % Cell::CellSize == 0);
     348                 : 
     349                 :             /* Start and end must belong to the same arena. */
     350       543539621 :             JS_ASSERT((first & ~ArenaMask) == arenaAddr);
     351       543539621 :             return;
     352                 :         }
     353                 : 
     354                 :         /* The span is not the last and we have more spans to follow. */
     355       105813109 :         JS_ASSERT(first <= last);
     356       105813077 :         size_t spanLengthWithoutOneThing = last - first;
     357       105813077 :         JS_ASSERT(spanLengthWithoutOneThing % Cell::CellSize == 0);
     358                 : 
     359       105813077 :         JS_ASSERT((first & ~ArenaMask) == arenaAddr);
     360                 : 
     361                 :         /*
     362                 :          * If there is not enough space before the arena end to allocate one
     363                 :          * more thing, then the span must be marked as the last one to avoid
     364                 :          * storing useless empty span reference.
     365                 :          */
     366       105813077 :         size_t beforeTail = ArenaSize - (last & ArenaMask);
     367       105813077 :         JS_ASSERT(beforeTail >= sizeof(FreeSpan) + Cell::CellSize);
     368                 : 
     369       105813077 :         FreeSpan *next = reinterpret_cast<FreeSpan *>(last);
     370                 : 
     371                 :         /*
     372                 :          * The GC things on the list of free spans come from one arena
     373                 :          * and the spans are linked in ascending address order with
     374                 :          * at least one non-free thing between spans.
     375                 :          */
     376       105813077 :         JS_ASSERT(last < next->first);
     377       105813077 :         JS_ASSERT(arenaAddr == next->arenaAddressUnchecked());
     378                 : 
     379       105813072 :         if (next->first > next->last) {
     380                 :             /*
     381                 :              * The next span is the empty span that terminates the list for
     382                 :              * arenas that do not have any free things at the end.
     383                 :              */
     384        19429078 :             JS_ASSERT(next->first - 1 == next->last);
     385        19429078 :             JS_ASSERT(arenaAddr + ArenaSize == next->first);
     386                 :         }
     387                 : #endif
     388                 :     }
     389                 : 
     390                 : };
     391                 : 
     392                 : /* Every arena has a header. */
     393                 : struct ArenaHeader {
     394                 :     friend struct FreeLists;
     395                 : 
     396                 :     JSCompartment   *compartment;
     397                 : 
     398                 :     /*
     399                 :      * ArenaHeader::next has two purposes: when unallocated, it points to the
     400                 :      * next available Arena's header. When allocated, it points to the next
     401                 :      * arena of the same size class and compartment.
     402                 :      */
     403                 :     ArenaHeader     *next;
     404                 : 
     405                 :   private:
     406                 :     /*
     407                 :      * The first span of free things in the arena. We encode it as the start
     408                 :      * and end offsets within the arena, not as FreeSpan structure, to
     409                 :      * minimize the header size.
     410                 :      */
     411                 :     size_t          firstFreeSpanOffsets;
     412                 : 
     413                 :     /*
     414                 :      * One of AllocKind constants or FINALIZE_LIMIT when the arena does not
     415                 :      * contain any GC things and is on the list of empty arenas in the GC
     416                 :      * chunk. The latter allows to quickly check if the arena is allocated
     417                 :      * during the conservative GC scanning without searching the arena in the
     418                 :      * list.
     419                 :      */
     420                 :     size_t       allocKind          : 8;
     421                 : 
     422                 :     /*
     423                 :      * When recursive marking uses too much stack the marking is delayed and
     424                 :      * the corresponding arenas are put into a stack using the following field
     425                 :      * as a linkage. To distinguish the bottom of the stack from the arenas
     426                 :      * not present in the stack we use an extra flag to tag arenas on the
     427                 :      * stack.
     428                 :      *
     429                 :      * Delayed marking is also used for arenas that we allocate into during an
     430                 :      * incremental GC. In this case, we intend to mark all the objects in the
     431                 :      * arena, and it's faster to do this marking in bulk.
     432                 :      *
     433                 :      * To minimize the ArenaHeader size we record the next delayed marking
     434                 :      * linkage as arenaAddress() >> ArenaShift and pack it with the allocKind
     435                 :      * field and hasDelayedMarking flag. We use 8 bits for the allocKind, not
     436                 :      * ArenaShift - 1, so the compiler can use byte-level memory instructions
     437                 :      * to access it.
     438                 :      */
     439                 :   public:
     440                 :     size_t       hasDelayedMarking  : 1;
     441                 :     size_t       allocatedDuringIncremental : 1;
     442                 :     size_t       markOverflow : 1;
     443                 :     size_t       nextDelayedMarking : JS_BITS_PER_WORD - 8 - 1 - 1 - 1;
     444                 : 
     445                 :     static void staticAsserts() {
     446                 :         /* We must be able to fit the allockind into uint8_t. */
     447                 :         JS_STATIC_ASSERT(FINALIZE_LIMIT <= 255);
     448                 : 
     449                 :         /*
     450                 :          * nextDelayedMarkingpacking assumes that ArenaShift has enough bits
     451                 :          * to cover allocKind and hasDelayedMarking.
     452                 :          */
     453                 :         JS_STATIC_ASSERT(ArenaShift >= 8 + 1 + 1 + 1);
     454                 :     }
     455                 : 
     456                 :     inline uintptr_t address() const;
     457                 :     inline Chunk *chunk() const;
     458                 : 
     459       775266408 :     bool allocated() const {
     460       775266408 :         JS_ASSERT(allocKind <= size_t(FINALIZE_LIMIT));
     461       775266408 :         return allocKind < size_t(FINALIZE_LIMIT);
     462                 :     }
     463                 : 
     464         2067427 :     void init(JSCompartment *comp, AllocKind kind) {
     465         2067427 :         JS_ASSERT(!allocated());
     466         2067427 :         JS_ASSERT(!markOverflow);
     467         2067427 :         JS_ASSERT(!allocatedDuringIncremental);
     468         2067427 :         JS_ASSERT(!hasDelayedMarking);
     469         2067427 :         compartment = comp;
     470                 : 
     471                 :         JS_STATIC_ASSERT(FINALIZE_LIMIT <= 255);
     472         2067427 :         allocKind = size_t(kind);
     473                 : 
     474                 :         /* See comments in FreeSpan::allocateFromNewArena. */
     475         2067427 :         firstFreeSpanOffsets = FreeSpan::FullArenaOffsets;
     476         2067427 :     }
     477                 : 
     478        13059560 :     void setAsNotAllocated() {
     479        13059560 :         allocKind = size_t(FINALIZE_LIMIT);
     480        13059560 :         markOverflow = 0;
     481        13059560 :         allocatedDuringIncremental = 0;
     482        13059560 :         hasDelayedMarking = 0;
     483        13059560 :         nextDelayedMarking = 0;
     484        13059560 :     }
     485                 : 
     486        44512118 :     uintptr_t arenaAddress() const {
     487        44512118 :         return address();
     488                 :     }
     489                 : 
     490         5631019 :     Arena *getArena() {
     491         5631019 :         return reinterpret_cast<Arena *>(arenaAddress());
     492                 :     }
     493                 : 
     494       481094773 :     AllocKind getAllocKind() const {
     495       481094773 :         JS_ASSERT(allocated());
     496       481094729 :         return AllocKind(allocKind);
     497                 :     }
     498                 : 
     499                 :     inline size_t getThingSize() const;
     500                 : 
     501        18375868 :     bool hasFreeThings() const {
     502        18375868 :         return firstFreeSpanOffsets != FreeSpan::FullArenaOffsets;
     503                 :     }
     504                 : 
     505                 :     inline bool isEmpty() const;
     506                 : 
     507          420185 :     void setAsFullyUsed() {
     508          420185 :         firstFreeSpanOffsets = FreeSpan::FullArenaOffsets;
     509          420185 :     }
     510                 : 
     511        16429899 :     FreeSpan getFirstFreeSpan() const {
     512                 : #ifdef DEBUG
     513        16429899 :         checkSynchronizedWithFreeList();
     514                 : #endif
     515        16429845 :         return FreeSpan::decodeOffsets(arenaAddress(), firstFreeSpanOffsets);
     516                 :     }
     517                 : 
     518         5058885 :     void setFirstFreeSpan(const FreeSpan *span) {
     519         5058885 :         JS_ASSERT(span->isWithinArena(arenaAddress()));
     520         5058885 :         firstFreeSpanOffsets = span->encodeAsOffsets();
     521         5058885 :     }
     522                 : 
     523                 : #ifdef DEBUG
     524                 :     void checkSynchronizedWithFreeList() const;
     525                 : #endif
     526                 : 
     527                 :     inline ArenaHeader *getNextDelayedMarking() const;
     528                 :     inline void setNextDelayedMarking(ArenaHeader *aheader);
     529                 : };
     530                 : 
     531                 : struct Arena {
     532                 :     /*
     533                 :      * Layout of an arena:
     534                 :      * An arena is 4K in size and 4K-aligned. It starts with the ArenaHeader
     535                 :      * descriptor followed by some pad bytes. The remainder of the arena is
     536                 :      * filled with the array of T things. The pad bytes ensure that the thing
     537                 :      * array ends exactly at the end of the arena.
     538                 :      *
     539                 :      * +-------------+-----+----+----+-----+----+
     540                 :      * | ArenaHeader | pad | T0 | T1 | ... | Tn |
     541                 :      * +-------------+-----+----+----+-----+----+
     542                 :      *
     543                 :      * <----------------------------------------> = ArenaSize bytes
     544                 :      * <-------------------> = first thing offset
     545                 :      */
     546                 :     ArenaHeader aheader;
     547                 :     uint8_t     data[ArenaSize - sizeof(ArenaHeader)];
     548                 : 
     549                 :   private:
     550                 :     static JS_FRIEND_DATA(const uint32_t) ThingSizes[];
     551                 :     static JS_FRIEND_DATA(const uint32_t) FirstThingOffsets[];
     552                 : 
     553                 :   public:
     554                 :     static void staticAsserts();
     555                 : 
     556       528541064 :     static size_t thingSize(AllocKind kind) {
     557       528541064 :         return ThingSizes[kind];
     558                 :     }
     559                 : 
     560        35132149 :     static size_t firstThingOffset(AllocKind kind) {
     561        35132149 :         return FirstThingOffsets[kind];
     562                 :     }
     563                 : 
     564        17408641 :     static size_t thingsPerArena(size_t thingSize) {
     565        17408641 :         JS_ASSERT(thingSize % Cell::CellSize == 0);
     566                 : 
     567                 :         /* We should be able to fit FreeSpan in any GC thing. */
     568        17408641 :         JS_ASSERT(thingSize >= sizeof(FreeSpan));
     569                 : 
     570        17408641 :         return (ArenaSize - sizeof(ArenaHeader)) / thingSize;
     571                 :     }
     572                 : 
     573            1089 :     static size_t thingsSpan(size_t thingSize) {
     574            1089 :         return thingsPerArena(thingSize) * thingSize;
     575                 :     }
     576                 : 
     577       502344013 :     static bool isAligned(uintptr_t thing, size_t thingSize) {
     578                 :         /* Things ends at the arena end. */
     579       502344013 :         uintptr_t tailOffset = (ArenaSize - thing) & ArenaMask;
     580       502344013 :         return tailOffset % thingSize == 0;
     581                 :     }
     582                 : 
     583        29525194 :     uintptr_t address() const {
     584        29525194 :         return aheader.address();
     585                 :     }
     586                 : 
     587        20821259 :     uintptr_t thingsStart(AllocKind thingKind) {
     588        20821259 :         return address() | firstThingOffset(thingKind);
     589                 :     }
     590                 : 
     591         8703935 :     uintptr_t thingsEnd() {
     592         8703935 :         return address() + ArenaSize;
     593                 :     }
     594                 : 
     595                 :     template <typename T>
     596                 :     bool finalize(JSContext *cx, AllocKind thingKind, size_t thingSize, bool background);
     597                 : };
     598                 : 
     599                 : /* The chunk header (located at the end of the chunk to preserve arena alignment). */
     600                 : struct ChunkInfo {
     601                 :     Chunk           *next;
     602                 :     Chunk           **prevp;
     603                 : 
     604                 :     /* Free arenas are linked together with aheader.next. */
     605                 :     ArenaHeader     *freeArenasHead;
     606                 : 
     607                 :     /*
     608                 :      * Decommitted arenas are tracked by a bitmap in the chunk header. We use
     609                 :      * this offset to start our search iteration close to a decommitted arena
     610                 :      * that we can allocate.
     611                 :      */
     612                 :     uint32_t        lastDecommittedArenaOffset;
     613                 : 
     614                 :     /* Number of free arenas, either committed or decommitted. */
     615                 :     uint32_t        numArenasFree;
     616                 : 
     617                 :     /* Number of free, committed arenas. */
     618                 :     uint32_t        numArenasFreeCommitted;
     619                 : 
     620                 :     /* Number of GC cycles this chunk has survived. */
     621                 :     uint32_t        age;
     622                 : };
     623                 : 
     624                 : /*
     625                 :  * Calculating ArenasPerChunk:
     626                 :  *
     627                 :  * In order to figure out how many Arenas will fit in a chunk, we need to know
     628                 :  * how much extra space is available after we allocate the header data. This
     629                 :  * is a problem because the header size depends on the number of arenas in the
     630                 :  * chunk. The two dependent fields are bitmap and decommittedArenas.
     631                 :  *
     632                 :  * For the mark bitmap, we know that each arena will use a fixed number of full
     633                 :  * bytes: ArenaBitmapBytes. The full size of the header data is this number
     634                 :  * multiplied by the eventual number of arenas we have in the header. We,
     635                 :  * conceptually, distribute this header data among the individual arenas and do
     636                 :  * not include it in the header. This way we do not have to worry about its
     637                 :  * variable size: it gets attached to the variable number we are computing.
     638                 :  *
     639                 :  * For the decommitted arena bitmap, we only have 1 bit per arena, so this
     640                 :  * technique will not work. Instead, we observe that we do not have enough
     641                 :  * header info to fill 8 full arenas: it is currently 4 on 64bit, less on
     642                 :  * 32bit. Thus, with current numbers, we need 64 bytes for decommittedArenas.
     643                 :  * This will not become 63 bytes unless we double the data required in the
     644                 :  * header. Therefore, we just compute the number of bytes required to track
     645                 :  * every possible arena and do not worry about slop bits, since there are too
     646                 :  * few to usefully allocate.
     647                 :  *
     648                 :  * To actually compute the number of arenas we can allocate in a chunk, we
     649                 :  * divide the amount of available space less the header info (not including
     650                 :  * the mark bitmap which is distributed into the arena size) by the size of
     651                 :  * the arena (with the mark bitmap bytes it uses).
     652                 :  */
     653                 : const size_t BytesPerArenaWithHeader = ArenaSize + ArenaBitmapBytes;
     654                 : const size_t ChunkDecommitBitmapBytes = ChunkSize / ArenaSize / JS_BITS_PER_BYTE;
     655                 : const size_t ChunkBytesAvailable = ChunkSize - sizeof(ChunkInfo) - ChunkDecommitBitmapBytes;
     656                 : const size_t ArenasPerChunk = ChunkBytesAvailable / BytesPerArenaWithHeader;
     657                 : 
     658                 : /* A chunk bitmap contains enough mark bits for all the cells in a chunk. */
     659                 : struct ChunkBitmap {
     660                 :     uintptr_t bitmap[ArenaBitmapWords * ArenasPerChunk];
     661                 : 
     662                 :     JS_ALWAYS_INLINE void getMarkWordAndMask(const Cell *cell, uint32_t color,
     663                 :                                              uintptr_t **wordp, uintptr_t *maskp);
     664                 : 
     665      1191989300 :     JS_ALWAYS_INLINE bool isMarked(const Cell *cell, uint32_t color) {
     666                 :         uintptr_t *word, mask;
     667      1191989300 :         getMarkWordAndMask(cell, color, &word, &mask);
     668      1191970148 :         return *word & mask;
     669                 :     }
     670                 : 
     671       935030078 :     JS_ALWAYS_INLINE bool markIfUnmarked(const Cell *cell, uint32_t color) {
     672                 :         uintptr_t *word, mask;
     673       935030078 :         getMarkWordAndMask(cell, BLACK, &word, &mask);
     674       935030078 :         if (*word & mask)
     675       439455202 :             return false;
     676       495574876 :         *word |= mask;
     677       495574876 :         if (color != BLACK) {
     678                 :             /*
     679                 :              * We use getMarkWordAndMask to recalculate both mask and word as
     680                 :              * doing just mask << color may overflow the mask.
     681                 :              */
     682        12148726 :             getMarkWordAndMask(cell, color, &word, &mask);
     683        12148726 :             if (*word & mask)
     684               0 :                 return false;
     685        12148726 :             *word |= mask;
     686                 :         }
     687       495574876 :         return true;
     688                 :     }
     689                 : 
     690                 :     JS_ALWAYS_INLINE void unmark(const Cell *cell, uint32_t color) {
     691                 :         uintptr_t *word, mask;
     692                 :         getMarkWordAndMask(cell, color, &word, &mask);
     693                 :         *word &= ~mask;
     694                 :     }
     695                 : 
     696          157935 :     void clear() {
     697          157935 :         PodArrayZero(bitmap);
     698          157935 :     }
     699                 : 
     700                 : #ifdef DEBUG
     701            2153 :     bool noBitsSet(ArenaHeader *aheader) {
     702                 :         /*
     703                 :          * We assume that the part of the bitmap corresponding to the arena
     704                 :          * has the exact number of words so we do not need to deal with a word
     705                 :          * that covers bits from two arenas.
     706                 :          */
     707                 :         JS_STATIC_ASSERT(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD);
     708                 : 
     709                 :         uintptr_t *word, unused;
     710            2153 :         getMarkWordAndMask(reinterpret_cast<Cell *>(aheader->address()), BLACK, &word, &unused);
     711           36601 :         for (size_t i = 0; i != ArenaBitmapWords; i++) {
     712           34448 :             if (word[i])
     713               0 :                 return false;
     714                 :         }
     715            2153 :         return true;
     716                 :     }
     717                 : #endif
     718                 : };
     719                 : 
     720                 : JS_STATIC_ASSERT(ArenaBitmapBytes * ArenasPerChunk == sizeof(ChunkBitmap));
     721                 : 
     722                 : typedef BitArray<ArenasPerChunk> PerArenaBitmap;
     723                 : 
     724                 : const size_t ChunkPadSize = ChunkSize
     725                 :                             - (sizeof(Arena) * ArenasPerChunk)
     726                 :                             - sizeof(ChunkBitmap)
     727                 :                             - sizeof(PerArenaBitmap)
     728                 :                             - sizeof(ChunkInfo);
     729                 : JS_STATIC_ASSERT(ChunkPadSize < BytesPerArenaWithHeader);
     730                 : 
     731                 : /*
     732                 :  * Chunks contain arenas and associated data structures (mark bitmap, delayed
     733                 :  * marking state).
     734                 :  */
     735                 : struct Chunk {
     736                 :     Arena           arenas[ArenasPerChunk];
     737                 : 
     738                 :     /* Pad to full size to ensure cache alignment of ChunkInfo. */
     739                 :     uint8_t         padding[ChunkPadSize];
     740                 : 
     741                 :     ChunkBitmap     bitmap;
     742                 :     PerArenaBitmap  decommittedArenas;
     743                 :     ChunkInfo       info;
     744                 : 
     745       217009915 :     static Chunk *fromAddress(uintptr_t addr) {
     746       217009915 :         addr &= ~ChunkMask;
     747       217009915 :         return reinterpret_cast<Chunk *>(addr);
     748                 :     }
     749                 : 
     750              -1 :     static bool withinArenasRange(uintptr_t addr) {
     751              -1 :         uintptr_t offset = addr & ChunkMask;
     752              -1 :         return offset < ArenasPerChunk * ArenaSize;
     753                 :     }
     754                 : 
     755        11626109 :     static size_t arenaIndex(uintptr_t addr) {
     756        11626109 :         JS_ASSERT(withinArenasRange(addr));
     757        11626109 :         return (addr & ChunkMask) >> ArenaShift;
     758                 :     }
     759                 : 
     760                 :     uintptr_t address() const {
     761                 :         uintptr_t addr = reinterpret_cast<uintptr_t>(this);
     762                 :         JS_ASSERT(!(addr & ChunkMask));
     763                 :         return addr;
     764                 :     }
     765                 : 
     766         2159685 :     bool unused() const {
     767         2159685 :         return info.numArenasFree == ArenasPerChunk;
     768                 :     }
     769                 : 
     770         4190858 :     bool hasAvailableArenas() const {
     771         4190858 :         return info.numArenasFree != 0;
     772                 :     }
     773                 : 
     774                 :     inline void addToAvailableList(JSCompartment *compartment);
     775                 :     inline void insertToAvailableList(Chunk **insertPoint);
     776                 :     inline void removeFromAvailableList();
     777                 : 
     778                 :     ArenaHeader *allocateArena(JSCompartment *comp, AllocKind kind);
     779                 : 
     780                 :     void releaseArena(ArenaHeader *aheader);
     781                 : 
     782                 :     static Chunk *allocate(JSRuntime *rt);
     783                 : 
     784                 :     /* Must be called with the GC lock taken. */
     785                 :     static inline void release(JSRuntime *rt, Chunk *chunk);
     786                 :     static inline void releaseList(JSRuntime *rt, Chunk *chunkListHead);
     787                 : 
     788                 :     /* Must be called with the GC lock taken. */
     789                 :     inline void prepareToBeFreed(JSRuntime *rt);
     790                 : 
     791                 :     /*
     792                 :      * Assuming that the info.prevp points to the next field of the previous
     793                 :      * chunk in a doubly-linked list, get that chunk.
     794                 :      */
     795               0 :     Chunk *getPrevious() {
     796               0 :         JS_ASSERT(info.prevp);
     797               0 :         return fromPointerToNext(info.prevp);
     798                 :     }
     799                 : 
     800                 :     /* Get the chunk from a pointer to its info.next field. */
     801               0 :     static Chunk *fromPointerToNext(Chunk **nextFieldPtr) {
     802               0 :         uintptr_t addr = reinterpret_cast<uintptr_t>(nextFieldPtr);
     803               0 :         JS_ASSERT((addr & ChunkMask) == offsetof(Chunk, info.next));
     804               0 :         return reinterpret_cast<Chunk *>(addr - offsetof(Chunk, info.next));
     805                 :     }
     806                 : 
     807                 :   private:
     808                 :     inline void init();
     809                 : 
     810                 :     /* Search for a decommitted arena to allocate. */
     811                 :     unsigned findDecommittedArenaOffset();
     812                 :     ArenaHeader* fetchNextDecommittedArena();
     813                 : 
     814                 :   public:
     815                 :     /* Unlink and return the freeArenasHead. */
     816                 :     inline ArenaHeader* fetchNextFreeArena(JSRuntime *rt);
     817                 : 
     818                 :     inline void addArenaToFreeList(JSRuntime *rt, ArenaHeader *aheader);
     819                 : };
     820                 : 
     821                 : JS_STATIC_ASSERT(sizeof(Chunk) == ChunkSize);
     822                 : 
     823                 : class ChunkPool {
     824                 :     Chunk   *emptyChunkListHead;
     825                 :     size_t  emptyCount;
     826                 : 
     827                 :   public:
     828           19910 :     ChunkPool()
     829                 :       : emptyChunkListHead(NULL),
     830           19910 :         emptyCount(0) { }
     831                 : 
     832              12 :     size_t getEmptyCount() const {
     833              12 :         return emptyCount;
     834                 :     }
     835                 : 
     836                 :     inline bool wantBackgroundAllocation(JSRuntime *rt) const;
     837                 : 
     838                 :     /* Must be called with the GC lock taken. */
     839                 :     inline Chunk *get(JSRuntime *rt);
     840                 : 
     841                 :     /* Must be called either during the GC or with the GC lock taken. */
     842                 :     inline void put(Chunk *chunk);
     843                 : 
     844                 :     /*
     845                 :      * Return the list of chunks that can be released outside the GC lock.
     846                 :      * Must be called either during the GC or with the GC lock taken.
     847                 :      */
     848                 :     Chunk *expire(JSRuntime *rt, bool releaseAll);
     849                 : 
     850                 :     /* Must be called with the GC lock taken. */
     851                 :     void expireAndFree(JSRuntime *rt, bool releaseAll);
     852                 : 
     853                 :     /* Must be called either during the GC or with the GC lock taken. */
     854                 :     JS_FRIEND_API(int64_t) countCleanDecommittedArenas(JSRuntime *rt);
     855                 : };
     856                 : 
     857                 : inline uintptr_t
     858              -1 : Cell::address() const
     859                 : {
     860              -1 :     uintptr_t addr = uintptr_t(this);
     861              -1 :     JS_ASSERT(addr % Cell::CellSize == 0);
     862              -1 :     JS_ASSERT(Chunk::withinArenasRange(addr));
     863              -1 :     return addr;
     864                 : }
     865                 : 
     866                 : inline ArenaHeader *
     867      -965361959 : Cell::arenaHeader() const
     868                 : {
     869      -965361959 :     uintptr_t addr = address();
     870      -965600691 :     addr &= ~ArenaMask;
     871      -965600691 :     return reinterpret_cast<ArenaHeader *>(addr);
     872                 : }
     873                 : 
     874                 : Chunk *
     875       923784894 : Cell::chunk() const
     876                 : {
     877       923784894 :     uintptr_t addr = uintptr_t(this);
     878       923784894 :     JS_ASSERT(addr % Cell::CellSize == 0);
     879       923784894 :     addr &= ~(ChunkSize - 1);
     880       923784894 :     return reinterpret_cast<Chunk *>(addr);
     881                 : }
     882                 : 
     883                 : AllocKind
     884       353805402 : Cell::getAllocKind() const
     885                 : {
     886       353805402 :     return arenaHeader()->getAllocKind();
     887                 : }
     888                 : 
     889                 : #ifdef DEBUG
     890                 : inline bool
     891       459700422 : Cell::isAligned() const
     892                 : {
     893       459700422 :     return Arena::isAligned(address(), arenaHeader()->getThingSize());
     894                 : }
     895                 : #endif
     896                 : 
     897                 : inline uintptr_t
     898        76108787 : ArenaHeader::address() const
     899                 : {
     900        76108787 :     uintptr_t addr = reinterpret_cast<uintptr_t>(this);
     901        76108787 :     JS_ASSERT(!(addr & ArenaMask));
     902        76108787 :     JS_ASSERT(Chunk::withinArenasRange(addr));
     903        76108625 :     return addr;
     904                 : }
     905                 : 
     906                 : inline Chunk *
     907         2069473 : ArenaHeader::chunk() const
     908                 : {
     909         2069473 :     return Chunk::fromAddress(address());
     910                 : }
     911                 : 
     912                 : inline bool
     913          272469 : ArenaHeader::isEmpty() const
     914                 : {
     915                 :     /* Arena is empty if its first span covers the whole arena. */
     916          272469 :     JS_ASSERT(allocated());
     917          272469 :     size_t firstThingOffset = Arena::firstThingOffset(getAllocKind());
     918          272469 :     return firstFreeSpanOffsets == FreeSpan::encodeOffsets(firstThingOffset, ArenaMask);
     919                 : }
     920                 : 
     921                 : inline size_t
     922       264325816 : ArenaHeader::getThingSize() const
     923                 : {
     924       264325816 :     JS_ASSERT(allocated());
     925       264325787 :     return Arena::thingSize(getAllocKind());
     926                 : }
     927                 : 
     928                 : inline ArenaHeader *
     929           52010 : ArenaHeader::getNextDelayedMarking() const
     930                 : {
     931           52010 :     return &reinterpret_cast<Arena *>(nextDelayedMarking << ArenaShift)->aheader;
     932                 : }
     933                 : 
     934                 : inline void
     935           52010 : ArenaHeader::setNextDelayedMarking(ArenaHeader *aheader)
     936                 : {
     937           52010 :     JS_ASSERT(!(uintptr_t(aheader) & ArenaMask));
     938           52010 :     hasDelayedMarking = 1;
     939           52010 :     nextDelayedMarking = aheader->arenaAddress() >> ArenaShift;
     940           52010 : }
     941                 : 
     942                 : JS_ALWAYS_INLINE void
     943       923787047 : ChunkBitmap::getMarkWordAndMask(const Cell *cell, uint32_t color,
     944                 :                                 uintptr_t **wordp, uintptr_t *maskp)
     945                 : {
     946       923787047 :     size_t bit = (cell->address() & ChunkMask) / Cell::CellSize + color;
     947       923787047 :     JS_ASSERT(bit < ArenaBitmapBits * ArenasPerChunk);
     948       923787047 :     *maskp = uintptr_t(1) << (bit % JS_BITS_PER_WORD);
     949       923787047 :     *wordp = &bitmap[bit / JS_BITS_PER_WORD];
     950       923787047 : }
     951                 : 
     952                 : static void
     953      2127093528 : AssertValidColor(const void *thing, uint32_t color)
     954                 : {
     955                 : #ifdef DEBUG
     956      2127093528 :     ArenaHeader *aheader = reinterpret_cast<const js::gc::Cell *>(thing)->arenaHeader();
     957      2126994586 :     JS_ASSERT_IF(color, color < aheader->getThingSize() / Cell::CellSize);
     958                 : #endif
     959      2126994586 : }
     960                 : 
     961                 : inline bool
     962      1192060736 : Cell::isMarked(uint32_t color) const
     963                 : {
     964      1192060736 :     AssertValidColor(this, color);
     965      1192013176 :     return chunk()->bitmap.isMarked(this, color);
     966                 : }
     967                 : 
     968                 : bool
     969       935030078 : Cell::markIfUnmarked(uint32_t color) const
     970                 : {
     971       935030078 :     AssertValidColor(this, color);
     972       935030078 :     return chunk()->bitmap.markIfUnmarked(this, color);
     973                 : }
     974                 : 
     975                 : void
     976                 : Cell::unmark(uint32_t color) const
     977                 : {
     978                 :     JS_ASSERT(color != BLACK);
     979                 :     AssertValidColor(this, color);
     980                 :     chunk()->bitmap.unmark(this, color);
     981                 : }
     982                 : 
     983                 : JSCompartment *
     984      1950543725 : Cell::compartment() const
     985                 : {
     986      1950543725 :     return arenaHeader()->compartment;
     987                 : }
     988                 : 
     989                 : static inline JSGCTraceKind
     990       101369472 : MapAllocToTraceKind(AllocKind thingKind)
     991                 : {
     992                 :     static const JSGCTraceKind map[FINALIZE_LIMIT] = {
     993                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT0 */
     994                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT0_BACKGROUND */
     995                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT2 */
     996                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT2_BACKGROUND */
     997                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT4 */
     998                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT4_BACKGROUND */
     999                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT8 */
    1000                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT8_BACKGROUND */
    1001                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT12 */
    1002                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT12_BACKGROUND */
    1003                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT16 */
    1004                 :         JSTRACE_OBJECT,     /* FINALIZE_OBJECT16_BACKGROUND */
    1005                 :         JSTRACE_SCRIPT,     /* FINALIZE_SCRIPT */
    1006                 :         JSTRACE_SHAPE,      /* FINALIZE_SHAPE */
    1007                 :         JSTRACE_BASE_SHAPE, /* FINALIZE_BASE_SHAPE */
    1008                 :         JSTRACE_TYPE_OBJECT,/* FINALIZE_TYPE_OBJECT */
    1009                 : #if JS_HAS_XML_SUPPORT      /* FINALIZE_XML */
    1010                 :         JSTRACE_XML,
    1011                 : #endif
    1012                 :         JSTRACE_STRING,     /* FINALIZE_SHORT_STRING */
    1013                 :         JSTRACE_STRING,     /* FINALIZE_STRING */
    1014                 :         JSTRACE_STRING,     /* FINALIZE_EXTERNAL_STRING */
    1015                 :     };
    1016       101369472 :     return map[thingKind];
    1017                 : }
    1018                 : 
    1019                 : inline JSGCTraceKind
    1020                 : GetGCThingTraceKind(const void *thing);
    1021                 : 
    1022                 : struct ArenaLists {
    1023                 : 
    1024                 :     /*
    1025                 :      * ArenaList::head points to the start of the list. Normally cursor points
    1026                 :      * to the first arena in the list with some free things and all arenas
    1027                 :      * before cursor are fully allocated. However, as the arena currently being
    1028                 :      * allocated from is considered full while its list of free spans is moved
    1029                 :      * into the freeList, during the GC or cell enumeration, when an
    1030                 :      * unallocated freeList is moved back to the arena, we can see an arena
    1031                 :      * with some free cells before the cursor. The cursor is an indirect
    1032                 :      * pointer to allow for efficient list insertion at the cursor point and
    1033                 :      * other list manipulations.
    1034                 :      */
    1035                 :     struct ArenaList {
    1036                 :         ArenaHeader     *head;
    1037                 :         ArenaHeader     **cursor;
    1038                 : 
    1039         2995653 :         ArenaList() {
    1040         2995653 :             clear();
    1041         2995653 :         }
    1042                 : 
    1043         3232982 :         void clear() {
    1044         3232982 :             head = NULL;
    1045         3232982 :             cursor = &head;
    1046         3232982 :         }
    1047                 :     };
    1048                 : 
    1049                 :   private:
    1050                 :     /*
    1051                 :      * For each arena kind its free list is represented as the first span with
    1052                 :      * free things. Initially all the spans are initialized as empty. After we
    1053                 :      * find a new arena with available things we move its first free span into
    1054                 :      * the list and set the arena as fully allocated. way we do not need to
    1055                 :      * update the arena header after the initial allocation. When starting the
    1056                 :      * GC we only move the head of the of the list of spans back to the arena
    1057                 :      * only for the arena that was not fully allocated.
    1058                 :      */
    1059                 :     FreeSpan       freeLists[FINALIZE_LIMIT];
    1060                 : 
    1061                 :     ArenaList      arenaLists[FINALIZE_LIMIT];
    1062                 : 
    1063                 : #ifdef JS_THREADSAFE
    1064                 :     /*
    1065                 :      * The background finalization adds the finalized arenas to the list at
    1066                 :      * the *cursor position. backgroundFinalizeState controls the interaction
    1067                 :      * between the GC lock and the access to the list from the allocation
    1068                 :      * thread.
    1069                 :      *
    1070                 :      * BFS_DONE indicates that the finalizations is not running or cannot
    1071                 :      * affect this arena list. The allocation thread can access the list
    1072                 :      * outside the GC lock.
    1073                 :      *
    1074                 :      * In BFS_RUN and BFS_JUST_FINISHED the allocation thread must take the
    1075                 :      * lock. The former indicates that the finalization still runs. The latter
    1076                 :      * signals that finalization just added to the list finalized arenas. In
    1077                 :      * that case the lock effectively serves as a read barrier to ensure that
    1078                 :      * the allocation thread see all the writes done during finalization.
    1079                 :      */
    1080                 :     enum BackgroundFinalizeState {
    1081                 :         BFS_DONE,
    1082                 :         BFS_RUN,
    1083                 :         BFS_JUST_FINISHED
    1084                 :     };
    1085                 : 
    1086                 :     volatile uintptr_t backgroundFinalizeState[FINALIZE_LIMIT];
    1087                 : #endif
    1088                 : 
    1089                 :   public:
    1090           45576 :     ArenaLists() {
    1091          957096 :         for (size_t i = 0; i != FINALIZE_LIMIT; ++i)
    1092          911520 :             freeLists[i].initAsEmpty();
    1093                 : #ifdef JS_THREADSAFE
    1094          957096 :         for (size_t i = 0; i != FINALIZE_LIMIT; ++i)
    1095          911520 :             backgroundFinalizeState[i] = BFS_DONE;
    1096                 : #endif
    1097           45576 :     }
    1098                 : 
    1099           45571 :     ~ArenaLists() {
    1100         1913982 :         for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
    1101                 : #ifdef JS_THREADSAFE
    1102                 :             /*
    1103                 :              * We can only call this during the shutdown after the last GC when
    1104                 :              * the background finalization is disabled.
    1105                 :              */
    1106          911420 :             JS_ASSERT(backgroundFinalizeState[i] == BFS_DONE);
    1107                 : #endif
    1108          911420 :             ArenaHeader **headp = &arenaLists[i].head;
    1109         1714314 :             while (ArenaHeader *aheader = *headp) {
    1110          401447 :                 *headp = aheader->next;
    1111          401447 :                 aheader->chunk()->releaseArena(aheader);
    1112                 :             }
    1113                 :         }
    1114           45571 :     }
    1115                 : 
    1116        11389367 :     const FreeSpan *getFreeList(AllocKind thingKind) const {
    1117        11389367 :         return &freeLists[thingKind];
    1118                 :     }
    1119                 : 
    1120          480896 :     ArenaHeader *getFirstArena(AllocKind thingKind) const {
    1121          480896 :         return arenaLists[thingKind].head;
    1122                 :     }
    1123                 : 
    1124           71386 :     bool arenaListsAreEmpty() const {
    1125          607802 :         for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
    1126                 : #ifdef JS_THREADSAFE
    1127                 :             /*
    1128                 :              * The arena cannot be empty if the background finalization is not yet
    1129                 :              * done.
    1130                 :              */
    1131          582149 :             if (backgroundFinalizeState[i] != BFS_DONE)
    1132           23328 :                 return false;
    1133                 : #endif
    1134          558821 :             if (arenaLists[i].head)
    1135           22405 :                 return false;
    1136                 :         }
    1137           25653 :         return true;
    1138                 :     }
    1139                 : 
    1140                 : #ifdef DEBUG
    1141             108 :     bool checkArenaListAllUnmarked() const {
    1142            2268 :         for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
    1143                 : # ifdef JS_THREADSAFE
    1144                 :             /* The background finalization must have stopped at this point. */
    1145            2339 :             JS_ASSERT(backgroundFinalizeState[i] == BFS_DONE ||
    1146            2339 :                       backgroundFinalizeState[i] == BFS_JUST_FINISHED);
    1147                 : # endif
    1148            4313 :             for (ArenaHeader *aheader = arenaLists[i].head; aheader; aheader = aheader->next) {
    1149            2153 :                 if (!aheader->chunk()->bitmap.noBitsSet(aheader))
    1150               0 :                     return false;
    1151                 :             }
    1152                 :         }
    1153             108 :         return true;
    1154                 :     }
    1155                 : #endif
    1156                 : 
    1157                 : #ifdef JS_THREADSAFE
    1158                 :     bool doneBackgroundFinalize(AllocKind kind) const {
    1159                 :         return backgroundFinalizeState[kind] == BFS_DONE;
    1160                 :     }
    1161                 : #endif
    1162                 : 
    1163                 :     /*
    1164                 :      * Return the free list back to the arena so the GC finalization will not
    1165                 :      * run the finalizers over unitialized bytes from free things.
    1166                 :      */
    1167          122478 :     void purge() {
    1168         2572038 :         for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
    1169         2449560 :             FreeSpan *headSpan = &freeLists[i];
    1170         2449560 :             if (!headSpan->isEmpty()) {
    1171          516062 :                 ArenaHeader *aheader = headSpan->arenaHeader();
    1172          516062 :                 aheader->setFirstFreeSpan(headSpan);
    1173          516062 :                 headSpan->initAsEmpty();
    1174                 :             }
    1175                 :         }
    1176          122478 :     }
    1177                 : 
    1178                 :     inline void prepareForIncrementalGC(JSRuntime *rt);
    1179                 : 
    1180                 :     /*
    1181                 :      * Temporarily copy the free list heads to the arenas so the code can see
    1182                 :      * the proper value in ArenaHeader::freeList when accessing the latter
    1183                 :      * outside the GC.
    1184                 :      */
    1185          134482 :     void copyFreeListsToArenas() {
    1186         2824122 :         for (size_t i = 0; i != FINALIZE_LIMIT; ++i)
    1187         2689640 :             copyFreeListToArena(AllocKind(i));
    1188          134482 :     }
    1189                 : 
    1190         2721625 :     void copyFreeListToArena(AllocKind thingKind) {
    1191         2721625 :         FreeSpan *headSpan = &freeLists[thingKind];
    1192         2721625 :         if (!headSpan->isEmpty()) {
    1193          663778 :             ArenaHeader *aheader = headSpan->arenaHeader();
    1194          663778 :             JS_ASSERT(!aheader->hasFreeThings());
    1195          663778 :             aheader->setFirstFreeSpan(headSpan);
    1196                 :         }
    1197         2721625 :     }
    1198                 : 
    1199                 :     /*
    1200                 :      * Clear the free lists in arenas that were temporarily set there using
    1201                 :      * copyToArenas.
    1202                 :      */
    1203          108819 :     void clearFreeListsInArenas() {
    1204         2285199 :         for (size_t i = 0; i != FINALIZE_LIMIT; ++i)
    1205         2176380 :             clearFreeListInArena(AllocKind(i));
    1206          108819 :     }
    1207                 : 
    1208                 : 
    1209         2208365 :     void clearFreeListInArena(AllocKind kind) {
    1210         2208365 :         FreeSpan *headSpan = &freeLists[kind];
    1211         2208365 :         if (!headSpan->isEmpty()) {
    1212          147716 :             ArenaHeader *aheader = headSpan->arenaHeader();
    1213          147716 :             JS_ASSERT(aheader->getFirstFreeSpan().isSameNonEmptySpan(headSpan));
    1214          147716 :             aheader->setAsFullyUsed();
    1215                 :         }
    1216         2208365 :     }
    1217                 : 
    1218                 :     /*
    1219                 :      * Check that the free list is either empty or were synchronized with the
    1220                 :      * arena using copyToArena().
    1221                 :      */
    1222          425066 :     bool isSynchronizedFreeList(AllocKind kind) {
    1223          425066 :         FreeSpan *headSpan = &freeLists[kind];
    1224          425066 :         if (headSpan->isEmpty())
    1225          353373 :             return true;
    1226           71693 :         ArenaHeader *aheader = headSpan->arenaHeader();
    1227           71693 :         if (aheader->hasFreeThings()) {
    1228                 :             /*
    1229                 :              * If the arena has a free list, it must be the same as one in
    1230                 :              * lists.
    1231                 :              */
    1232           39708 :             JS_ASSERT(aheader->getFirstFreeSpan().isSameNonEmptySpan(headSpan));
    1233           39708 :             return true;
    1234                 :         }
    1235           31985 :         return false;
    1236                 :     }
    1237                 : 
    1238       270373604 :     JS_ALWAYS_INLINE void *allocateFromFreeList(AllocKind thingKind, size_t thingSize) {
    1239       270373604 :         return freeLists[thingKind].allocate(thingSize);
    1240                 :     }
    1241                 : 
    1242                 :     static void *refillFreeList(JSContext *cx, AllocKind thingKind);
    1243                 : 
    1244           25663 :     void checkEmptyFreeLists() {
    1245                 : #ifdef DEBUG
    1246          538923 :         for (size_t i = 0; i < mozilla::ArrayLength(freeLists); ++i)
    1247          513260 :             JS_ASSERT(freeLists[i].isEmpty());
    1248                 : #endif
    1249           25663 :     }
    1250                 : 
    1251                 :     void checkEmptyFreeList(AllocKind kind) {
    1252                 :         JS_ASSERT(freeLists[kind].isEmpty());
    1253                 :     }
    1254                 : 
    1255                 :     void finalizeObjects(JSContext *cx);
    1256                 :     void finalizeStrings(JSContext *cx);
    1257                 :     void finalizeShapes(JSContext *cx);
    1258                 :     void finalizeScripts(JSContext *cx);
    1259                 : 
    1260                 : #ifdef JS_THREADSAFE
    1261                 :     static void backgroundFinalize(JSContext *cx, ArenaHeader *listHead);
    1262                 : #endif
    1263                 : 
    1264                 :   private:
    1265                 :     inline void finalizeNow(JSContext *cx, AllocKind thingKind);
    1266                 :     inline void finalizeLater(JSContext *cx, AllocKind thingKind);
    1267                 : 
    1268                 :     inline void *allocateFromArena(JSCompartment *comp, AllocKind thingKind);
    1269                 : };
    1270                 : 
    1271                 : /*
    1272                 :  * Initial allocation size for data structures holding chunks is set to hold
    1273                 :  * chunks with total capacity of 16MB to avoid buffer resizes during browser
    1274                 :  * startup.
    1275                 :  */
    1276                 : const size_t INITIAL_CHUNK_CAPACITY = 16 * 1024 * 1024 / ChunkSize;
    1277                 : 
    1278                 : /* The number of GC cycles an empty chunk can survive before been released. */
    1279                 : const size_t MAX_EMPTY_CHUNK_AGE = 4;
    1280                 : 
    1281                 : inline Cell *
    1282                 : AsCell(JSObject *obj)
    1283                 : {
    1284                 :     return reinterpret_cast<Cell *>(obj);
    1285                 : }
    1286                 : 
    1287                 : } /* namespace gc */
    1288                 : 
    1289                 : struct GCPtrHasher
    1290                 : {
    1291                 :     typedef void *Lookup;
    1292                 : 
    1293               0 :     static HashNumber hash(void *key) {
    1294               0 :         return HashNumber(uintptr_t(key) >> JS_GCTHING_ZEROBITS);
    1295                 :     }
    1296                 : 
    1297               0 :     static bool match(void *l, void *k) { return l == k; }
    1298                 : };
    1299                 : 
    1300                 : typedef HashMap<void *, uint32_t, GCPtrHasher, SystemAllocPolicy> GCLocks;
    1301                 : 
    1302                 : struct RootInfo {
    1303        21106968 :     RootInfo() {}
    1304          479080 :     RootInfo(const char *name, JSGCRootType type) : name(name), type(type) {}
    1305                 :     const char *name;
    1306                 :     JSGCRootType type;
    1307                 : };
    1308                 : 
    1309                 : typedef js::HashMap<void *,
    1310                 :                     RootInfo,
    1311                 :                     js::DefaultHasher<void *>,
    1312                 :                     js::SystemAllocPolicy> RootedValueMap;
    1313                 : 
    1314                 : } /* namespace js */
    1315                 : 
    1316                 : extern JS_FRIEND_API(JSGCTraceKind)
    1317                 : js_GetGCThingTraceKind(void *thing);
    1318                 : 
    1319                 : extern JSBool
    1320                 : js_InitGC(JSRuntime *rt, uint32_t maxbytes);
    1321                 : 
    1322                 : extern void
    1323                 : js_FinishGC(JSRuntime *rt);
    1324                 : 
    1325                 : extern JSBool
    1326                 : js_AddRoot(JSContext *cx, js::Value *vp, const char *name);
    1327                 : 
    1328                 : extern JSBool
    1329                 : js_AddGCThingRoot(JSContext *cx, void **rp, const char *name);
    1330                 : 
    1331                 : #ifdef DEBUG
    1332                 : extern void
    1333                 : js_DumpNamedRoots(JSRuntime *rt,
    1334                 :                   void (*dump)(const char *name, void *rp, JSGCRootType type, void *data),
    1335                 :                   void *data);
    1336                 : #endif
    1337                 : 
    1338                 : extern uint32_t
    1339                 : js_MapGCRoots(JSRuntime *rt, JSGCRootMapFun map, void *data);
    1340                 : 
    1341                 : /* Table of pointers with count valid members. */
    1342                 : typedef struct JSPtrTable {
    1343                 :     size_t      count;
    1344                 :     void        **array;
    1345                 : } JSPtrTable;
    1346                 : 
    1347                 : extern JSBool
    1348                 : js_LockGCThingRT(JSRuntime *rt, void *thing);
    1349                 : 
    1350                 : extern void
    1351                 : js_UnlockGCThingRT(JSRuntime *rt, void *thing);
    1352                 : 
    1353                 : extern JS_FRIEND_API(bool)
    1354                 : IsAboutToBeFinalized(const js::gc::Cell *thing);
    1355                 : 
    1356                 : extern bool
    1357                 : IsAboutToBeFinalized(const js::Value &value);
    1358                 : 
    1359                 : extern bool
    1360                 : js_IsAddressableGCThing(JSRuntime *rt, uintptr_t w, js::gc::AllocKind *thingKind, void **thing);
    1361                 : 
    1362                 : namespace js {
    1363                 : 
    1364                 : extern void
    1365                 : MarkCompartmentActive(js::StackFrame *fp);
    1366                 : 
    1367                 : extern void
    1368                 : TraceRuntime(JSTracer *trc);
    1369                 : 
    1370                 : extern JS_FRIEND_API(void)
    1371                 : MarkContext(JSTracer *trc, JSContext *acx);
    1372                 : 
    1373                 : /* Must be called with GC lock taken. */
    1374                 : extern void
    1375                 : TriggerGC(JSRuntime *rt, js::gcreason::Reason reason);
    1376                 : 
    1377                 : /* Must be called with GC lock taken. */
    1378                 : extern void
    1379                 : TriggerCompartmentGC(JSCompartment *comp, js::gcreason::Reason reason);
    1380                 : 
    1381                 : extern void
    1382                 : MaybeGC(JSContext *cx);
    1383                 : 
    1384                 : extern void
    1385                 : ShrinkGCBuffers(JSRuntime *rt);
    1386                 : 
    1387                 : /*
    1388                 :  * Kinds of js_GC invocation.
    1389                 :  */
    1390                 : typedef enum JSGCInvocationKind {
    1391                 :     /* Normal invocation. */
    1392                 :     GC_NORMAL           = 0,
    1393                 : 
    1394                 :     /* Minimize GC triggers and release empty GC chunks right away. */
    1395                 :     GC_SHRINK             = 1
    1396                 : } JSGCInvocationKind;
    1397                 : 
    1398                 : /* Pass NULL for |comp| to get a full GC. */
    1399                 : extern void
    1400                 : GC(JSContext *cx, JSCompartment *comp, JSGCInvocationKind gckind, js::gcreason::Reason reason);
    1401                 : 
    1402                 : extern void
    1403                 : GCSlice(JSContext *cx, JSCompartment *comp, JSGCInvocationKind gckind, js::gcreason::Reason reason);
    1404                 : 
    1405                 : extern void
    1406                 : GCDebugSlice(JSContext *cx, int64_t objCount);
    1407                 : 
    1408                 : } /* namespace js */
    1409                 : 
    1410                 : namespace js {
    1411                 : 
    1412                 : void
    1413                 : InitTracer(JSTracer *trc, JSRuntime *rt, JSTraceCallback callback);
    1414                 : 
    1415                 : #ifdef JS_THREADSAFE
    1416                 : 
    1417           19908 : class GCHelperThread {
    1418                 :     enum State {
    1419                 :         IDLE,
    1420                 :         SWEEPING,
    1421                 :         ALLOCATING,
    1422                 :         CANCEL_ALLOCATION,
    1423                 :         SHUTDOWN
    1424                 :     };
    1425                 : 
    1426                 :     /*
    1427                 :      * During the finalization we do not free immediately. Rather we add the
    1428                 :      * corresponding pointers to a buffer which we later release on a
    1429                 :      * separated thread.
    1430                 :      *
    1431                 :      * The buffer is implemented as a vector of 64K arrays of pointers, not as
    1432                 :      * a simple vector, to avoid realloc calls during the vector growth and to
    1433                 :      * not bloat the binary size of the inlined freeLater method. Any OOM
    1434                 :      * during buffer growth results in the pointer being freed immediately.
    1435                 :      */
    1436                 :     static const size_t FREE_ARRAY_SIZE = size_t(1) << 16;
    1437                 :     static const size_t FREE_ARRAY_LENGTH = FREE_ARRAY_SIZE / sizeof(void *);
    1438                 : 
    1439                 :     JSRuntime         *const rt;
    1440                 :     PRThread          *thread;
    1441                 :     PRCondVar         *wakeup;
    1442                 :     PRCondVar         *done;
    1443                 :     volatile State    state;
    1444                 : 
    1445                 :     JSContext         *finalizationContext;
    1446                 :     bool              shrinkFlag;
    1447                 : 
    1448                 :     Vector<void **, 16, js::SystemAllocPolicy> freeVector;
    1449                 :     void            **freeCursor;
    1450                 :     void            **freeCursorEnd;
    1451                 : 
    1452                 :     Vector<js::gc::ArenaHeader *, 64, js::SystemAllocPolicy> finalizeVector;
    1453                 : 
    1454                 :     bool    backgroundAllocation;
    1455                 : 
    1456                 :     friend struct js::gc::ArenaLists;
    1457                 : 
    1458                 :     JS_FRIEND_API(void)
    1459                 :     replenishAndFreeLater(void *ptr);
    1460                 : 
    1461           22779 :     static void freeElementsAndArray(void **array, void **end) {
    1462           22779 :         JS_ASSERT(array <= end);
    1463         2851968 :         for (void **p = array; p != end; ++p)
    1464         2829189 :             js::Foreground::free_(*p);
    1465           22779 :         js::Foreground::free_(array);
    1466           22779 :     }
    1467                 : 
    1468                 :     static void threadMain(void* arg);
    1469                 :     void threadLoop();
    1470                 : 
    1471                 :     /* Must be called with the GC lock taken. */
    1472                 :     void doSweep();
    1473                 : 
    1474                 :   public:
    1475           19910 :     GCHelperThread(JSRuntime *rt)
    1476                 :       : rt(rt),
    1477                 :         thread(NULL),
    1478                 :         wakeup(NULL),
    1479                 :         done(NULL),
    1480                 :         state(IDLE),
    1481                 :         finalizationContext(NULL),
    1482                 :         shrinkFlag(false),
    1483                 :         freeCursor(NULL),
    1484                 :         freeCursorEnd(NULL),
    1485           19910 :         backgroundAllocation(true)
    1486           19910 :     { }
    1487                 : 
    1488                 :     bool init();
    1489                 :     void finish();
    1490                 : 
    1491                 :     /* Must be called with the GC lock taken. */
    1492                 :     void startBackgroundSweep(JSContext *cx, bool shouldShrink);
    1493                 : 
    1494                 :     /* Must be called with the GC lock taken. */
    1495                 :     void startBackgroundShrink();
    1496                 : 
    1497                 :     /* Must be called with the GC lock taken. */
    1498                 :     void waitBackgroundSweepEnd();
    1499                 : 
    1500                 :     /* Must be called with the GC lock taken. */
    1501                 :     void waitBackgroundSweepOrAllocEnd();
    1502                 : 
    1503                 :     /* Must be called with the GC lock taken. */
    1504                 :     inline void startBackgroundAllocationIfIdle();
    1505                 : 
    1506           46233 :     bool canBackgroundAllocate() const {
    1507           46233 :         return backgroundAllocation;
    1508                 :     }
    1509                 : 
    1510                 :     void disableBackgroundAllocation() {
    1511                 :         backgroundAllocation = false;
    1512                 :     }
    1513                 : 
    1514                 :     PRThread *getThread() const {
    1515                 :         return thread;
    1516                 :     }
    1517                 : 
    1518                 :     /*
    1519                 :      * Outside the GC lock may give true answer when in fact the sweeping has
    1520                 :      * been done.
    1521                 :      */
    1522         7943653 :     bool sweeping() const {
    1523         7943653 :         return state == SWEEPING;
    1524                 :     }
    1525                 : 
    1526                 :     bool shouldShrink() const {
    1527                 :         JS_ASSERT(sweeping());
    1528                 :         return shrinkFlag;
    1529                 :     }
    1530                 : 
    1531         2829189 :     void freeLater(void *ptr) {
    1532         2829189 :         JS_ASSERT(!sweeping());
    1533         2829189 :         if (freeCursor != freeCursorEnd)
    1534         2806410 :             *freeCursor++ = ptr;
    1535                 :         else
    1536           22779 :             replenishAndFreeLater(ptr);
    1537         2829189 :     }
    1538                 : 
    1539                 :     /* Must be called with the GC lock taken. */
    1540                 :     bool prepareForBackgroundSweep();
    1541                 : };
    1542                 : 
    1543                 : #endif /* JS_THREADSAFE */
    1544                 : 
    1545                 : struct GCChunkHasher {
    1546                 :     typedef gc::Chunk *Lookup;
    1547                 : 
    1548                 :     /*
    1549                 :      * Strip zeros for better distribution after multiplying by the golden
    1550                 :      * ratio.
    1551                 :      */
    1552       215119814 :     static HashNumber hash(gc::Chunk *chunk) {
    1553       215119814 :         JS_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask));
    1554       215119814 :         return HashNumber(uintptr_t(chunk) >> gc::ChunkShift);
    1555                 :     }
    1556                 : 
    1557        11673595 :     static bool match(gc::Chunk *k, gc::Chunk *l) {
    1558        11673595 :         JS_ASSERT(!(uintptr_t(k) & gc::ChunkMask));
    1559        11673595 :         JS_ASSERT(!(uintptr_t(l) & gc::ChunkMask));
    1560        11673595 :         return k == l;
    1561                 :     }
    1562                 : };
    1563                 : 
    1564                 : typedef HashSet<js::gc::Chunk *, GCChunkHasher, SystemAllocPolicy> GCChunkSet;
    1565                 : 
    1566                 : template<class T>
    1567                 : struct MarkStack {
    1568                 :     T *stack;
    1569                 :     T *tos;
    1570                 :     T *limit;
    1571                 : 
    1572                 :     T *ballast;
    1573                 :     T *ballastLimit;
    1574                 : 
    1575                 :     size_t sizeLimit;
    1576                 : 
    1577           19910 :     MarkStack(size_t sizeLimit)
    1578                 :       : stack(NULL),
    1579                 :         tos(NULL),
    1580                 :         limit(NULL),
    1581                 :         ballast(NULL),
    1582                 :         ballastLimit(NULL),
    1583           19910 :         sizeLimit(sizeLimit) { }
    1584                 : 
    1585           19908 :     ~MarkStack() {
    1586           19908 :         if (stack != ballast)
    1587               0 :             js_free(stack);
    1588           19908 :         js_free(ballast);
    1589           19908 :     }
    1590                 : 
    1591           19910 :     bool init(size_t ballastcap) {
    1592           19910 :         JS_ASSERT(!stack);
    1593                 : 
    1594           19910 :         if (ballastcap == 0)
    1595               0 :             return true;
    1596                 : 
    1597           19910 :         ballast = (T *)js_malloc(sizeof(T) * ballastcap);
    1598           19910 :         if (!ballast)
    1599               0 :             return false;
    1600           19910 :         ballastLimit = ballast + ballastcap;
    1601           19910 :         initFromBallast();
    1602           19910 :         return true;
    1603                 :     }
    1604                 : 
    1605           73800 :     void initFromBallast() {
    1606           73800 :         stack = ballast;
    1607           73800 :         limit = ballastLimit;
    1608           73800 :         if (size_t(limit - stack) > sizeLimit)
    1609               0 :             limit = stack + sizeLimit;
    1610           73800 :         tos = stack;
    1611           73800 :     }
    1612                 : 
    1613               0 :     void setSizeLimit(size_t size) {
    1614               0 :         JS_ASSERT(isEmpty());
    1615                 : 
    1616               0 :         sizeLimit = size;
    1617               0 :         reset();
    1618               0 :     }
    1619                 : 
    1620        19320797 :     bool push(T item) {
    1621        19320797 :         if (tos == limit) {
    1622              18 :             if (!enlarge())
    1623               0 :                 return false;
    1624                 :         }
    1625        19320797 :         JS_ASSERT(tos < limit);
    1626        19320797 :         *tos++ = item;
    1627        19320797 :         return true;
    1628                 :     }
    1629                 : 
    1630        46072024 :     bool push(T item1, T item2, T item3) {
    1631        46072024 :         T *nextTos = tos + 3;
    1632        46072024 :         if (nextTos > limit) {
    1633               0 :             if (!enlarge())
    1634               0 :                 return false;
    1635               0 :             nextTos = tos + 3;
    1636                 :         }
    1637        46072024 :         JS_ASSERT(nextTos <= limit);
    1638        46072024 :         tos[0] = item1;
    1639        46072024 :         tos[1] = item2;
    1640        46072024 :         tos[2] = item3;
    1641        46072024 :         tos = nextTos;
    1642        46072024 :         return true;
    1643                 :     }
    1644                 : 
    1645       223488169 :     bool isEmpty() const {
    1646       223488169 :         return tos == stack;
    1647                 :     }
    1648                 : 
    1649       157532845 :     T pop() {
    1650       157532845 :         JS_ASSERT(!isEmpty());
    1651       157532845 :         return *--tos;
    1652                 :     }
    1653                 : 
    1654         8981314 :     ptrdiff_t position() const {
    1655         8981314 :         return tos - stack;
    1656                 :     }
    1657                 : 
    1658           53890 :     void reset() {
    1659           53890 :         if (stack != ballast)
    1660              18 :             js_free(stack);
    1661           53890 :         initFromBallast();
    1662           53890 :         JS_ASSERT(stack == ballast);
    1663           53890 :     }
    1664                 : 
    1665              18 :     bool enlarge() {
    1666              18 :         size_t tosIndex = tos - stack;
    1667              18 :         size_t cap = limit - stack;
    1668              18 :         if (cap == sizeLimit)
    1669               0 :             return false;
    1670              18 :         size_t newcap = cap * 2;
    1671              18 :         if (newcap == 0)
    1672               0 :             newcap = 32;
    1673              18 :         if (newcap > sizeLimit)
    1674               0 :             newcap = sizeLimit;
    1675                 : 
    1676                 :         T *newStack;
    1677              18 :         if (stack == ballast) {
    1678              18 :             newStack = (T *)js_malloc(sizeof(T) * newcap);
    1679              18 :             if (!newStack)
    1680               0 :                 return false;
    1681          589860 :             for (T *src = stack, *dst = newStack; src < tos; )
    1682          589824 :                 *dst++ = *src++;
    1683                 :         } else {
    1684               0 :             newStack = (T *)js_realloc(stack, sizeof(T) * newcap);
    1685               0 :             if (!newStack)
    1686               0 :                 return false;
    1687                 :         }
    1688              18 :         stack = newStack;
    1689              18 :         tos = stack + tosIndex;
    1690              18 :         limit = newStack + newcap;
    1691              18 :         return true;
    1692                 :     }
    1693                 : 
    1694               3 :     size_t sizeOfExcludingThis(JSMallocSizeOfFun mallocSizeOf) const {
    1695               3 :         size_t n = 0;
    1696               3 :         if (stack != ballast)
    1697               0 :             n += mallocSizeOf(stack);
    1698               3 :         n += mallocSizeOf(ballast);
    1699               3 :         return n;
    1700                 :     }
    1701                 : };
    1702                 : 
    1703                 : /*
    1704                 :  * This class records how much work has been done in a given GC slice, so that
    1705                 :  * we can return before pausing for too long. Some slices are allowed to run for
    1706                 :  * unlimited time, and others are bounded. To reduce the number of gettimeofday
    1707                 :  * calls, we only check the time every 1000 operations.
    1708                 :  */
    1709                 : struct SliceBudget {
    1710                 :     int64_t deadline; /* in microseconds */
    1711                 :     intptr_t counter;
    1712                 : 
    1713                 :     static const intptr_t CounterReset = 1000;
    1714                 : 
    1715                 :     static const int64_t Unlimited = 0;
    1716                 :     static int64_t TimeBudget(int64_t millis);
    1717                 :     static int64_t WorkBudget(int64_t work);
    1718                 : 
    1719                 :     /* Equivalent to SliceBudget(UnlimitedBudget). */
    1720                 :     SliceBudget();
    1721                 : 
    1722                 :     /* Instantiate as SliceBudget(Time/WorkBudget(n)). */
    1723                 :     SliceBudget(int64_t budget);
    1724                 : 
    1725               0 :     void reset() {
    1726               0 :         deadline = INT64_MAX;
    1727               0 :         counter = INTPTR_MAX;
    1728               0 :     }
    1729                 : 
    1730        54233629 :     void step() {
    1731        54233629 :         counter--;
    1732        54233629 :     }
    1733                 : 
    1734                 :     bool checkOverBudget();
    1735                 : 
    1736       119724928 :     bool isOverBudget() {
    1737       119724928 :         if (counter > 0)
    1738       119724928 :             return false;
    1739               0 :         return checkOverBudget();
    1740                 :     }
    1741                 : };
    1742                 : 
    1743                 : static const size_t MARK_STACK_LENGTH = 32768;
    1744                 : 
    1745           19908 : struct GCMarker : public JSTracer {
    1746                 :   private:
    1747                 :     /*
    1748                 :      * We use a common mark stack to mark GC things of different types and use
    1749                 :      * the explicit tags to distinguish them when it cannot be deduced from
    1750                 :      * the context of push or pop operation.
    1751                 :      */
    1752                 :     enum StackTag {
    1753                 :         ValueArrayTag,
    1754                 :         ObjectTag,
    1755                 :         TypeTag,
    1756                 :         XmlTag,
    1757                 :         SavedValueArrayTag,
    1758                 :         LastTag = SavedValueArrayTag
    1759                 :     };
    1760                 : 
    1761                 :     static const uintptr_t StackTagMask = 7;
    1762                 : 
    1763                 :     static void staticAsserts() {
    1764                 :         JS_STATIC_ASSERT(StackTagMask >= uintptr_t(LastTag));
    1765                 :         JS_STATIC_ASSERT(StackTagMask <= gc::Cell::CellMask);
    1766                 :     }
    1767                 : 
    1768                 :   public:
    1769                 :     explicit GCMarker();
    1770                 :     bool init();
    1771                 : 
    1772               0 :     void setSizeLimit(size_t size) { stack.setSizeLimit(size); }
    1773               0 :     size_t sizeLimit() const { return stack.sizeLimit; }
    1774                 : 
    1775                 :     void start(JSRuntime *rt);
    1776                 :     void stop();
    1777                 :     void reset();
    1778                 : 
    1779        17216759 :     void pushObject(JSObject *obj) {
    1780        17216759 :         pushTaggedPtr(ObjectTag, obj);
    1781        17216759 :     }
    1782                 : 
    1783         2101514 :     void pushType(types::TypeObject *type) {
    1784         2101514 :         pushTaggedPtr(TypeTag, type);
    1785         2101514 :     }
    1786                 : 
    1787            2522 :     void pushXML(JSXML *xml) {
    1788            2522 :         pushTaggedPtr(XmlTag, xml);
    1789            2522 :     }
    1790                 : 
    1791       416657168 :     uint32_t getMarkColor() const {
    1792       416657168 :         return color;
    1793                 :     }
    1794                 : 
    1795                 :     /*
    1796                 :      * The only valid color transition during a GC is from black to gray. It is
    1797                 :      * wrong to switch the mark color from gray to black. The reason is that the
    1798                 :      * cycle collector depends on the invariant that there are no black to gray
    1799                 :      * edges in the GC heap. This invariant lets the CC not trace through black
    1800                 :      * objects. If this invariant is violated, the cycle collector may free
    1801                 :      * objects that are still reachable.
    1802                 :      */
    1803           51092 :     void setMarkColorGray() {
    1804           51092 :         JS_ASSERT(isDrained());
    1805           51092 :         JS_ASSERT(color == gc::BLACK);
    1806           51092 :         color = gc::GRAY;
    1807           51092 :     }
    1808                 : 
    1809                 :     inline void delayMarkingArena(gc::ArenaHeader *aheader);
    1810                 :     void delayMarkingChildren(const void *thing);
    1811                 :     void markDelayedChildren(gc::ArenaHeader *aheader);
    1812                 :     bool markDelayedChildren(SliceBudget &budget);
    1813          102504 :     bool hasDelayedChildren() const {
    1814          102504 :         return !!unmarkedArenaStackTop;
    1815                 :     }
    1816                 : 
    1817          462626 :     bool isDrained() {
    1818          462626 :         return isMarkStackEmpty() && !unmarkedArenaStackTop;
    1819                 :     }
    1820                 : 
    1821                 :     bool drainMarkStack(SliceBudget &budget);
    1822                 : 
    1823                 :     /*
    1824                 :      * Gray marking must be done after all black marking is complete. However,
    1825                 :      * we do not have write barriers on XPConnect roots. Therefore, XPConnect
    1826                 :      * roots must be accumulated in the first slice of incremental GC. We
    1827                 :      * accumulate these roots in the GrayRootMarker and then mark them later,
    1828                 :      * after black marking is complete. This accumulation can fail, but in that
    1829                 :      * case we switch to non-incremental GC.
    1830                 :      */
    1831                 :     bool hasBufferedGrayRoots() const;
    1832                 :     void startBufferingGrayRoots();
    1833                 :     void endBufferingGrayRoots();
    1834                 :     void markBufferedGrayRoots();
    1835                 : 
    1836                 :     static void GrayCallback(JSTracer *trc, void **thing, JSGCTraceKind kind);
    1837                 : 
    1838                 :     size_t sizeOfExcludingThis(JSMallocSizeOfFun mallocSizeOf) const;
    1839                 : 
    1840                 :     MarkStack<uintptr_t> stack;
    1841                 : 
    1842                 :   private:
    1843                 : #ifdef DEBUG
    1844                 :     void checkCompartment(void *p);
    1845                 : #else
    1846                 :     void checkCompartment(void *p) {}
    1847                 : #endif
    1848                 : 
    1849        19320795 :     void pushTaggedPtr(StackTag tag, void *ptr) {
    1850        19320795 :         checkCompartment(ptr);
    1851        19320795 :         uintptr_t addr = reinterpret_cast<uintptr_t>(ptr);
    1852        19320795 :         JS_ASSERT(!(addr & StackTagMask));
    1853        19320795 :         if (!stack.push(addr | uintptr_t(tag)))
    1854               0 :             delayMarkingChildren(ptr);
    1855        19320795 :     }
    1856                 : 
    1857        46072024 :     void pushValueArray(JSObject *obj, void *start, void *end) {
    1858        46072024 :         checkCompartment(obj);
    1859                 : 
    1860        46072024 :         JS_ASSERT(start <= end);
    1861        46072024 :         uintptr_t tagged = reinterpret_cast<uintptr_t>(obj) | GCMarker::ValueArrayTag;
    1862        46072024 :         uintptr_t startAddr = reinterpret_cast<uintptr_t>(start);
    1863        46072024 :         uintptr_t endAddr = reinterpret_cast<uintptr_t>(end);
    1864                 : 
    1865                 :         /*
    1866                 :          * Push in the reverse order so obj will be on top. If we cannot push
    1867                 :          * the array, we trigger delay marking for the whole object.
    1868                 :          */
    1869        46072024 :         if (!stack.push(endAddr, startAddr, tagged))
    1870               0 :             delayMarkingChildren(obj);
    1871        46072024 :     }
    1872                 : 
    1873          464025 :     bool isMarkStackEmpty() {
    1874          464025 :         return stack.isEmpty();
    1875                 :     }
    1876                 : 
    1877                 :     bool restoreValueArray(JSObject *obj, void **vpp, void **endp);
    1878                 :     void saveValueRanges();
    1879                 :     inline void processMarkStackTop(SliceBudget &budget);
    1880                 :     void processMarkStackOther(uintptr_t tag, uintptr_t addr);
    1881                 : 
    1882                 :     void appendGrayRoot(void *thing, JSGCTraceKind kind);
    1883                 : 
    1884                 :     /* The color is only applied to objects, functions and xml. */
    1885                 :     uint32_t color;
    1886                 : 
    1887                 :     DebugOnly<bool> started;
    1888                 : 
    1889                 :     /* Pointer to the top of the stack of arenas we are delaying marking on. */
    1890                 :     js::gc::ArenaHeader *unmarkedArenaStackTop;
    1891                 :     /* Count of arenas that are currently in the stack. */
    1892                 :     DebugOnly<size_t> markLaterArenas;
    1893                 : 
    1894          467078 :     struct GrayRoot {
    1895                 :         void *thing;
    1896                 :         JSGCTraceKind kind;
    1897                 : #ifdef DEBUG
    1898                 :         JSTraceNamePrinter debugPrinter;
    1899                 :         const void *debugPrintArg;
    1900                 :         size_t debugPrintIndex;
    1901                 : #endif
    1902                 : 
    1903          199525 :         GrayRoot(void *thing, JSGCTraceKind kind)
    1904          199525 :           : thing(thing), kind(kind) {}
    1905                 :     };
    1906                 : 
    1907                 :     bool grayFailed;
    1908                 :     Vector<GrayRoot, 0, SystemAllocPolicy> grayRoots;
    1909                 : };
    1910                 : 
    1911                 : void
    1912                 : SetMarkStackLimit(JSRuntime *rt, size_t limit);
    1913                 : 
    1914                 : void
    1915                 : MarkStackRangeConservatively(JSTracer *trc, Value *begin, Value *end);
    1916                 : 
    1917                 : typedef void (*IterateChunkCallback)(JSRuntime *rt, void *data, gc::Chunk *chunk);
    1918                 : typedef void (*IterateArenaCallback)(JSRuntime *rt, void *data, gc::Arena *arena,
    1919                 :                                      JSGCTraceKind traceKind, size_t thingSize);
    1920                 : typedef void (*IterateCellCallback)(JSRuntime *rt, void *data, void *thing,
    1921                 :                                     JSGCTraceKind traceKind, size_t thingSize);
    1922                 : 
    1923                 : /*
    1924                 :  * This function calls |compartmentCallback| on every compartment,
    1925                 :  * |arenaCallback| on every in-use arena, and |cellCallback| on every in-use
    1926                 :  * cell in the GC heap.
    1927                 :  */
    1928                 : extern JS_FRIEND_API(void)
    1929                 : IterateCompartmentsArenasCells(JSRuntime *rt, void *data,
    1930                 :                                JSIterateCompartmentCallback compartmentCallback,
    1931                 :                                IterateArenaCallback arenaCallback,
    1932                 :                                IterateCellCallback cellCallback);
    1933                 : 
    1934                 : /*
    1935                 :  * Invoke chunkCallback on every in-use chunk.
    1936                 :  */
    1937                 : extern JS_FRIEND_API(void)
    1938                 : IterateChunks(JSRuntime *rt, void *data, IterateChunkCallback chunkCallback);
    1939                 : 
    1940                 : /*
    1941                 :  * Invoke cellCallback on every in-use object of the specified thing kind for
    1942                 :  * the given compartment or for all compartments if it is null.
    1943                 :  */
    1944                 : extern JS_FRIEND_API(void)
    1945                 : IterateCells(JSRuntime *rt, JSCompartment *compartment, gc::AllocKind thingKind,
    1946                 :              void *data, IterateCellCallback cellCallback);
    1947                 : 
    1948                 : } /* namespace js */
    1949                 : 
    1950                 : extern void
    1951                 : js_FinalizeStringRT(JSRuntime *rt, JSString *str);
    1952                 : 
    1953                 : /*
    1954                 :  * Macro to test if a traversal is the marking phase of the GC.
    1955                 :  */
    1956                 : #define IS_GC_MARKING_TRACER(trc) \
    1957                 :     ((trc)->callback == NULL || (trc)->callback == GCMarker::GrayCallback)
    1958                 : 
    1959                 : namespace js {
    1960                 : namespace gc {
    1961                 : 
    1962                 : JSCompartment *
    1963                 : NewCompartment(JSContext *cx, JSPrincipals *principals);
    1964                 : 
    1965                 : /* Tries to run a GC no matter what (used for GC zeal). */
    1966                 : void
    1967                 : RunDebugGC(JSContext *cx);
    1968                 : 
    1969                 : void
    1970                 : SetDeterministicGC(JSContext *cx, bool enabled);
    1971                 : 
    1972                 : #if defined(JSGC_ROOT_ANALYSIS) && defined(DEBUG) && !defined(JS_THREADSAFE)
    1973                 : /* Overwrites stack references to GC things which have not been rooted. */
    1974                 : void CheckStackRoots(JSContext *cx);
    1975                 : 
    1976                 : inline void MaybeCheckStackRoots(JSContext *cx) { CheckStackRoots(cx); }
    1977                 : #else
    1978       245571019 : inline void MaybeCheckStackRoots(JSContext *cx) {}
    1979                 : #endif
    1980                 : 
    1981                 : const int ZealPokeValue = 1;
    1982                 : const int ZealAllocValue = 2;
    1983                 : const int ZealFrameGCValue = 3;
    1984                 : const int ZealVerifierValue = 4;
    1985                 : const int ZealFrameVerifierValue = 5;
    1986                 : 
    1987                 : #ifdef JS_GC_ZEAL
    1988                 : 
    1989                 : /* Check that write barriers have been used correctly. See jsgc.cpp. */
    1990                 : void
    1991                 : VerifyBarriers(JSContext *cx);
    1992                 : 
    1993                 : void
    1994                 : MaybeVerifyBarriers(JSContext *cx, bool always = false);
    1995                 : 
    1996                 : #else
    1997                 : 
    1998                 : static inline void
    1999                 : VerifyBarriers(JSContext *cx)
    2000                 : {
    2001                 : }
    2002                 : 
    2003                 : static inline void
    2004                 : MaybeVerifyBarriers(JSContext *cx, bool always = false)
    2005                 : {
    2006                 : }
    2007                 : 
    2008                 : #endif
    2009                 : 
    2010                 : } /* namespace gc */
    2011                 : 
    2012                 : static inline JSCompartment *
    2013                 : GetObjectCompartment(JSObject *obj) { return reinterpret_cast<js::gc::Cell *>(obj)->compartment(); }
    2014                 : 
    2015                 : } /* namespace js */
    2016                 : 
    2017                 : #endif /* jsgc_h___ */

Generated by: LCOV version 1.7