1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "jsgcmark.h"
8 : #include "jsprf.h"
9 : #include "jsscope.h"
10 : #include "jsstr.h"
11 :
12 : #include "jsobjinlines.h"
13 : #include "jsscopeinlines.h"
14 :
15 : #include "vm/String-inl.h"
16 : #include "methodjit/MethodJIT.h"
17 :
18 : /*
19 : * There are two mostly separate mark paths. The first is a fast path used
20 : * internally in the GC. The second is a slow path used for root marking and
21 : * for API consumers like the cycle collector or Class::trace implementations.
22 : *
23 : * The fast path uses explicit stacks. The basic marking process during a GC is
24 : * that all roots are pushed on to a mark stack, and then each item on the
25 : * stack is scanned (possibly pushing more stuff) until the stack is empty.
26 : *
27 : * PushMarkStack pushes a GC thing onto the mark stack. In some cases (shapes
28 : * or strings) it eagerly marks the object rather than pushing it. Popping and
29 : * scanning is done by the processMarkStackTop method. For efficiency reasons
30 : * like tail recursion elimination that method also implements the scanning of
31 : * objects. For other GC things it uses helper methods.
32 : *
33 : * Most of the marking code outside jsgcmark uses functions like MarkObject,
34 : * MarkString, etc. These functions check if an object is in the compartment
35 : * currently being GCed. If it is, they call PushMarkStack. Roots are pushed
36 : * this way as well as pointers traversed inside trace hooks (for things like
37 : * IteratorClass). It it always valid to call a MarkX function instead of
38 : * PushMarkStack, although it may be slower.
39 : *
40 : * The MarkX functions also handle non-GC object traversal. In this case, they
41 : * call a callback for each object visited. This is a recursive process; the
42 : * mark stacks are not involved. These callbacks may ask for the outgoing
43 : * pointers to be visited. Eventually, this leads to the MarkChildren functions
44 : * being called. These functions duplicate much of the functionality of
45 : * scanning functions, but they don't push onto an explicit stack.
46 : */
47 :
48 : namespace js {
49 : namespace gc {
50 :
51 : static inline void
52 : PushMarkStack(GCMarker *gcmarker, JSXML *thing);
53 :
54 : static inline void
55 : PushMarkStack(GCMarker *gcmarker, JSObject *thing);
56 :
57 : static inline void
58 : PushMarkStack(GCMarker *gcmarker, JSFunction *thing);
59 :
60 : static inline void
61 : PushMarkStack(GCMarker *gcmarker, JSScript *thing);
62 :
63 : static inline void
64 : PushMarkStack(GCMarker *gcmarker, Shape *thing);
65 :
66 : static inline void
67 : PushMarkStack(GCMarker *gcmarker, JSString *thing);
68 :
69 : static inline void
70 : PushMarkStack(GCMarker *gcmarker, types::TypeObject *thing);
71 :
72 : /*** Object Marking ***/
73 :
74 : template<typename T>
75 : static inline void
76 459700422 : CheckMarkedThing(JSTracer *trc, T *thing)
77 : {
78 459700422 : JS_ASSERT(trc);
79 459700422 : JS_ASSERT(thing);
80 459700422 : JS_ASSERT(trc->debugPrinter || trc->debugPrintArg);
81 459700422 : JS_ASSERT_IF(trc->runtime->gcCurrentCompartment, IS_GC_MARKING_TRACER(trc));
82 :
83 459700422 : JS_ASSERT(thing->isAligned());
84 :
85 459700422 : JS_ASSERT(thing->compartment());
86 459700422 : JS_ASSERT(thing->compartment()->rt == trc->runtime);
87 459700422 : }
88 :
89 : template<typename T>
90 : void
91 459700422 : MarkInternal(JSTracer *trc, T *thing)
92 : {
93 459700422 : CheckMarkedThing(trc, thing);
94 :
95 459700422 : JSRuntime *rt = trc->runtime;
96 :
97 459700422 : JS_ASSERT_IF(rt->gcCheckCompartment,
98 : thing->compartment() == rt->gcCheckCompartment ||
99 : thing->compartment() == rt->atomsCompartment);
100 :
101 : /*
102 : * Don't mark things outside a compartment if we are in a per-compartment
103 : * GC.
104 : */
105 459700422 : if (!rt->gcCurrentCompartment || thing->compartment() == rt->gcCurrentCompartment) {
106 458612551 : if (!trc->callback) {
107 413351176 : PushMarkStack(static_cast<GCMarker *>(trc), thing);
108 : } else {
109 45261375 : void *tmp = (void *)thing;
110 45261375 : trc->callback(trc, &tmp, GetGCThingTraceKind(thing));
111 45261375 : JS_ASSERT(tmp == thing);
112 : }
113 : }
114 :
115 : #ifdef DEBUG
116 459700422 : trc->debugPrinter = NULL;
117 459700422 : trc->debugPrintArg = NULL;
118 : #endif
119 459700422 : }
120 :
121 : #define JS_ROOT_MARKING_ASSERT(trc) \
122 : JS_ASSERT_IF(IS_GC_MARKING_TRACER(trc), \
123 : trc->runtime->gcIncrementalState == NO_INCREMENTAL || \
124 : trc->runtime->gcIncrementalState == MARK_ROOTS);
125 :
126 :
127 : template <typename T>
128 : static void
129 370974034 : MarkUnbarriered(JSTracer *trc, T **thingp, const char *name)
130 : {
131 370974034 : JS_SET_TRACING_NAME(trc, name);
132 370974034 : MarkInternal(trc, *thingp);
133 370974034 : }
134 :
135 : template <typename T>
136 : static void
137 28680678 : Mark(JSTracer *trc, HeapPtr<T> *thing, const char *name)
138 : {
139 28680678 : JS_SET_TRACING_NAME(trc, name);
140 28680678 : MarkInternal(trc, thing->get());
141 28680678 : }
142 :
143 : template <typename T>
144 : static void
145 20770622 : MarkRoot(JSTracer *trc, T **thingp, const char *name)
146 : {
147 20770622 : JS_ROOT_MARKING_ASSERT(trc);
148 20770622 : JS_SET_TRACING_NAME(trc, name);
149 20770622 : MarkInternal(trc, *thingp);
150 20770622 : }
151 :
152 : template <typename T>
153 : static void
154 2261072 : MarkRange(JSTracer *trc, size_t len, HeapPtr<T> *vec, const char *name)
155 : {
156 6914742 : for (size_t i = 0; i < len; ++i) {
157 4653670 : if (T *obj = vec[i]) {
158 4653670 : JS_SET_TRACING_INDEX(trc, name, i);
159 4653670 : MarkInternal(trc, obj);
160 : }
161 : }
162 2261072 : }
163 :
164 : template <typename T>
165 : static void
166 620 : MarkRootRange(JSTracer *trc, size_t len, T **vec, const char *name)
167 : {
168 620 : JS_ROOT_MARKING_ASSERT(trc);
169 3521 : for (size_t i = 0; i < len; ++i) {
170 2901 : JS_SET_TRACING_INDEX(trc, name, i);
171 2901 : MarkInternal(trc, vec[i]);
172 : }
173 620 : }
174 :
175 : #define DeclMarkerImpl(base, type) \
176 : void \
177 : Mark##base(JSTracer *trc, HeapPtr<type> *thing, const char *name) \
178 : { \
179 : Mark<type>(trc, thing, name); \
180 : } \
181 : \
182 : void \
183 : Mark##base##Root(JSTracer *trc, type **thingp, const char *name) \
184 : { \
185 : MarkRoot<type>(trc, thingp, name); \
186 : } \
187 : \
188 : void \
189 : Mark##base##Unbarriered(JSTracer *trc, type **thingp, const char *name) \
190 : { \
191 : MarkUnbarriered<type>(trc, thingp, name); \
192 : } \
193 : \
194 : void Mark##base##Range(JSTracer *trc, size_t len, HeapPtr<type> *vec, const char *name) \
195 : { \
196 : MarkRange<type>(trc, len, vec, name); \
197 : } \
198 : \
199 : void Mark##base##RootRange(JSTracer *trc, size_t len, type **vec, const char *name) \
200 : { \
201 : MarkRootRange<type>(trc, len, vec, name); \
202 : } \
203 :
204 3713440 : DeclMarkerImpl(BaseShape, BaseShape)
205 38145 : DeclMarkerImpl(BaseShape, UnownedBaseShape)
206 1396 : DeclMarkerImpl(Object, ArgumentsObject)
207 66908 : DeclMarkerImpl(Object, GlobalObject)
208 12226033 : DeclMarkerImpl(Object, JSObject)
209 7412034 : DeclMarkerImpl(Object, JSFunction)
210 13676959 : DeclMarkerImpl(Script, JSScript)
211 13211621 : DeclMarkerImpl(Shape, Shape)
212 368807677 : DeclMarkerImpl(String, JSAtom)
213 160486 : DeclMarkerImpl(String, JSString)
214 7645 : DeclMarkerImpl(String, JSFlatString)
215 9563 : DeclMarkerImpl(String, JSLinearString)
216 3352556 : DeclMarkerImpl(TypeObject, types::TypeObject)
217 : #if JS_HAS_XML_SUPPORT
218 2563 : DeclMarkerImpl(XML, JSXML)
219 : #endif
220 :
221 : /*** Externally Typed Marking ***/
222 :
223 : void
224 30761011 : MarkKind(JSTracer *trc, void *thing, JSGCTraceKind kind)
225 : {
226 30761011 : JS_ASSERT(thing);
227 30761011 : JS_ASSERT(kind == GetGCThingTraceKind(thing));
228 30761011 : switch (kind) {
229 : case JSTRACE_OBJECT:
230 24094619 : MarkInternal(trc, reinterpret_cast<JSObject *>(thing));
231 24094619 : break;
232 : case JSTRACE_STRING:
233 3549112 : MarkInternal(trc, reinterpret_cast<JSString *>(thing));
234 3549112 : break;
235 : case JSTRACE_SCRIPT:
236 1765239 : MarkInternal(trc, static_cast<JSScript *>(thing));
237 1765239 : break;
238 : case JSTRACE_SHAPE:
239 1177371 : MarkInternal(trc, reinterpret_cast<Shape *>(thing));
240 1177371 : break;
241 : case JSTRACE_BASE_SHAPE:
242 139341 : MarkInternal(trc, reinterpret_cast<BaseShape *>(thing));
243 139341 : break;
244 : case JSTRACE_TYPE_OBJECT:
245 35284 : MarkInternal(trc, reinterpret_cast<types::TypeObject *>(thing));
246 35284 : break;
247 : #if JS_HAS_XML_SUPPORT
248 : case JSTRACE_XML:
249 45 : MarkInternal(trc, static_cast<JSXML *>(thing));
250 45 : break;
251 : #endif
252 : }
253 30761011 : }
254 :
255 : void
256 134797 : MarkGCThingRoot(JSTracer *trc, void *thing, const char *name)
257 : {
258 134797 : JS_ROOT_MARKING_ASSERT(trc);
259 134797 : JS_SET_TRACING_NAME(trc, name);
260 134797 : if (!thing)
261 0 : return;
262 134797 : MarkKind(trc, thing, GetGCThingTraceKind(thing));
263 : }
264 :
265 : /*** ID Marking ***/
266 :
267 : static inline void
268 4677644 : MarkIdInternal(JSTracer *trc, jsid *id)
269 : {
270 4677644 : if (JSID_IS_STRING(*id)) {
271 3756763 : JSString *str = JSID_TO_STRING(*id);
272 3756763 : MarkInternal(trc, str);
273 3756763 : *id = ATOM_TO_JSID(reinterpret_cast<JSAtom *>(str));
274 920881 : } else if (JS_UNLIKELY(JSID_IS_OBJECT(*id))) {
275 0 : JSObject *obj = JSID_TO_OBJECT(*id);
276 0 : MarkInternal(trc, obj);
277 0 : *id = OBJECT_TO_JSID(obj);
278 : }
279 4677644 : }
280 :
281 : void
282 4625704 : MarkId(JSTracer *trc, HeapId *id, const char *name)
283 : {
284 4625704 : JS_SET_TRACING_NAME(trc, name);
285 4625704 : MarkIdInternal(trc, id->unsafeGet());
286 4625704 : }
287 :
288 : void
289 0 : MarkIdRoot(JSTracer *trc, jsid *id, const char *name)
290 : {
291 0 : JS_ROOT_MARKING_ASSERT(trc);
292 0 : JS_SET_TRACING_NAME(trc, name);
293 0 : MarkIdInternal(trc, id);
294 0 : }
295 :
296 : void
297 0 : MarkIdRange(JSTracer *trc, size_t len, HeapId *vec, const char *name)
298 : {
299 0 : for (size_t i = 0; i < len; ++i) {
300 0 : JS_SET_TRACING_INDEX(trc, name, i);
301 0 : MarkIdInternal(trc, vec[i].unsafeGet());
302 : }
303 0 : }
304 :
305 : void
306 1639 : MarkIdRootRange(JSTracer *trc, size_t len, jsid *vec, const char *name)
307 : {
308 1639 : JS_ROOT_MARKING_ASSERT(trc);
309 53579 : for (size_t i = 0; i < len; ++i) {
310 51940 : JS_SET_TRACING_INDEX(trc, name, i);
311 51940 : MarkIdInternal(trc, &vec[i]);
312 : }
313 1639 : }
314 :
315 : /*** Value Marking ***/
316 :
317 : static inline void
318 30229303 : MarkValueInternal(JSTracer *trc, Value *v)
319 : {
320 30229303 : if (v->isMarkable()) {
321 10203631 : JS_ASSERT(v->toGCThing());
322 10203631 : return MarkKind(trc, v->toGCThing(), v->gcKind());
323 : }
324 : }
325 :
326 : void
327 15280 : MarkValue(JSTracer *trc, HeapValue *v, const char *name)
328 : {
329 15280 : JS_SET_TRACING_NAME(trc, name);
330 15280 : MarkValueInternal(trc, v->unsafeGet());
331 15280 : }
332 :
333 : void
334 61631 : MarkValueRoot(JSTracer *trc, Value *v, const char *name)
335 : {
336 61631 : JS_ROOT_MARKING_ASSERT(trc);
337 61631 : JS_SET_TRACING_NAME(trc, name);
338 61631 : MarkValueInternal(trc, v);
339 61631 : }
340 :
341 : void
342 8395877 : MarkValueRange(JSTracer *trc, size_t len, HeapValue *vec, const char *name)
343 : {
344 24886077 : for (size_t i = 0; i < len; ++i) {
345 16490200 : JS_SET_TRACING_INDEX(trc, name, i);
346 16490200 : MarkValueInternal(trc, vec[i].unsafeGet());
347 : }
348 8395877 : }
349 :
350 : void
351 660175 : MarkValueRootRange(JSTracer *trc, size_t len, Value *vec, const char *name)
352 : {
353 660175 : JS_ROOT_MARKING_ASSERT(trc);
354 3896933 : for (size_t i = 0; i < len; ++i) {
355 3236758 : JS_SET_TRACING_INDEX(trc, name, i);
356 3236758 : MarkValueInternal(trc, &vec[i]);
357 : }
358 660175 : }
359 :
360 : /*** Slot Marking ***/
361 :
362 : void
363 83036 : MarkSlot(JSTracer *trc, HeapSlot *s, const char *name)
364 : {
365 83036 : JS_SET_TRACING_NAME(trc, name);
366 83036 : MarkValueInternal(trc, s->unsafeGet());
367 83036 : }
368 :
369 : void
370 21738 : MarkArraySlots(JSTracer *trc, size_t len, HeapSlot *vec, const char *name)
371 : {
372 69070 : for (size_t i = 0; i < len; ++i) {
373 47332 : JS_SET_TRACING_INDEX(trc, name, i);
374 47332 : MarkValueInternal(trc, vec[i].unsafeGet());
375 : }
376 21738 : }
377 :
378 : void
379 3283626 : MarkObjectSlots(JSTracer *trc, JSObject *obj, uint32_t start, uint32_t nslots)
380 : {
381 3283626 : JS_ASSERT(obj->isNative());
382 12953456 : for (uint32_t i = start; i < (start + nslots); ++i) {
383 9669830 : JS_SET_TRACING_DETAILS(trc, js_PrintObjectSlotName, obj, i);
384 9669830 : MarkValueInternal(trc, obj->nativeGetSlotRef(i).unsafeGet());
385 : }
386 3283626 : }
387 :
388 : void
389 169656 : MarkCrossCompartmentSlot(JSTracer *trc, HeapSlot *s, const char *name)
390 : {
391 169656 : if (s->isMarkable()) {
392 83405 : Cell *cell = (Cell *)s->toGCThing();
393 83405 : JSRuntime *rt = trc->runtime;
394 83405 : if (rt->gcCurrentCompartment && cell->compartment() != rt->gcCurrentCompartment)
395 420 : return;
396 :
397 : /* In case we're called from a write barrier. */
398 82985 : if (rt->gcIncrementalCompartment && cell->compartment() != rt->gcIncrementalCompartment)
399 0 : return;
400 :
401 82985 : MarkSlot(trc, s, name);
402 : }
403 : }
404 :
405 : /*** Special Marking ***/
406 :
407 : void
408 100743 : MarkObject(JSTracer *trc, HeapPtr<GlobalObject, JSScript *> *thingp, const char *name)
409 : {
410 100743 : JS_SET_TRACING_NAME(trc, name);
411 100743 : MarkInternal(trc, thingp->get());
412 100743 : }
413 :
414 : void
415 625236 : MarkValueUnbarriered(JSTracer *trc, Value *v, const char *name)
416 : {
417 625236 : JS_SET_TRACING_NAME(trc, name);
418 625236 : MarkValueInternal(trc, v);
419 625236 : }
420 :
421 : /*** Push Mark Stack ***/
422 :
423 : #define JS_COMPARTMENT_ASSERT(rt, thing) \
424 : JS_ASSERT_IF((rt)->gcCurrentCompartment, \
425 : (thing)->compartment() == (rt)->gcCurrentCompartment);
426 :
427 : #define JS_COMPARTMENT_ASSERT_STR(rt, thing) \
428 : JS_ASSERT_IF((rt)->gcCurrentCompartment, \
429 : (thing)->compartment() == (rt)->gcCurrentCompartment || \
430 : (thing)->compartment() == (rt)->atomsCompartment);
431 :
432 : static void
433 2565 : PushMarkStack(GCMarker *gcmarker, JSXML *thing)
434 : {
435 2565 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
436 :
437 2565 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
438 2522 : gcmarker->pushXML(thing);
439 2565 : }
440 :
441 : static void
442 52114292 : PushMarkStack(GCMarker *gcmarker, JSObject *thing)
443 : {
444 52114292 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
445 :
446 52114292 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
447 12286755 : gcmarker->pushObject(thing);
448 52114292 : }
449 :
450 : static void
451 7660417 : PushMarkStack(GCMarker *gcmarker, JSFunction *thing)
452 : {
453 7660417 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
454 :
455 7660417 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
456 4930004 : gcmarker->pushObject(thing);
457 7660417 : }
458 :
459 : static void
460 54284325 : PushMarkStack(GCMarker *gcmarker, types::TypeObject *thing)
461 : {
462 54284325 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
463 :
464 54284325 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
465 2101514 : gcmarker->pushType(thing);
466 54284325 : }
467 :
468 : static void
469 : MarkChildren(JSTracer *trc, JSScript *script);
470 :
471 : static void
472 15235806 : PushMarkStack(GCMarker *gcmarker, JSScript *thing)
473 : {
474 15235806 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
475 :
476 : /*
477 : * We mark scripts directly rather than pushing on the stack as they can
478 : * refer to other scripts only indirectly (like via nested functions) and
479 : * we cannot get to deep recursion.
480 : */
481 15235806 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
482 6746105 : MarkChildren(gcmarker, thing);
483 15235806 : }
484 :
485 : static void
486 : ScanShape(GCMarker *gcmarker, Shape *shape);
487 :
488 : static void
489 61927653 : PushMarkStack(GCMarker *gcmarker, Shape *thing)
490 : {
491 61927653 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
492 :
493 : /* We mark shapes directly rather than pushing on the stack. */
494 61927653 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
495 12846781 : ScanShape(gcmarker, thing);
496 61927653 : }
497 :
498 : static inline void
499 : ScanBaseShape(GCMarker *gcmarker, BaseShape *base);
500 :
501 : static void
502 91648431 : PushMarkStack(GCMarker *gcmarker, BaseShape *thing)
503 : {
504 91648431 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
505 :
506 : /* We mark base shapes directly rather than pushing on the stack. */
507 91648431 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
508 14780514 : ScanBaseShape(gcmarker, thing);
509 91648431 : }
510 :
511 : static void
512 91518768 : ScanShape(GCMarker *gcmarker, Shape *shape)
513 : {
514 : restart:
515 91518768 : PushMarkStack(gcmarker, shape->base());
516 :
517 91518768 : const HeapId &id = shape->propidRef();
518 91518768 : if (JSID_IS_STRING(id))
519 61059502 : PushMarkStack(gcmarker, JSID_TO_STRING(id));
520 30459266 : else if (JS_UNLIKELY(JSID_IS_OBJECT(id)))
521 0 : PushMarkStack(gcmarker, JSID_TO_OBJECT(id));
522 :
523 91518768 : shape = shape->previous();
524 91518768 : if (shape && shape->markIfUnmarked(gcmarker->getMarkColor()))
525 78671987 : goto restart;
526 12846781 : }
527 :
528 : static inline void
529 14780514 : ScanBaseShape(GCMarker *gcmarker, BaseShape *base)
530 : {
531 14780514 : base->assertConsistency();
532 :
533 14780514 : if (base->hasGetterObject())
534 3387018 : PushMarkStack(gcmarker, base->getterObject());
535 :
536 14780514 : if (base->hasSetterObject())
537 531664 : PushMarkStack(gcmarker, base->setterObject());
538 :
539 14780514 : if (JSObject *parent = base->getObjectParent())
540 14254957 : PushMarkStack(gcmarker, parent);
541 :
542 : /*
543 : * All children of the owned base shape are consistent with its
544 : * unowned one, thus we do not need to trace through children of the
545 : * unowned base shape.
546 : */
547 14780514 : if (base->isOwned()) {
548 1557471 : UnownedBaseShape *unowned = base->baseUnowned();
549 1557471 : JS_ASSERT(base->compartment() == unowned->compartment());
550 1557471 : unowned->markIfUnmarked(gcmarker->getMarkColor());
551 : }
552 14780514 : }
553 :
554 : static inline void
555 314904668 : ScanLinearString(GCMarker *gcmarker, JSLinearString *str)
556 : {
557 314904668 : JS_COMPARTMENT_ASSERT_STR(gcmarker->runtime, str);
558 314904668 : JS_ASSERT(str->isMarked());
559 :
560 : /*
561 : * Add extra asserts to confirm the static type to detect incorrect string
562 : * mutations.
563 : */
564 314904668 : JS_ASSERT(str->JSString::isLinear());
565 629881815 : while (str->isDependent()) {
566 496237 : str = str->asDependent().base();
567 496237 : JS_ASSERT(str->JSString::isLinear());
568 496237 : JS_COMPARTMENT_ASSERT_STR(gcmarker->runtime, str);
569 496237 : if (!str->markIfUnmarked())
570 423758 : break;
571 : }
572 314904668 : }
573 :
574 : /*
575 : * The function tries to scan the whole rope tree using the marking stack as
576 : * temporary storage. If that becomes full, the unscanned ropes are added to
577 : * the delayed marking list. When the function returns, the marking stack is
578 : * at the same depth as it was on entry. This way we avoid using tags when
579 : * pushing ropes to the stack as ropes never leaks to other users of the
580 : * stack. This also assumes that a rope can only point to other ropes or
581 : * linear strings, it cannot refer to GC things of other types.
582 : */
583 : static void
584 2993770 : ScanRope(GCMarker *gcmarker, JSRope *rope)
585 : {
586 2993770 : ptrdiff_t savedPos = gcmarker->stack.position();
587 7595171 : for (;;) {
588 10588941 : JS_ASSERT(GetGCThingTraceKind(rope) == JSTRACE_STRING);
589 10588941 : JS_ASSERT(rope->JSString::isRope());
590 10588941 : JS_COMPARTMENT_ASSERT_STR(gcmarker->runtime, rope);
591 10588941 : JS_ASSERT(rope->isMarked());
592 10588941 : JSRope *next = NULL;
593 :
594 10588941 : JSString *right = rope->rightChild();
595 10588941 : if (right->markIfUnmarked()) {
596 2327944 : if (right->isLinear())
597 51501 : ScanLinearString(gcmarker, &right->asLinear());
598 : else
599 2276443 : next = &right->asRope();
600 : }
601 :
602 10588941 : JSString *left = rope->leftChild();
603 10588941 : if (left->markIfUnmarked()) {
604 5323278 : if (left->isLinear()) {
605 4550 : ScanLinearString(gcmarker, &left->asLinear());
606 : } else {
607 : /*
608 : * When both children are ropes, set aside the right one to
609 : * scan it later.
610 : */
611 5318728 : if (next && !gcmarker->stack.push(reinterpret_cast<uintptr_t>(next)))
612 0 : gcmarker->delayMarkingChildren(next);
613 5318728 : next = &left->asRope();
614 : }
615 : }
616 10588941 : if (next) {
617 7595169 : rope = next;
618 2993772 : } else if (savedPos != gcmarker->stack.position()) {
619 2 : JS_ASSERT(savedPos < gcmarker->stack.position());
620 2 : rope = reinterpret_cast<JSRope *>(gcmarker->stack.pop());
621 : } else {
622 : break;
623 : }
624 : }
625 2993770 : JS_ASSERT(savedPos == gcmarker->stack.position());
626 2993770 : }
627 :
628 : static inline void
629 317842387 : ScanString(GCMarker *gcmarker, JSString *str)
630 : {
631 317842387 : if (str->isLinear())
632 314848617 : ScanLinearString(gcmarker, &str->asLinear());
633 : else
634 2993770 : ScanRope(gcmarker, &str->asRope());
635 317842387 : }
636 :
637 : static inline void
638 412156597 : PushMarkStack(GCMarker *gcmarker, JSString *str)
639 : {
640 412156597 : JS_COMPARTMENT_ASSERT_STR(gcmarker->runtime, str);
641 :
642 : /*
643 : * As string can only refer to other strings we fully scan its GC graph
644 : * using the explicit stack when navigating the rope tree to avoid
645 : * dealing with strings on the stack in drainMarkStack.
646 : */
647 412156597 : if (str->markIfUnmarked())
648 273009463 : ScanString(gcmarker, str);
649 412156597 : }
650 :
651 : void
652 3334743 : MarkChildren(JSTracer *trc, JSObject *obj)
653 : {
654 3334743 : obj->markChildren(trc);
655 3334743 : }
656 :
657 : static void
658 17368244 : MarkChildren(JSTracer *trc, JSString *str)
659 : {
660 17368244 : if (str->isDependent())
661 1019 : str->asDependent().markChildren(trc);
662 17367225 : else if (str->isRope())
663 2154 : str->asRope().markChildren(trc);
664 17368244 : }
665 :
666 : static void
667 6825145 : MarkChildren(JSTracer *trc, JSScript *script)
668 : {
669 6825145 : script->markChildren(trc);
670 6825145 : }
671 :
672 : static void
673 3706908 : MarkChildren(JSTracer *trc, Shape *shape)
674 : {
675 3706908 : shape->markChildren(trc);
676 3706908 : }
677 :
678 : static void
679 1145720 : MarkChildren(JSTracer *trc, BaseShape *base)
680 : {
681 1145720 : base->markChildren(trc);
682 1145720 : }
683 :
684 : /*
685 : * This function is used by the cycle collector to trace through the
686 : * children of a BaseShape (and its baseUnowned(), if any). The cycle
687 : * collector does not directly care about BaseShapes, so only the
688 : * getter, setter, and parent are marked. Furthermore, the parent is
689 : * marked only if it isn't the same as prevParent, which will be
690 : * updated to the current shape's parent.
691 : */
692 : inline void
693 916688 : MarkCycleCollectorChildren(JSTracer *trc, BaseShape *base, JSObject **prevParent)
694 : {
695 916688 : JS_ASSERT(base);
696 :
697 : /*
698 : * The cycle collector does not need to trace unowned base shapes,
699 : * as they have the same getter, setter and parent as the original
700 : * base shape.
701 : */
702 916688 : base->assertConsistency();
703 :
704 916688 : if (base->hasGetterObject()) {
705 12017 : JSObject *tmp = base->getterObject();
706 12017 : MarkObjectUnbarriered(trc, &tmp, "getter");
707 12017 : JS_ASSERT(tmp == base->getterObject());
708 : }
709 :
710 916688 : if (base->hasSetterObject()) {
711 470 : JSObject *tmp = base->setterObject();
712 470 : MarkObjectUnbarriered(trc, &tmp, "setter");
713 470 : JS_ASSERT(tmp == base->setterObject());
714 : }
715 :
716 916688 : JSObject *parent = base->getObjectParent();
717 916688 : if (parent && parent != *prevParent) {
718 314583 : MarkObjectUnbarriered(trc, &parent, "parent");
719 314583 : JS_ASSERT(parent == base->getObjectParent());
720 314583 : *prevParent = parent;
721 : }
722 916688 : }
723 :
724 : /*
725 : * This function is used by the cycle collector to trace through a
726 : * shape. The cycle collector does not care about shapes or base
727 : * shapes, so those are not marked. Instead, any shapes or base shapes
728 : * that are encountered have their children marked. Stack space is
729 : * bounded. If two shapes in a row have the same parent pointer, the
730 : * parent pointer will only be marked once.
731 : */
732 : void
733 375083 : MarkCycleCollectorChildren(JSTracer *trc, Shape *shape)
734 : {
735 375083 : JSObject *prevParent = NULL;
736 916688 : do {
737 916688 : MarkCycleCollectorChildren(trc, shape->base(), &prevParent);
738 916688 : MarkId(trc, &shape->propidRef(), "propid");
739 916688 : shape = shape->previous();
740 : } while (shape);
741 375083 : }
742 :
743 : static void
744 2100670 : ScanTypeObject(GCMarker *gcmarker, types::TypeObject *type)
745 : {
746 2100670 : if (!type->singleton) {
747 1998688 : unsigned count = type->getPropertyCount();
748 2005334 : for (unsigned i = 0; i < count; i++) {
749 6646 : types::Property *prop = type->getProperty(i);
750 6646 : if (prop && JSID_IS_STRING(prop->id))
751 3947 : PushMarkStack(gcmarker, JSID_TO_STRING(prop->id));
752 : }
753 : }
754 :
755 2100670 : if (type->proto)
756 2037380 : PushMarkStack(gcmarker, type->proto);
757 :
758 2100670 : if (type->newScript) {
759 9 : PushMarkStack(gcmarker, type->newScript->fun);
760 9 : PushMarkStack(gcmarker, type->newScript->shape);
761 : }
762 :
763 2100670 : if (type->interpretedFunction)
764 179204 : PushMarkStack(gcmarker, type->interpretedFunction);
765 :
766 2100670 : if (type->singleton && !type->lazy())
767 59990 : PushMarkStack(gcmarker, type->singleton);
768 :
769 2100670 : if (type->interpretedFunction)
770 179204 : PushMarkStack(gcmarker, type->interpretedFunction);
771 2100670 : }
772 :
773 : static void
774 409827 : MarkChildren(JSTracer *trc, types::TypeObject *type)
775 : {
776 409827 : if (!type->singleton) {
777 398875 : unsigned count = type->getPropertyCount();
778 400965 : for (unsigned i = 0; i < count; i++) {
779 2090 : types::Property *prop = type->getProperty(i);
780 2090 : if (prop)
781 2090 : MarkId(trc, &prop->id, "type_prop");
782 : }
783 : }
784 :
785 409827 : if (type->proto)
786 395371 : MarkObject(trc, &type->proto, "type_proto");
787 :
788 409827 : if (type->singleton && !type->lazy())
789 7219 : MarkObject(trc, &type->singleton, "type_singleton");
790 :
791 409827 : if (type->newScript) {
792 124 : MarkObject(trc, &type->newScript->fun, "type_new_function");
793 124 : MarkShape(trc, &type->newScript->shape, "type_new_shape");
794 : }
795 :
796 409827 : if (type->interpretedFunction)
797 34380 : MarkObject(trc, &type->interpretedFunction, "type_function");
798 409827 : }
799 :
800 : #ifdef JS_HAS_XML_SUPPORT
801 : static void
802 2522 : MarkChildren(JSTracer *trc, JSXML *xml)
803 : {
804 2522 : js_TraceXML(trc, xml);
805 2522 : }
806 : #endif
807 :
808 : template<typename T>
809 : void
810 0 : PushArenaTyped(GCMarker *gcmarker, ArenaHeader *aheader)
811 : {
812 0 : for (CellIterUnderGC i(aheader); !i.done(); i.next())
813 0 : PushMarkStack(gcmarker, i.get<T>());
814 0 : }
815 :
816 : void
817 0 : PushArena(GCMarker *gcmarker, ArenaHeader *aheader)
818 : {
819 0 : switch (MapAllocToTraceKind(aheader->getAllocKind())) {
820 : case JSTRACE_OBJECT:
821 0 : PushArenaTyped<JSObject>(gcmarker, aheader);
822 0 : break;
823 :
824 : case JSTRACE_STRING:
825 0 : PushArenaTyped<JSString>(gcmarker, aheader);
826 0 : break;
827 :
828 : case JSTRACE_SCRIPT:
829 0 : PushArenaTyped<JSScript>(gcmarker, aheader);
830 0 : break;
831 :
832 : case JSTRACE_SHAPE:
833 0 : PushArenaTyped<js::Shape>(gcmarker, aheader);
834 0 : break;
835 :
836 : case JSTRACE_BASE_SHAPE:
837 0 : PushArenaTyped<js::BaseShape>(gcmarker, aheader);
838 0 : break;
839 :
840 : case JSTRACE_TYPE_OBJECT:
841 0 : PushArenaTyped<js::types::TypeObject>(gcmarker, aheader);
842 0 : break;
843 :
844 : #if JS_HAS_XML_SUPPORT
845 : case JSTRACE_XML:
846 0 : PushArenaTyped<JSXML>(gcmarker, aheader);
847 0 : break;
848 : #endif
849 : }
850 0 : }
851 :
852 : } /* namespace gc */
853 :
854 : using namespace js::gc;
855 :
856 : struct SlotArrayLayout
857 : {
858 : union {
859 : HeapSlot *end;
860 : js::Class *clasp;
861 : };
862 : union {
863 : HeapSlot *start;
864 : uintptr_t index;
865 : };
866 : JSObject *obj;
867 :
868 : static void staticAsserts() {
869 : /* This should have the same layout as three mark stack items. */
870 : JS_STATIC_ASSERT(sizeof(SlotArrayLayout) == 3 * sizeof(uintptr_t));
871 : }
872 : };
873 :
874 : /*
875 : * During incremental GC, we return from drainMarkStack without having processed
876 : * the entire stack. At that point, JS code can run and reallocate slot arrays
877 : * that are stored on the stack. To prevent this from happening, we replace all
878 : * ValueArrayTag stack items with SavedValueArrayTag. In the latter, slots
879 : * pointers are replaced with slot indexes.
880 : *
881 : * We also replace the slot array end pointer (which can be derived from the obj
882 : * pointer) with the object's class. During JS executation, array slowification
883 : * can cause the layout of slots to change. We can observe that slowification
884 : * happened if the class changed; in that case, we completely rescan the array.
885 : */
886 : void
887 0 : GCMarker::saveValueRanges()
888 : {
889 0 : for (uintptr_t *p = stack.tos; p > stack.stack; ) {
890 0 : uintptr_t tag = *--p & StackTagMask;
891 0 : if (tag == ValueArrayTag) {
892 0 : p -= 2;
893 0 : SlotArrayLayout *arr = reinterpret_cast<SlotArrayLayout *>(p);
894 0 : JSObject *obj = arr->obj;
895 :
896 0 : if (obj->getClass() == &ArrayClass) {
897 0 : HeapSlot *vp = obj->getDenseArrayElements();
898 0 : JS_ASSERT(arr->start >= vp &&
899 0 : arr->end == vp + obj->getDenseArrayInitializedLength());
900 0 : arr->index = arr->start - vp;
901 : } else {
902 0 : HeapSlot *vp = obj->fixedSlots();
903 0 : unsigned nfixed = obj->numFixedSlots();
904 0 : if (arr->start == arr->end) {
905 0 : arr->index = obj->slotSpan();
906 0 : } if (arr->start >= vp && arr->start < vp + nfixed) {
907 0 : JS_ASSERT(arr->end == vp + Min(nfixed, obj->slotSpan()));
908 0 : arr->index = arr->start - vp;
909 : } else {
910 0 : JS_ASSERT(arr->start >= obj->slots &&
911 0 : arr->end == obj->slots + obj->slotSpan() - nfixed);
912 0 : arr->index = (arr->start - obj->slots) + nfixed;
913 : }
914 : }
915 0 : arr->clasp = obj->getClass();
916 0 : p[2] |= SavedValueArrayTag;
917 0 : } else if (tag == SavedValueArrayTag) {
918 0 : p -= 2;
919 : }
920 : }
921 0 : }
922 :
923 : JS_NEVER_INLINE bool
924 0 : GCMarker::restoreValueArray(JSObject *obj, void **vpp, void **endp)
925 : {
926 0 : uintptr_t start = stack.pop();
927 0 : js::Class *clasp = reinterpret_cast<js::Class *>(stack.pop());
928 :
929 0 : JS_ASSERT(obj->getClass() == clasp ||
930 0 : (clasp == &ArrayClass && obj->getClass() == &SlowArrayClass));
931 :
932 0 : if (clasp == &ArrayClass) {
933 0 : if (obj->getClass() != &ArrayClass)
934 0 : return false;
935 :
936 0 : uint32_t initlen = obj->getDenseArrayInitializedLength();
937 0 : HeapSlot *vp = obj->getDenseArrayElements();
938 0 : if (start < initlen) {
939 0 : *vpp = vp + start;
940 0 : *endp = vp + initlen;
941 : } else {
942 : /* The object shrunk, in which case no scanning is needed. */
943 0 : *vpp = *endp = vp;
944 : }
945 : } else {
946 0 : HeapSlot *vp = obj->fixedSlots();
947 0 : unsigned nfixed = obj->numFixedSlots();
948 0 : unsigned nslots = obj->slotSpan();
949 0 : if (start < nfixed) {
950 0 : *vpp = vp + start;
951 0 : *endp = vp + Min(nfixed, nslots);
952 0 : } else if (start < nslots) {
953 0 : *vpp = obj->slots + start - nfixed;
954 0 : *endp = obj->slots + nslots - nfixed;
955 : } else {
956 : /* The object shrunk, in which case no scanning is needed. */
957 0 : *vpp = *endp = obj->slots;
958 : }
959 : }
960 :
961 0 : JS_ASSERT(*vpp <= *endp);
962 0 : return true;
963 : }
964 :
965 : void
966 2103192 : GCMarker::processMarkStackOther(uintptr_t tag, uintptr_t addr)
967 : {
968 2103192 : if (tag == TypeTag) {
969 2100670 : ScanTypeObject(this, reinterpret_cast<types::TypeObject *>(addr));
970 2522 : } else if (tag == SavedValueArrayTag) {
971 0 : JS_ASSERT(!(addr & Cell::CellMask));
972 0 : JSObject *obj = reinterpret_cast<JSObject *>(addr);
973 : HeapValue *vp, *end;
974 0 : if (restoreValueArray(obj, (void **)&vp, (void **)&end))
975 0 : pushValueArray(obj, vp, end);
976 : else
977 0 : pushObject(obj);
978 : } else {
979 2522 : JS_ASSERT(tag == XmlTag);
980 2522 : MarkChildren(this, reinterpret_cast<JSXML *>(addr));
981 : }
982 2103192 : }
983 :
984 : inline void
985 65388795 : GCMarker::processMarkStackTop(SliceBudget &budget)
986 : {
987 : /*
988 : * The function uses explicit goto and implements the scanning of the
989 : * object directly. It allows to eliminate the tail recursion and
990 : * significantly improve the marking performance, see bug 641025.
991 : */
992 : HeapSlot *vp, *end;
993 : JSObject *obj;
994 :
995 65388795 : uintptr_t addr = stack.pop();
996 65388795 : uintptr_t tag = addr & StackTagMask;
997 65388795 : addr &= ~StackTagMask;
998 :
999 65388795 : if (tag == ValueArrayTag) {
1000 : JS_STATIC_ASSERT(ValueArrayTag == 0);
1001 46072024 : JS_ASSERT(!(addr & Cell::CellMask));
1002 46072024 : obj = reinterpret_cast<JSObject *>(addr);
1003 46072024 : uintptr_t addr2 = stack.pop();
1004 46072024 : uintptr_t addr3 = stack.pop();
1005 46072024 : JS_ASSERT(addr2 <= addr3);
1006 46072024 : JS_ASSERT((addr3 - addr2) % sizeof(Value) == 0);
1007 46072024 : vp = reinterpret_cast<HeapSlot *>(addr2);
1008 46072024 : end = reinterpret_cast<HeapSlot *>(addr3);
1009 46072024 : goto scan_value_array;
1010 : }
1011 :
1012 19316771 : if (tag == ObjectTag) {
1013 17213579 : obj = reinterpret_cast<JSObject *>(addr);
1014 17213579 : JS_COMPARTMENT_ASSERT(runtime, obj);
1015 17213579 : goto scan_obj;
1016 : }
1017 :
1018 2103192 : processMarkStackOther(tag, addr);
1019 2103192 : return;
1020 :
1021 : scan_value_array:
1022 100297278 : JS_ASSERT(vp <= end);
1023 375224899 : while (vp != end) {
1024 211650393 : const Value &v = *vp++;
1025 211650393 : if (v.isString()) {
1026 84541435 : JSString *str = v.toString();
1027 84541435 : JS_COMPARTMENT_ASSERT_STR(runtime, str);
1028 84541435 : if (str->markIfUnmarked())
1029 44832924 : ScanString(this, str);
1030 127108958 : } else if (v.isObject()) {
1031 47949909 : JSObject *obj2 = &v.toObject();
1032 47949909 : JS_COMPARTMENT_ASSERT(runtime, obj2);
1033 47949909 : if (obj2->markIfUnmarked(getMarkColor())) {
1034 37020050 : pushValueArray(obj, vp, end);
1035 37020050 : obj = obj2;
1036 37020050 : goto scan_obj;
1037 : }
1038 : }
1039 : }
1040 63277228 : return;
1041 :
1042 : scan_obj:
1043 : {
1044 54233629 : JS_COMPARTMENT_ASSERT(runtime, obj);
1045 :
1046 54233629 : budget.step();
1047 54233629 : if (budget.isOverBudget()) {
1048 0 : pushObject(obj);
1049 0 : return;
1050 : }
1051 :
1052 54233629 : types::TypeObject *type = obj->typeFromGC();
1053 54233629 : PushMarkStack(this, type);
1054 :
1055 54233629 : Shape *shape = obj->lastProperty();
1056 54233629 : PushMarkStack(this, shape);
1057 :
1058 : /* Call the trace hook if necessary. */
1059 54233629 : Class *clasp = shape->getObjectClass();
1060 54233629 : if (clasp->trace) {
1061 47371781 : if (clasp == &ArrayClass) {
1062 1325635 : JS_ASSERT(!shape->isNative());
1063 1325635 : vp = obj->getDenseArrayElements();
1064 1325635 : end = vp + obj->getDenseArrayInitializedLength();
1065 1325635 : goto scan_value_array;
1066 : } else {
1067 0 : JS_ASSERT_IF(runtime->gcIncrementalState != NO_INCREMENTAL,
1068 46046146 : clasp->flags & JSCLASS_IMPLEMENTS_BARRIERS);
1069 : }
1070 46046146 : clasp->trace(this, obj);
1071 : }
1072 :
1073 52907994 : if (!shape->isNative())
1074 8375 : return;
1075 :
1076 52899619 : unsigned nslots = obj->slotSpan();
1077 52899619 : vp = obj->fixedSlots();
1078 52899619 : if (obj->slots) {
1079 9051974 : unsigned nfixed = obj->numFixedSlots();
1080 9051974 : if (nslots > nfixed) {
1081 9051974 : pushValueArray(obj, vp, vp + nfixed);
1082 9051974 : vp = obj->slots;
1083 9051974 : end = vp + (nslots - nfixed);
1084 9051974 : goto scan_value_array;
1085 : }
1086 : }
1087 43847645 : JS_ASSERT(nslots <= obj->numFixedSlots());
1088 43847645 : end = vp + nslots;
1089 43847645 : goto scan_value_array;
1090 : }
1091 : }
1092 :
1093 : bool
1094 102504 : GCMarker::drainMarkStack(SliceBudget &budget)
1095 : {
1096 : #ifdef DEBUG
1097 102504 : JSRuntime *rt = runtime;
1098 :
1099 : struct AutoCheckCompartment {
1100 : JSRuntime *runtime;
1101 102504 : AutoCheckCompartment(JSRuntime *rt) : runtime(rt) {
1102 102504 : runtime->gcCheckCompartment = runtime->gcCurrentCompartment;
1103 102504 : }
1104 102504 : ~AutoCheckCompartment() { runtime->gcCheckCompartment = NULL; }
1105 205008 : } acc(rt);
1106 : #endif
1107 :
1108 102504 : if (budget.isOverBudget())
1109 0 : return false;
1110 :
1111 0 : for (;;) {
1112 65593803 : while (!stack.isEmpty()) {
1113 65388795 : processMarkStackTop(budget);
1114 65388795 : if (budget.isOverBudget()) {
1115 0 : saveValueRanges();
1116 0 : return false;
1117 : }
1118 : }
1119 :
1120 102504 : if (!hasDelayedChildren())
1121 : break;
1122 :
1123 : /*
1124 : * Mark children of things that caused too deep recursion during the
1125 : * above tracing. Don't do this until we're done with everything
1126 : * else.
1127 : */
1128 0 : if (!markDelayedChildren(budget)) {
1129 0 : saveValueRanges();
1130 0 : return false;
1131 : }
1132 : }
1133 :
1134 102504 : return true;
1135 : }
1136 :
1137 : void
1138 26044482 : TraceChildren(JSTracer *trc, void *thing, JSGCTraceKind kind)
1139 : {
1140 26044482 : switch (kind) {
1141 : case JSTRACE_OBJECT:
1142 3334743 : MarkChildren(trc, static_cast<JSObject *>(thing));
1143 3334743 : break;
1144 :
1145 : case JSTRACE_STRING:
1146 17368244 : MarkChildren(trc, static_cast<JSString *>(thing));
1147 17368244 : break;
1148 :
1149 : case JSTRACE_SCRIPT:
1150 79040 : MarkChildren(trc, static_cast<JSScript *>(thing));
1151 79040 : break;
1152 :
1153 : case JSTRACE_SHAPE:
1154 3706908 : MarkChildren(trc, static_cast<Shape *>(thing));
1155 3706908 : break;
1156 :
1157 : case JSTRACE_BASE_SHAPE:
1158 1145720 : MarkChildren(trc, static_cast<BaseShape *>(thing));
1159 1145720 : break;
1160 :
1161 : case JSTRACE_TYPE_OBJECT:
1162 409827 : MarkChildren(trc, (types::TypeObject *)thing);
1163 409827 : break;
1164 :
1165 : #if JS_HAS_XML_SUPPORT
1166 : case JSTRACE_XML:
1167 0 : MarkChildren(trc, static_cast<JSXML *>(thing));
1168 0 : break;
1169 : #endif
1170 : }
1171 26044482 : }
1172 :
1173 : void
1174 8642335 : CallTracer(JSTracer *trc, void *thing, JSGCTraceKind kind)
1175 : {
1176 8642335 : JS_ASSERT(thing);
1177 8642335 : MarkKind(trc, thing, kind);
1178 8642335 : }
1179 :
1180 : } /* namespace js */
|