1 : /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=4 sw=4 et tw=99:
3 : *
4 : * ***** BEGIN LICENSE BLOCK *****
5 : * Version: MPL 1.1/GPL 2.0/LGPL 2.1
6 : *
7 : * The contents of this file are subject to the Mozilla Public License Version
8 : * 1.1 (the "License"); you may not use this file except in compliance with
9 : * the License. You may obtain a copy of the License at
10 : * http://www.mozilla.org/MPL/
11 : *
12 : * Software distributed under the License is distributed on an "AS IS" basis,
13 : * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
14 : * for the specific language governing rights and limitations under the
15 : * License.
16 : *
17 : * The Original Code is Mozilla Jaegermonkey.
18 : *
19 : * The Initial Developer of the Original Code is the Mozilla Foundation.
20 : *
21 : * Portions created by the Initial Developer are Copyright (C) 2010
22 : * the Initial Developer. All Rights Reserved.
23 : *
24 : * Contributor(s):
25 : * Andrew Drake <drakedevel@gmail.com>
26 : *
27 : * Alternatively, the contents of this file may be used under the terms of
28 : * either the GNU General Public License Version 2 or later (the "GPL"), or
29 : * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
30 : * in which case the provisions of the GPL or the LGPL are applicable instead
31 : * of those above. If you wish to allow use of your version of this file only
32 : * under the terms of either the GPL or the LGPL, and not to allow others to
33 : * use your version of this file under the terms of the MPL, indicate your
34 : * decision by deleting the provisions above and replace them with the notice
35 : * and other provisions required by the GPL or the LGPL. If you do not delete
36 : * the provisions above, a recipient may use your version of this file under
37 : * the terms of any one of the MPL, the GPL or the LGPL.
38 : *
39 : * ***** END LICENSE BLOCK ***** */
40 :
41 : #ifdef JS_METHODJIT
42 :
43 : #include "Retcon.h"
44 : #include "MethodJIT.h"
45 : #include "Compiler.h"
46 : #include "StubCalls.h"
47 : #include "jsdbgapi.h"
48 : #include "jsnum.h"
49 : #include "assembler/assembler/LinkBuffer.h"
50 : #include "assembler/assembler/RepatchBuffer.h"
51 :
52 : #include "jscntxtinlines.h"
53 : #include "jsinterpinlines.h"
54 :
55 : using namespace js;
56 : using namespace js::mjit;
57 :
58 : namespace js {
59 : namespace mjit {
60 :
61 : static inline void
62 110077 : SetRejoinState(StackFrame *fp, const CallSite &site, void **location)
63 : {
64 110077 : if (site.rejoin == REJOIN_SCRIPTED) {
65 70773 : fp->setRejoin(ScriptedRejoin(site.pcOffset));
66 70773 : *location = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolineScripted);
67 : } else {
68 39304 : fp->setRejoin(StubRejoin(site.rejoin));
69 39304 : *location = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline);
70 : }
71 110077 : }
72 :
73 : static inline bool
74 1232765 : CallsiteMatches(uint8_t *codeStart, const CallSite &site, void *location)
75 : {
76 1232765 : if (codeStart + site.codeOffset == location)
77 110077 : return true;
78 :
79 : #ifdef JS_CPU_ARM
80 : if (codeStart + site.codeOffset + 4 == location)
81 : return true;
82 : #endif
83 :
84 1122688 : return false;
85 : }
86 :
87 : void
88 110005 : Recompiler::patchCall(JITChunk *chunk, StackFrame *fp, void **location)
89 : {
90 110005 : uint8_t* codeStart = (uint8_t *)chunk->code.m_code.executableAddress();
91 :
92 110005 : CallSite *callSites_ = chunk->callSites();
93 1232636 : for (uint32_t i = 0; i < chunk->nCallSites; i++) {
94 1232636 : if (CallsiteMatches(codeStart, callSites_[i], *location)) {
95 110005 : JS_ASSERT(callSites_[i].inlineIndex == analyze::CrossScriptSSA::OUTER_FRAME);
96 110005 : SetRejoinState(fp, callSites_[i], location);
97 : return;
98 : }
99 : }
100 :
101 0 : JS_NOT_REACHED("failed to find call site");
102 : }
103 :
104 : void
105 1453 : Recompiler::patchNative(JSCompartment *compartment, JITChunk *chunk, StackFrame *fp,
106 : jsbytecode *pc, RejoinState rejoin)
107 : {
108 : /*
109 : * There is a native call or getter IC at pc which triggered recompilation.
110 : * The recompilation could have been triggered either by the native call
111 : * itself, or by a SplatApplyArgs preparing for the native call. Either
112 : * way, we don't want to patch up the call, but will instead steal the pool
113 : * for the IC so it doesn't get freed with the JITChunk, and patch up the
114 : * jump at the end to go to the interpoline.
115 : *
116 : * When doing this, we do not reset the the IC itself; the JITChunk must
117 : * be dead and about to be released due to the recompilation (or a GC).
118 : */
119 1453 : fp->setRejoin(StubRejoin(rejoin));
120 :
121 : /* :XXX: We might crash later if this fails. */
122 1453 : compartment->jaegerCompartment()->orphanedNativeFrames.append(fp);
123 :
124 2906 : DebugOnly<bool> found = false;
125 :
126 : /*
127 : * Find and patch all native call stubs attached to the given PC. There may
128 : * be multiple ones for getter stubs attached to e.g. a GETELEM.
129 : */
130 8474 : for (unsigned i = 0; i < chunk->nativeCallStubs.length(); i++) {
131 7021 : NativeCallStub &stub = chunk->nativeCallStubs[i];
132 7021 : if (stub.pc != pc)
133 5568 : continue;
134 :
135 1453 : found = true;
136 :
137 : /* Check for pools that were already patched. */
138 1453 : if (!stub.pool)
139 590 : continue;
140 :
141 : /* Patch the native fallthrough to go to the interpoline. */
142 : {
143 : #if (defined(JS_NO_FASTCALL) && defined(JS_CPU_X86)) || defined(_WIN64)
144 : /* Win64 needs stack adjustment */
145 : void *interpoline = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolinePatched);
146 : #else
147 863 : void *interpoline = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline);
148 : #endif
149 863 : uint8_t *start = (uint8_t *)stub.jump.executableAddress();
150 1726 : JSC::RepatchBuffer repatch(JSC::JITCode(start - 32, 64));
151 : #ifdef JS_CPU_X64
152 : repatch.repatch(stub.jump, interpoline);
153 : #else
154 863 : repatch.relink(stub.jump, JSC::CodeLocationLabel(interpoline));
155 : #endif
156 : }
157 :
158 : /* :XXX: We leak the pool if this fails. Oh well. */
159 863 : compartment->jaegerCompartment()->orphanedNativePools.append(stub.pool);
160 :
161 : /* Mark as stolen in case there are multiple calls on the stack. */
162 863 : stub.pool = NULL;
163 : }
164 :
165 1453 : JS_ASSERT(found);
166 1453 : }
167 :
168 : void
169 171101 : Recompiler::patchFrame(JSCompartment *compartment, VMFrame *f, JSScript *script)
170 : {
171 : /*
172 : * Check if the VMFrame returns directly into the script's jitcode. This
173 : * depends on the invariant that f->fp() reflects the frame at the point
174 : * where the call occurred, irregardless of any frames which were pushed
175 : * inside the call.
176 : */
177 171101 : StackFrame *fp = f->fp();
178 171101 : void **addr = f->returnAddressLocation();
179 171101 : RejoinState rejoin = (RejoinState) f->stubRejoin;
180 171101 : if (rejoin == REJOIN_NATIVE ||
181 : rejoin == REJOIN_NATIVE_LOWERED ||
182 : rejoin == REJOIN_NATIVE_GETTER) {
183 : /* Native call. */
184 2926 : if (fp->script() == script) {
185 1453 : patchNative(compartment, fp->jit()->chunk(f->regs.pc), fp, f->regs.pc, rejoin);
186 1453 : f->stubRejoin = REJOIN_NATIVE_PATCHED;
187 : }
188 169638 : } else if (rejoin == REJOIN_NATIVE_PATCHED) {
189 : /* Already patched, don't do anything. */
190 169047 : } else if (rejoin) {
191 : /* Recompilation triggered by CompileFunction. */
192 19 : if (fp->script() == script) {
193 0 : fp->setRejoin(StubRejoin(rejoin));
194 0 : *addr = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline);
195 0 : f->stubRejoin = 0;
196 : }
197 : } else {
198 169028 : if (script->jitCtor) {
199 325 : JITChunk *chunk = script->jitCtor->findCodeChunk(*addr);
200 325 : if (chunk)
201 311 : patchCall(chunk, fp, addr);
202 : }
203 169028 : if (script->jitNormal) {
204 44620 : JITChunk *chunk = script->jitNormal->findCodeChunk(*addr);
205 44620 : if (chunk)
206 38921 : patchCall(chunk, fp, addr);
207 : }
208 : }
209 171101 : }
210 :
211 : StackFrame *
212 140 : Recompiler::expandInlineFrameChain(StackFrame *outer, InlineFrame *inner)
213 : {
214 : StackFrame *parent;
215 140 : if (inner->parent)
216 11 : parent = expandInlineFrameChain(outer, inner->parent);
217 : else
218 129 : parent = outer;
219 :
220 140 : JaegerSpew(JSpew_Recompile, "Expanding inline frame\n");
221 :
222 140 : StackFrame *fp = (StackFrame *) ((uint8_t *)outer + sizeof(Value) * inner->depth);
223 140 : fp->initInlineFrame(inner->fun, parent, inner->parentpc);
224 140 : uint32_t pcOffset = inner->parentpc - parent->script()->code;
225 :
226 140 : void **location = fp->addressOfNativeReturnAddress();
227 140 : *location = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolineScripted);
228 140 : parent->setRejoin(ScriptedRejoin(pcOffset));
229 :
230 140 : return fp;
231 : }
232 :
233 : /*
234 : * Whether a given return address for a frame indicates it returns directly
235 : * into JIT code.
236 : */
237 : static inline bool
238 70945 : JITCodeReturnAddress(void *data)
239 : {
240 : return data != NULL /* frame is interpreted */
241 : && data != JS_FUNC_TO_DATA_PTR(void *, JaegerTrampolineReturn)
242 : && data != JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline)
243 : #if (defined(JS_NO_FASTCALL) && defined(JS_CPU_X86)) || defined(_WIN64)
244 : && data != JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolinePatched)
245 : #endif
246 70945 : && data != JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolineScripted);
247 : }
248 :
249 : /*
250 : * Expand all inlined frames within fp per 'inlined' and update next and regs
251 : * to refer to the new innermost frame.
252 : */
253 : void
254 129 : Recompiler::expandInlineFrames(JSCompartment *compartment,
255 : StackFrame *fp, mjit::CallSite *inlined,
256 : StackFrame *next, VMFrame *f)
257 : {
258 129 : JS_ASSERT_IF(next, next->prev() == fp && next->prevInline() == inlined);
259 :
260 : /*
261 : * Treat any frame expansion as a recompilation event, so that f.jit() is
262 : * stable if no recompilations have occurred.
263 : */
264 129 : compartment->types.frameExpansions++;
265 :
266 129 : jsbytecode *pc = next ? next->prevpc(NULL) : f->regs.pc;
267 129 : JITChunk *chunk = fp->jit()->chunk(pc);
268 :
269 : /*
270 : * Patch the VMFrame's return address if it is returning at the given inline site.
271 : * Note there is no worry about handling a native or CompileFunction call here,
272 : * as such IC stubs are not generated within inline frames.
273 : */
274 129 : void **frameAddr = f->returnAddressLocation();
275 129 : uint8_t* codeStart = (uint8_t *)chunk->code.m_code.executableAddress();
276 :
277 129 : InlineFrame *inner = &chunk->inlineFrames()[inlined->inlineIndex];
278 129 : jsbytecode *innerpc = inner->fun->script()->code + inlined->pcOffset;
279 :
280 129 : StackFrame *innerfp = expandInlineFrameChain(fp, inner);
281 :
282 : /* Check if the VMFrame returns into the inlined frame. */
283 129 : if (f->stubRejoin && f->fp() == fp) {
284 : /* The VMFrame is calling CompileFunction. */
285 0 : JS_ASSERT(f->stubRejoin != REJOIN_NATIVE &&
286 : f->stubRejoin != REJOIN_NATIVE_LOWERED &&
287 : f->stubRejoin != REJOIN_NATIVE_GETTER &&
288 0 : f->stubRejoin != REJOIN_NATIVE_PATCHED);
289 0 : innerfp->setRejoin(StubRejoin((RejoinState) f->stubRejoin));
290 0 : *frameAddr = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline);
291 0 : f->stubRejoin = 0;
292 : }
293 129 : if (CallsiteMatches(codeStart, *inlined, *frameAddr)) {
294 : /* The VMFrame returns directly into the expanded frame. */
295 72 : SetRejoinState(innerfp, *inlined, frameAddr);
296 : }
297 :
298 129 : if (f->fp() == fp) {
299 74 : JS_ASSERT(f->regs.inlined() == inlined);
300 74 : f->regs.expandInline(innerfp, innerpc);
301 : }
302 :
303 : /*
304 : * Note: unlike the case for recompilation, during frame expansion we don't
305 : * need to worry about the next VMFrame holding a reference to the inlined
306 : * frame in its entryncode. entryncode is non-NULL only if the next frame's
307 : * code was discarded and has executed via the Interpoline, which can only
308 : * happen after all inline frames have been expanded.
309 : */
310 :
311 129 : if (next) {
312 55 : next->resetInlinePrev(innerfp, innerpc);
313 55 : void **addr = next->addressOfNativeReturnAddress();
314 55 : if (JITCodeReturnAddress(*addr)) {
315 55 : innerfp->setRejoin(ScriptedRejoin(inlined->pcOffset));
316 55 : *addr = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolineScripted);
317 : }
318 : }
319 129 : }
320 :
321 : void
322 3839701 : ExpandInlineFrames(JSCompartment *compartment)
323 : {
324 3839701 : if (!compartment || !compartment->hasJaegerCompartment())
325 373091 : return;
326 :
327 10029549 : for (VMFrame *f = compartment->jaegerCompartment()->activeFrame();
328 : f != NULL;
329 : f = f->previous) {
330 :
331 6562939 : if (f->regs.inlined())
332 74 : mjit::Recompiler::expandInlineFrames(compartment, f->fp(), f->regs.inlined(), NULL, f);
333 :
334 6562939 : StackFrame *end = f->entryfp->prev();
335 6562939 : StackFrame *next = NULL;
336 15214971 : for (StackFrame *fp = f->fp(); fp != end; fp = fp->prev()) {
337 8979203 : if (!next) {
338 6562994 : next = fp;
339 6562994 : continue;
340 : }
341 : mjit::CallSite *inlined;
342 2416209 : next->prevpc(&inlined);
343 2416209 : if (inlined) {
344 55 : mjit::Recompiler::expandInlineFrames(compartment, fp, inlined, next, f);
345 55 : fp = next;
346 55 : next = NULL;
347 : } else {
348 2416154 : if (fp->downFramesExpanded())
349 327171 : break;
350 2088983 : next = fp;
351 : }
352 2089038 : fp->setDownFramesExpanded();
353 : }
354 : }
355 : }
356 :
357 : void
358 133015 : ClearAllFrames(JSCompartment *compartment)
359 : {
360 133015 : if (!compartment || !compartment->hasJaegerCompartment())
361 100634 : return;
362 :
363 32381 : ExpandInlineFrames(compartment);
364 :
365 163412 : for (VMFrame *f = compartment->jaegerCompartment()->activeFrame();
366 : f != NULL;
367 : f = f->previous) {
368 :
369 131031 : Recompiler::patchFrame(compartment, f, f->fp()->script());
370 :
371 : // Clear ncode values from all frames associated with the VMFrame.
372 : // Patching the VMFrame's return address will cause all its frames to
373 : // finish in the interpreter, unless the interpreter enters one of the
374 : // intermediate frames at a loop boundary (where EnterMethodJIT will
375 : // overwrite ncode). However, leaving stale values for ncode in stack
376 : // frames can confuse the recompiler, which may see the VMFrame before
377 : // it has resumed execution.
378 :
379 184491 : for (StackFrame *fp = f->fp(); fp != f->entryfp; fp = fp->prev())
380 53460 : fp->setNativeReturnAddress(NULL);
381 : }
382 : }
383 :
384 : /*
385 : * Recompilation can be triggered either by the debugger (turning debug mode on for
386 : * a script or setting/clearing a trap), or by dynamic changes in type information
387 : * from type inference. When recompiling we don't immediately recompile the JIT
388 : * code, but destroy the old code and remove all references to the code, including
389 : * those from active stack frames. Things to do:
390 : *
391 : * - Purge scripted call inline caches calling into the script.
392 : *
393 : * - For frames with an ncode return address in the original script, redirect
394 : * to the interpoline.
395 : *
396 : * - For VMFrames with a stub call return address in the original script,
397 : * redirect to the interpoline.
398 : *
399 : * - For VMFrames whose entryncode address (the value of entryfp->ncode before
400 : * being clobbered with JaegerTrampolineReturn) is in the original script,
401 : * redirect that entryncode to the interpoline.
402 : */
403 : void
404 34144 : Recompiler::clearStackReferences(JSContext *cx, JSScript *script)
405 : {
406 34144 : JS_ASSERT(script->hasJITCode());
407 :
408 : JaegerSpew(JSpew_Recompile, "recompiling script (file \"%s\") (line \"%d\") (length \"%d\")\n",
409 34144 : script->filename, script->lineno, script->length);
410 :
411 68288 : types::AutoEnterTypeInference enter(cx, true);
412 :
413 : /*
414 : * The strategy for this goes as follows:
415 : *
416 : * 1) Scan the stack, looking at all return addresses that could go into JIT
417 : * code.
418 : * 2) If an address corresponds to a call site registered by |callSite| during
419 : * the last compilation, patch it to go to the interpoline.
420 : * 3) Purge the old compiled state.
421 : */
422 :
423 : // Find all JIT'd stack frames to account for return addresses that will
424 : // need to be patched after recompilation.
425 74214 : for (VMFrame *f = script->compartment()->jaegerCompartment()->activeFrame();
426 : f != NULL;
427 : f = f->previous) {
428 :
429 : // Scan all frames owned by this VMFrame.
430 40070 : StackFrame *end = f->entryfp->prev();
431 40070 : StackFrame *next = NULL;
432 189159 : for (StackFrame *fp = f->fp(); fp != end; fp = fp->prev()) {
433 149089 : if (fp->script() != script) {
434 43730 : next = fp;
435 43730 : continue;
436 : }
437 :
438 105359 : if (next) {
439 : // check for a scripted call returning into the recompiled script.
440 : // this misses scanning the entry fp, which cannot return directly
441 : // into JIT code.
442 70890 : void **addr = next->addressOfNativeReturnAddress();
443 :
444 70890 : if (JITCodeReturnAddress(*addr)) {
445 70773 : JITChunk *chunk = fp->jit()->findCodeChunk(*addr);
446 70773 : patchCall(chunk, fp, addr);
447 : }
448 : }
449 :
450 105359 : next = fp;
451 : }
452 :
453 40070 : patchFrame(cx->compartment, f, script);
454 : }
455 :
456 34144 : cx->compartment->types.recompilations++;
457 34144 : }
458 :
459 : void
460 33526 : Recompiler::clearStackReferencesAndChunk(JSContext *cx, JSScript *script,
461 : JITScript *jit, size_t chunkIndex,
462 : bool resetUses)
463 : {
464 33526 : Recompiler::clearStackReferences(cx, script);
465 :
466 33526 : bool releaseChunk = true;
467 33526 : if (jit->nchunks > 1) {
468 : // If we are in the middle of a native call from a native or getter IC,
469 : // we need to make sure all JIT code for the script is purged, as
470 : // otherwise we will have orphaned the native stub but pointers to it
471 : // still exist in the containing chunk.
472 1420 : for (VMFrame *f = cx->compartment->jaegerCompartment()->activeFrame();
473 : f != NULL;
474 : f = f->previous) {
475 677 : if (f->fp()->script() == script) {
476 0 : JS_ASSERT(f->stubRejoin != REJOIN_NATIVE &&
477 : f->stubRejoin != REJOIN_NATIVE_LOWERED &&
478 674 : f->stubRejoin != REJOIN_NATIVE_GETTER);
479 674 : if (f->stubRejoin == REJOIN_NATIVE_PATCHED) {
480 95 : mjit::ReleaseScriptCode(cx, script);
481 95 : releaseChunk = false;
482 95 : break;
483 : }
484 : }
485 : }
486 : }
487 :
488 33526 : if (releaseChunk)
489 33431 : jit->destroyChunk(cx, chunkIndex, resetUses);
490 33526 : }
491 :
492 : } /* namespace mjit */
493 : } /* namespace js */
494 :
495 : #endif /* JS_METHODJIT */
496 :
|