Merge "ART: Improve JitProfiling perf in mips/mips64 mterp."
diff --git a/runtime/interpreter/mterp/mips/bincmp.S b/runtime/interpreter/mterp/mips/bincmp.S
index 70057f6..68df5c3 100644
--- a/runtime/interpreter/mterp/mips/bincmp.S
+++ b/runtime/interpreter/mterp/mips/bincmp.S
@@ -1,7 +1,6 @@
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -9,29 +8,11 @@
GET_OPA4(a0) # a0 <- A+
GET_OPB(a1) # a1 <- B
GET_VREG(a3, a1) # a3 <- vB
- GET_VREG(a2, a0) # a2 <- vA
- b${revcmp} a2, a3, 1f # branch to 1 if comparison failed
+ GET_VREG(a0, a0) # a0 <- vA
FETCH_S(rINST, 1) # rINST<- branch offset, in code units
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a2, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- bgez a2, .L_${opcode}_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-
-%break
-
-.L_${opcode}_finish:
+ b${condition} a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB)
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
diff --git a/runtime/interpreter/mterp/mips/entry.S b/runtime/interpreter/mterp/mips/entry.S
index 5771a4f..c806a67 100644
--- a/runtime/interpreter/mterp/mips/entry.S
+++ b/runtime/interpreter/mterp/mips/entry.S
@@ -60,6 +60,12 @@
/* Starting ibase */
lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF)
+ /* Set up for backwards branches & osr profiling */
+ lw a0, OFF_FP_METHOD(rFP)
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ JAL(MterpSetUpHotnessCountdown) # (method, shadow_frame)
+ move rPROFILE, v0 # Starting hotness countdown to rPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST() # load rINST from rPC
GET_INST_OPCODE(t0) # extract opcode from rINST
diff --git a/runtime/interpreter/mterp/mips/footer.S b/runtime/interpreter/mterp/mips/footer.S
index 083dc15..1363751 100644
--- a/runtime/interpreter/mterp/mips/footer.S
+++ b/runtime/interpreter/mterp/mips/footer.S
@@ -112,20 +112,110 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in lr.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * rPROFILE <= signed hotness countdown (expanded to 32 bits)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
*/
-MterpCheckSuspendAndContinue:
- lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF) # refresh rIBASE
+MterpCommonTakenBranchNoFlags:
+ bgtz rINST, .L_forward_branch # don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+#if JIT_CHECK_OSR != -1
+# error "JIT_CHECK_OSR must be -1."
+#endif
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_osr_check
+ blt rPROFILE, t0, .L_resume_backward_branch
+ subu rPROFILE, 1
+ beqz rPROFILE, .L_add_batch # counted down to zero - report
+.L_resume_backward_branch:
+ lw ra, THREAD_FLAGS_OFFSET(rSELF)
+ REFRESH_IBASE()
+ addu a2, rINST, rINST # a2<- byte offset
+ FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
and ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
- bnez ra, 1f
+ bnez ra, .L_suspend_request_pending
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
-1:
+
+.L_suspend_request_pending:
EXPORT_PC()
move a0, rSELF
JAL(MterpSuspendCheck) # (self)
bnez v0, MterpFallback
+ REFRESH_IBASE() # might have changed during suspend
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
+
+.L_no_count_backwards:
+ li t0, JIT_CHECK_OSR # check for possible OSR re-entry
+ bne rPROFILE, t0, .L_resume_backward_branch
+.L_osr_check:
+ move a0, rSELF
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rINST
+ EXPORT_PC()
+ JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ bnez v0, MterpOnStackReplacement
+ b .L_resume_backward_branch
+
+.L_forward_branch:
+ li t0, JIT_CHECK_OSR # check for possible OSR re-entry
+ beq rPROFILE, t0, .L_check_osr_forward
+.L_resume_forward_branch:
+ add a2, rINST, rINST # a2<- byte offset
+ FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
+
+.L_check_osr_forward:
+ move a0, rSELF
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rINST
+ EXPORT_PC()
+ JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ bnez v0, MterpOnStackReplacement
+ b .L_resume_forward_branch
+
+.L_add_batch:
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1)
+ lw a0, OFF_FP_METHOD(rFP)
+ move a2, rSELF
+ JAL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ move rPROFILE, v0 # restore new hotness countdown to rPROFILE
+ b .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ move a0, rSELF
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ li a2, 2
+ EXPORT_PC()
+ JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ bnez v0, MterpOnStackReplacement
+ FETCH_ADVANCE_INST(2)
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
@@ -172,6 +262,26 @@
sw v1, 4(a2)
li v0, 1 # signal return to caller.
MterpDone:
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ blez rPROFILE, .L_pop_and_return # if > 0, we may have some counts to report.
+
+MterpProfileActive:
+ move rINST, v0 # stash return value
+ /* Report cached hotness counts */
+ lw a0, OFF_FP_METHOD(rFP)
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rSELF
+ sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1)
+ JAL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ move v0, rINST # restore return value
+
+.L_pop_and_return:
/* Restore from the stack and return. Frame size = STACK_SIZE */
STACK_LOAD_FULL()
jalr zero, ra
diff --git a/runtime/interpreter/mterp/mips/header.S b/runtime/interpreter/mterp/mips/header.S
index 37ab21d..a3a6744 100644
--- a/runtime/interpreter/mterp/mips/header.S
+++ b/runtime/interpreter/mterp/mips/header.S
@@ -51,7 +51,11 @@
s2 rSELF self (Thread) pointer
s3 rIBASE interpreted instruction base pointer, used for computed goto
s4 rINST first 16-bit code unit of current instruction
+ s5 rOBJ object pointer
s6 rREFS base of object references in shadow frame (ideally, we'll get rid of this later).
+ s7 rTEMP used as temp storage that can survive a function call
+ s8 rPROFILE branch profiling countdown
+
*/
/* single-purpose registers, given names for clarity */
@@ -63,6 +67,7 @@
#define rOBJ s5
#define rREFS s6
#define rTEMP s7
+#define rPROFILE s8
#define rARG0 a0
#define rARG1 a1
@@ -160,7 +165,7 @@
#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
#define MTERP_PROFILE_BRANCHES 1
#define MTERP_LOGGING 0
@@ -482,3 +487,6 @@
STACK_LOAD(s8, 120); \
STACK_LOAD(ra, 124); \
DELETE_STACK(STACK_SIZE)
+
+#define REFRESH_IBASE() \
+ lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF)
diff --git a/runtime/interpreter/mterp/mips/op_goto.S b/runtime/interpreter/mterp/mips/op_goto.S
index d6f21c9..57182a5 100644
--- a/runtime/interpreter/mterp/mips/op_goto.S
+++ b/runtime/interpreter/mterp/mips/op_goto.S
@@ -5,34 +5,6 @@
* double to get a byte offset.
*/
/* goto +AA */
-#if MTERP_PROFILE_BRANCHES
sll a0, rINST, 16 # a0 <- AAxx0000
sra rINST, a0, 24 # rINST <- ssssssAA (sign-extended)
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a2, rINST, rINST # a2 <- byte offset
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- /* If backwards branch refresh rIBASE */
- bgez a2, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#else
- sll a0, rINST, 16 # a0 <- AAxx0000
- sra rINST, a0, 24 # rINST <- ssssssAA (sign-extended)
- addu a2, rINST, rINST # a2 <- byte offset
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- /* If backwards branch refresh rIBASE */
- bgez a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/mips/op_goto_16.S b/runtime/interpreter/mterp/mips/op_goto_16.S
index cec4432..06c96cd 100644
--- a/runtime/interpreter/mterp/mips/op_goto_16.S
+++ b/runtime/interpreter/mterp/mips/op_goto_16.S
@@ -5,30 +5,5 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
-#if MTERP_PROFILE_BRANCHES
FETCH_S(rINST, 1) # rINST <- ssssAAAA (sign-extended)
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a1, rINST, rINST # a1 <- byte offset, flags set
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#else
- FETCH_S(rINST, 1) # rINST <- ssssAAAA (sign-extended)
- addu a1, rINST, rINST # a1 <- byte offset, flags set
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/mips/op_goto_32.S b/runtime/interpreter/mterp/mips/op_goto_32.S
index 083acd1..67f52e9 100644
--- a/runtime/interpreter/mterp/mips/op_goto_32.S
+++ b/runtime/interpreter/mterp/mips/op_goto_32.S
@@ -8,36 +8,8 @@
* our "backward branch" test must be "<=0" instead of "<0".
*/
/* goto/32 +AAAAAAAA */
-#if MTERP_PROFILE_BRANCHES
FETCH(a0, 1) # a0 <- aaaa (lo)
FETCH(a1, 2) # a1 <- AAAA (hi)
sll a1, a1, 16
or rINST, a0, a1 # rINST <- AAAAaaaa
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#else
- FETCH(a0, 1) # a0 <- aaaa (lo)
- FETCH(a1, 2) # a1 <- AAAA (hi)
- sll a1, a1, 16
- or rINST, a0, a1 # rINST <- AAAAaaaa
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/mips/op_if_eq.S b/runtime/interpreter/mterp/mips/op_if_eq.S
index e7190d8..d6f9987 100644
--- a/runtime/interpreter/mterp/mips/op_if_eq.S
+++ b/runtime/interpreter/mterp/mips/op_if_eq.S
@@ -1 +1 @@
-%include "mips/bincmp.S" { "revcmp":"ne" }
+%include "mips/bincmp.S" { "condition":"eq" }
diff --git a/runtime/interpreter/mterp/mips/op_if_eqz.S b/runtime/interpreter/mterp/mips/op_if_eqz.S
index 0a78fd9..c52b76a 100644
--- a/runtime/interpreter/mterp/mips/op_if_eqz.S
+++ b/runtime/interpreter/mterp/mips/op_if_eqz.S
@@ -1 +1 @@
-%include "mips/zcmp.S" { "revcmp":"ne" }
+%include "mips/zcmp.S" { "condition":"eq" }
diff --git a/runtime/interpreter/mterp/mips/op_if_ge.S b/runtime/interpreter/mterp/mips/op_if_ge.S
index b2629ba..bd06ff5 100644
--- a/runtime/interpreter/mterp/mips/op_if_ge.S
+++ b/runtime/interpreter/mterp/mips/op_if_ge.S
@@ -1 +1 @@
-%include "mips/bincmp.S" { "revcmp":"lt" }
+%include "mips/bincmp.S" { "condition":"ge" }
diff --git a/runtime/interpreter/mterp/mips/op_if_gez.S b/runtime/interpreter/mterp/mips/op_if_gez.S
index b02f677..549231a 100644
--- a/runtime/interpreter/mterp/mips/op_if_gez.S
+++ b/runtime/interpreter/mterp/mips/op_if_gez.S
@@ -1 +1 @@
-%include "mips/zcmp.S" { "revcmp":"lt" }
+%include "mips/zcmp.S" { "condition":"ge" }
diff --git a/runtime/interpreter/mterp/mips/op_if_gt.S b/runtime/interpreter/mterp/mips/op_if_gt.S
index f620d4a..0be3091 100644
--- a/runtime/interpreter/mterp/mips/op_if_gt.S
+++ b/runtime/interpreter/mterp/mips/op_if_gt.S
@@ -1 +1 @@
-%include "mips/bincmp.S" { "revcmp":"le" }
+%include "mips/bincmp.S" { "condition":"gt" }
diff --git a/runtime/interpreter/mterp/mips/op_if_gtz.S b/runtime/interpreter/mterp/mips/op_if_gtz.S
index 5e5dd70..5c7bcc4 100644
--- a/runtime/interpreter/mterp/mips/op_if_gtz.S
+++ b/runtime/interpreter/mterp/mips/op_if_gtz.S
@@ -1 +1 @@
-%include "mips/zcmp.S" { "revcmp":"le" }
+%include "mips/zcmp.S" { "condition":"gt" }
diff --git a/runtime/interpreter/mterp/mips/op_if_le.S b/runtime/interpreter/mterp/mips/op_if_le.S
index a4e8b1a..c35c1a2 100644
--- a/runtime/interpreter/mterp/mips/op_if_le.S
+++ b/runtime/interpreter/mterp/mips/op_if_le.S
@@ -1 +1 @@
-%include "mips/bincmp.S" { "revcmp":"gt" }
+%include "mips/bincmp.S" { "condition":"le" }
diff --git a/runtime/interpreter/mterp/mips/op_if_lez.S b/runtime/interpreter/mterp/mips/op_if_lez.S
index af551a6..3dc6543 100644
--- a/runtime/interpreter/mterp/mips/op_if_lez.S
+++ b/runtime/interpreter/mterp/mips/op_if_lez.S
@@ -1 +1 @@
-%include "mips/zcmp.S" { "revcmp":"gt" }
+%include "mips/zcmp.S" { "condition":"le" }
diff --git a/runtime/interpreter/mterp/mips/op_if_lt.S b/runtime/interpreter/mterp/mips/op_if_lt.S
index f33b9a4..3f3386c 100644
--- a/runtime/interpreter/mterp/mips/op_if_lt.S
+++ b/runtime/interpreter/mterp/mips/op_if_lt.S
@@ -1 +1 @@
-%include "mips/bincmp.S" { "revcmp":"ge" }
+%include "mips/bincmp.S" { "condition":"lt" }
diff --git a/runtime/interpreter/mterp/mips/op_if_ltz.S b/runtime/interpreter/mterp/mips/op_if_ltz.S
index 18fcb1d..e6d6ed6 100644
--- a/runtime/interpreter/mterp/mips/op_if_ltz.S
+++ b/runtime/interpreter/mterp/mips/op_if_ltz.S
@@ -1 +1 @@
-%include "mips/zcmp.S" { "revcmp":"ge" }
+%include "mips/zcmp.S" { "condition":"lt" }
diff --git a/runtime/interpreter/mterp/mips/op_if_ne.S b/runtime/interpreter/mterp/mips/op_if_ne.S
index e0a102b..3d7bf35 100644
--- a/runtime/interpreter/mterp/mips/op_if_ne.S
+++ b/runtime/interpreter/mterp/mips/op_if_ne.S
@@ -1 +1 @@
-%include "mips/bincmp.S" { "revcmp":"eq" }
+%include "mips/bincmp.S" { "condition":"ne" }
diff --git a/runtime/interpreter/mterp/mips/op_if_nez.S b/runtime/interpreter/mterp/mips/op_if_nez.S
index d1866a0..d121eae 100644
--- a/runtime/interpreter/mterp/mips/op_if_nez.S
+++ b/runtime/interpreter/mterp/mips/op_if_nez.S
@@ -1 +1 @@
-%include "mips/zcmp.S" { "revcmp":"eq" }
+%include "mips/zcmp.S" { "condition":"ne" }
diff --git a/runtime/interpreter/mterp/mips/op_packed_switch.S b/runtime/interpreter/mterp/mips/op_packed_switch.S
index 93fae97..ffa4f47 100644
--- a/runtime/interpreter/mterp/mips/op_packed_switch.S
+++ b/runtime/interpreter/mterp/mips/op_packed_switch.S
@@ -9,7 +9,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
FETCH(a0, 1) # a0 <- bbbb (lo)
FETCH(a1, 2) # a1 <- BBBB (hi)
GET_OPA(a3) # a3 <- AA
@@ -19,39 +18,4 @@
EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2
JAL($func) # a0 <- code-unit branch offset
move rINST, v0
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, .L${opcode}_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-#else
- FETCH(a0, 1) # a0 <- bbbb (lo)
- FETCH(a1, 2) # a1 <- BBBB (hi)
- GET_OPA(a3) # a3 <- AA
- sll t0, a1, 16
- or a0, a0, t0 # a0 <- BBBBbbbb
- GET_VREG(a1, a3) # a1 <- vAA
- EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2
- JAL($func) # a0 <- code-unit branch offset
- move rINST, v0
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
-
-%break
-
-.L${opcode}_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/mips/zcmp.S b/runtime/interpreter/mterp/mips/zcmp.S
index 1fa1385..8d3a198 100644
--- a/runtime/interpreter/mterp/mips/zcmp.S
+++ b/runtime/interpreter/mterp/mips/zcmp.S
@@ -1,32 +1,16 @@
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
GET_OPA(a0) # a0 <- AA
- GET_VREG(a2, a0) # a2 <- vAA
+ GET_VREG(a0, a0) # a0 <- vAA
FETCH_S(rINST, 1) # rINST <- branch offset, in code units
- b${revcmp} a2, zero, 1f # branch to 1 if comparison failed
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a1, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 3f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-3:
+ b${condition} a0, zero, MterpCommonTakenBranchNoFlags
+ li t0, JIT_CHECK_OSR # possible OSR re-entry?
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
diff --git a/runtime/interpreter/mterp/mips64/bincmp.S b/runtime/interpreter/mterp/mips64/bincmp.S
index aa5e74b..07b1210 100644
--- a/runtime/interpreter/mterp/mips64/bincmp.S
+++ b/runtime/interpreter/mterp/mips64/bincmp.S
@@ -12,21 +12,9 @@
lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC)
GET_VREG a0, a2 # a0 <- vA
GET_VREG a1, a3 # a1 <- vB
- b${condition}c a0, a1, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ b${condition}c a0, a1, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
diff --git a/runtime/interpreter/mterp/mips64/entry.S b/runtime/interpreter/mterp/mips64/entry.S
index ae6c26b..cc48d45 100644
--- a/runtime/interpreter/mterp/mips64/entry.S
+++ b/runtime/interpreter/mterp/mips64/entry.S
@@ -57,6 +57,8 @@
.cfi_rel_offset 20, STACK_OFFSET_S4
sd s5, STACK_OFFSET_S5(sp)
.cfi_rel_offset 21, STACK_OFFSET_S5
+ sd s6, STACK_OFFSET_S6(sp)
+ .cfi_rel_offset 22, STACK_OFFSET_S6
/* Remember the return register */
sd a3, SHADOWFRAME_RESULT_REGISTER_OFFSET(a2)
@@ -77,6 +79,12 @@
/* Starting ibase */
REFRESH_IBASE
+ /* Set up for backwards branches & osr profiling */
+ ld a0, OFF_FP_METHOD(rFP)
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ jal MterpSetUpHotnessCountdown
+ move rPROFILE, v0 # Starting hotness countdown to rPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST
GET_INST_OPCODE v0
diff --git a/runtime/interpreter/mterp/mips64/footer.S b/runtime/interpreter/mterp/mips64/footer.S
index 14d5fe0..9994169 100644
--- a/runtime/interpreter/mterp/mips64/footer.S
+++ b/runtime/interpreter/mterp/mips64/footer.S
@@ -71,23 +71,110 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in ra.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * rPROFILE <= signed hotness countdown (expanded to 64 bits)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
*/
- .extern MterpSuspendCheck
-MterpCheckSuspendAndContinue:
+MterpCommonTakenBranchNoFlags:
+ bgtzc rINST, .L_forward_branch # don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+ li v0, JIT_CHECK_OSR
+ beqc rPROFILE, v0, .L_osr_check
+ bltc rPROFILE, v0, .L_resume_backward_branch
+ dsubu rPROFILE, 1
+ beqzc rPROFILE, .L_add_batch # counted down to zero - report
+.L_resume_backward_branch:
+ lw ra, THREAD_FLAGS_OFFSET(rSELF)
REFRESH_IBASE
- and ra, ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
- bnez ra, check1
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
-check1:
+ daddu a2, rINST, rINST # a2<- byte offset
+ FETCH_ADVANCE_INST_RB a2 # update rPC, load rINST
+ and ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
+ bnezc ra, .L_suspend_request_pending
+ GET_INST_OPCODE v0 # extract opcode from rINST
+ GOTO_OPCODE v0 # jump to next instruction
+
+.L_suspend_request_pending:
EXPORT_PC
move a0, rSELF
- jal MterpSuspendCheck # (self)
- bnezc v0, MterpFallback # Something in the environment changed, switch interpreters
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ jal MterpSuspendCheck # (self)
+ bnezc v0, MterpFallback
+ REFRESH_IBASE # might have changed during suspend
+ GET_INST_OPCODE v0 # extract opcode from rINST
+ GOTO_OPCODE v0 # jump to next instruction
+
+.L_no_count_backwards:
+ li v0, JIT_CHECK_OSR # check for possible OSR re-entry
+ bnec rPROFILE, v0, .L_resume_backward_branch
+.L_osr_check:
+ move a0, rSELF
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rINST
+ EXPORT_PC
+ jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset)
+ bnezc v0, MterpOnStackReplacement
+ b .L_resume_backward_branch
+
+.L_forward_branch:
+ li v0, JIT_CHECK_OSR # check for possible OSR re-entry
+ beqc rPROFILE, v0, .L_check_osr_forward
+.L_resume_forward_branch:
+ daddu a2, rINST, rINST # a2<- byte offset
+ FETCH_ADVANCE_INST_RB a2 # update rPC, load rINST
+ GET_INST_OPCODE v0 # extract opcode from rINST
+ GOTO_OPCODE v0 # jump to next instruction
+
+.L_check_osr_forward:
+ move a0, rSELF
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rINST
+ EXPORT_PC
+ jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset)
+ bnezc v0, MterpOnStackReplacement
+ b .L_resume_forward_branch
+
+.L_add_batch:
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1)
+ ld a0, OFF_FP_METHOD(rFP)
+ move a2, rSELF
+ jal MterpAddHotnessBatch # (method, shadow_frame, self)
+ move rPROFILE, v0 # restore new hotness countdown to rPROFILE
+ b .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ move a0, rSELF
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ li a2, 2
+ EXPORT_PC
+ jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset)
+ bnezc v0, MterpOnStackReplacement
+ FETCH_ADVANCE_INST 2
+ GET_INST_OPCODE v0 # extract opcode from rINST
+ GOTO_OPCODE v0 # jump to next instruction
/*
* On-stack replacement has happened, and now we've returned from the compiled method.
@@ -143,6 +230,28 @@
check2:
li v0, 1 # signal return to caller.
MterpDone:
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ blez rPROFILE, .L_pop_and_return # if > 0, we may have some counts to report.
+
+MterpProfileActive:
+ move rINST, v0 # stash return value
+ /* Report cached hotness counts */
+ ld a0, OFF_FP_METHOD(rFP)
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rSELF
+ sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1)
+ jal MterpAddHotnessBatch # (method, shadow_frame, self)
+ move v0, rINST # restore return value
+
+.L_pop_and_return:
+ ld s6, STACK_OFFSET_S6(sp)
+ .cfi_restore 22
ld s5, STACK_OFFSET_S5(sp)
.cfi_restore 21
ld s4, STACK_OFFSET_S4(sp)
@@ -169,4 +278,5 @@
.cfi_adjust_cfa_offset -STACK_SIZE
.cfi_endproc
+ .set reorder
.size ExecuteMterpImpl, .-ExecuteMterpImpl
diff --git a/runtime/interpreter/mterp/mips64/header.S b/runtime/interpreter/mterp/mips64/header.S
index dd0fbe0..b67df20 100644
--- a/runtime/interpreter/mterp/mips64/header.S
+++ b/runtime/interpreter/mterp/mips64/header.S
@@ -51,16 +51,18 @@
s3 rINST first 16-bit code unit of current instruction
s4 rIBASE interpreted instruction base pointer, used for computed goto
s5 rREFS base of object references in shadow frame (ideally, we'll get rid of this later).
+ s6 rPROFILE jit profile hotness countdown
*/
/* During bringup, we'll use the shadow frame model instead of rFP */
/* single-purpose registers, given names for clarity */
-#define rPC s0
-#define rFP s1
-#define rSELF s2
-#define rINST s3
-#define rIBASE s4
-#define rREFS s5
+#define rPC s0
+#define rFP s1
+#define rSELF s2
+#define rINST s3
+#define rIBASE s4
+#define rREFS s5
+#define rPROFILE s6
/*
* This is a #include, not a %include, because we want the C pre-processor
@@ -80,7 +82,7 @@
#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
#define MTERP_PROFILE_BRANCHES 1
#define MTERP_LOGGING 0
@@ -121,6 +123,17 @@
.endm
/*
+ * Fetch the next instruction from an offset specified by _reg and advance xPC.
+ * xPC to point to the next instruction. "_reg" must specify the distance
+ * in bytes, *not* 16-bit code units, and may be a signed value. Must not set flags.
+ *
+ */
+.macro FETCH_ADVANCE_INST_RB reg
+ daddu rPC, rPC, \reg
+ FETCH_INST
+.endm
+
+/*
* Fetch the next instruction from the specified offset. Advances rPC
* to point to the next instruction.
*
@@ -267,7 +280,8 @@
#define STACK_OFFSET_S3 40
#define STACK_OFFSET_S4 48
#define STACK_OFFSET_S5 56
-#define STACK_SIZE 64
+#define STACK_OFFSET_S6 64
+#define STACK_SIZE 80 /* needs 16 byte alignment */
/* Constants for float/double_to_int/long conversions */
#define INT_MIN 0x80000000
diff --git a/runtime/interpreter/mterp/mips64/op_goto.S b/runtime/interpreter/mterp/mips64/op_goto.S
index 7c7d0ec..68fc83d 100644
--- a/runtime/interpreter/mterp/mips64/op_goto.S
+++ b/runtime/interpreter/mterp/mips64/op_goto.S
@@ -5,21 +5,6 @@
* double to get a byte offset.
*/
/* goto +AA */
- .extern MterpProfileBranch
srl rINST, rINST, 8
seb rINST, rINST # rINST <- offset (sign-extended AA)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/mips64/op_goto_16.S b/runtime/interpreter/mterp/mips64/op_goto_16.S
index 566e3a7..ae56066 100644
--- a/runtime/interpreter/mterp/mips64/op_goto_16.S
+++ b/runtime/interpreter/mterp/mips64/op_goto_16.S
@@ -5,20 +5,5 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
- .extern MterpProfileBranch
lh rINST, 2(rPC) # rINST <- offset (sign-extended AAAA)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/mips64/op_goto_32.S b/runtime/interpreter/mterp/mips64/op_goto_32.S
index b260083..498b6d6 100644
--- a/runtime/interpreter/mterp/mips64/op_goto_32.S
+++ b/runtime/interpreter/mterp/mips64/op_goto_32.S
@@ -8,22 +8,7 @@
* our "backward branch" test must be "<=0" instead of "<0".
*/
/* goto/32 +AAAAAAAA */
- .extern MterpProfileBranch
lh rINST, 2(rPC) # rINST <- aaaa (low)
lh a1, 4(rPC) # a1 <- AAAA (high)
ins rINST, a1, 16, 16 # rINST <- offset (sign-extended AAAAaaaa)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/mips64/op_packed_switch.S b/runtime/interpreter/mterp/mips64/op_packed_switch.S
index 2c6eb2f..27ce580 100644
--- a/runtime/interpreter/mterp/mips64/op_packed_switch.S
+++ b/runtime/interpreter/mterp/mips64/op_packed_switch.S
@@ -19,18 +19,4 @@
dlsa a0, a0, rPC, 1 # a0 <- PC + BBBBbbbb*2
jal $func # v0 <- code-unit branch offset
move rINST, v0
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/mips64/zcmp.S b/runtime/interpreter/mterp/mips64/zcmp.S
index 0e0477f..75db49e 100644
--- a/runtime/interpreter/mterp/mips64/zcmp.S
+++ b/runtime/interpreter/mterp/mips64/zcmp.S
@@ -6,25 +6,12 @@
* For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
- .extern MterpProfileBranch
srl a2, rINST, 8 # a2 <- AA
lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB)
GET_VREG a0, a2 # a0 <- vAA
- b${condition}zc a0, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ b${condition}zc a0, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
diff --git a/runtime/interpreter/mterp/out/mterp_mips.S b/runtime/interpreter/mterp/out/mterp_mips.S
index b134129..daa6f2a 100644
--- a/runtime/interpreter/mterp/out/mterp_mips.S
+++ b/runtime/interpreter/mterp/out/mterp_mips.S
@@ -58,7 +58,11 @@
s2 rSELF self (Thread) pointer
s3 rIBASE interpreted instruction base pointer, used for computed goto
s4 rINST first 16-bit code unit of current instruction
+ s5 rOBJ object pointer
s6 rREFS base of object references in shadow frame (ideally, we'll get rid of this later).
+ s7 rTEMP used as temp storage that can survive a function call
+ s8 rPROFILE branch profiling countdown
+
*/
/* single-purpose registers, given names for clarity */
@@ -70,6 +74,7 @@
#define rOBJ s5
#define rREFS s6
#define rTEMP s7
+#define rPROFILE s8
#define rARG0 a0
#define rARG1 a1
@@ -167,7 +172,7 @@
#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
#define MTERP_PROFILE_BRANCHES 1
#define MTERP_LOGGING 0
@@ -490,6 +495,9 @@
STACK_LOAD(ra, 124); \
DELETE_STACK(STACK_SIZE)
+#define REFRESH_IBASE() \
+ lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF)
+
/* File: mips/entry.S */
/*
* Copyright (C) 2016 The Android Open Source Project
@@ -553,6 +561,12 @@
/* Starting ibase */
lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF)
+ /* Set up for backwards branches & osr profiling */
+ lw a0, OFF_FP_METHOD(rFP)
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ JAL(MterpSetUpHotnessCountdown) # (method, shadow_frame)
+ move rPROFILE, v0 # Starting hotness countdown to rPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST() # load rINST from rPC
GET_INST_OPCODE(t0) # extract opcode from rINST
@@ -1284,37 +1298,9 @@
* double to get a byte offset.
*/
/* goto +AA */
-#if MTERP_PROFILE_BRANCHES
sll a0, rINST, 16 # a0 <- AAxx0000
sra rINST, a0, 24 # rINST <- ssssssAA (sign-extended)
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a2, rINST, rINST # a2 <- byte offset
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- /* If backwards branch refresh rIBASE */
- bgez a2, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#else
- sll a0, rINST, 16 # a0 <- AAxx0000
- sra rINST, a0, 24 # rINST <- ssssssAA (sign-extended)
- addu a2, rINST, rINST # a2 <- byte offset
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- /* If backwards branch refresh rIBASE */
- bgez a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1327,33 +1313,8 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
-#if MTERP_PROFILE_BRANCHES
FETCH_S(rINST, 1) # rINST <- ssssAAAA (sign-extended)
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a1, rINST, rINST # a1 <- byte offset, flags set
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#else
- FETCH_S(rINST, 1) # rINST <- ssssAAAA (sign-extended)
- addu a1, rINST, rINST # a1 <- byte offset, flags set
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1369,39 +1330,11 @@
* our "backward branch" test must be "<=0" instead of "<0".
*/
/* goto/32 +AAAAAAAA */
-#if MTERP_PROFILE_BRANCHES
FETCH(a0, 1) # a0 <- aaaa (lo)
FETCH(a1, 2) # a1 <- AAAA (hi)
sll a1, a1, 16
or rINST, a0, a1 # rINST <- AAAAaaaa
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#else
- FETCH(a0, 1) # a0 <- aaaa (lo)
- FETCH(a1, 2) # a1 <- AAAA (hi)
- sll a1, a1, 16
- or rINST, a0, a1 # rINST <- AAAAaaaa
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1417,7 +1350,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
FETCH(a0, 1) # a0 <- bbbb (lo)
FETCH(a1, 2) # a1 <- BBBB (hi)
GET_OPA(a3) # a3 <- AA
@@ -1427,37 +1359,7 @@
EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2
JAL(MterpDoPackedSwitch) # a0 <- code-unit branch offset
move rINST, v0
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, .Lop_packed_switch_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-#else
- FETCH(a0, 1) # a0 <- bbbb (lo)
- FETCH(a1, 2) # a1 <- BBBB (hi)
- GET_OPA(a3) # a3 <- AA
- sll t0, a1, 16
- or a0, a0, t0 # a0 <- BBBBbbbb
- GET_VREG(a1, a3) # a1 <- vAA
- EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2
- JAL(MterpDoPackedSwitch) # a0 <- code-unit branch offset
- move rINST, v0
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
-
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1474,7 +1376,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
FETCH(a0, 1) # a0 <- bbbb (lo)
FETCH(a1, 2) # a1 <- BBBB (hi)
GET_OPA(a3) # a3 <- AA
@@ -1484,37 +1385,7 @@
EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2
JAL(MterpDoSparseSwitch) # a0 <- code-unit branch offset
move rINST, v0
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, .Lop_sparse_switch_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-#else
- FETCH(a0, 1) # a0 <- bbbb (lo)
- FETCH(a1, 2) # a1 <- BBBB (hi)
- GET_OPA(a3) # a3 <- AA
- sll t0, a1, 16
- or a0, a0, t0 # a0 <- BBBBbbbb
- GET_VREG(a1, a3) # a1 <- vAA
- EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2
- JAL(MterpDoSparseSwitch) # a0 <- code-unit branch offset
- move rINST, v0
- addu a1, rINST, rINST # a1 <- byte offset
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgtz a1, 1f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-1:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-#endif
-
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
@@ -1772,9 +1643,8 @@
/* File: mips/op_if_eq.S */
/* File: mips/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1782,27 +1652,14 @@
GET_OPA4(a0) # a0 <- A+
GET_OPB(a1) # a1 <- B
GET_VREG(a3, a1) # a3 <- vB
- GET_VREG(a2, a0) # a2 <- vA
- bne a2, a3, 1f # branch to 1 if comparison failed
+ GET_VREG(a0, a0) # a0 <- vA
FETCH_S(rINST, 1) # rINST<- branch offset, in code units
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a2, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- bgez a2, .L_op_if_eq_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-
+ beq a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB)
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
/* ------------------------------ */
@@ -1811,9 +1668,8 @@
/* File: mips/op_if_ne.S */
/* File: mips/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1821,27 +1677,14 @@
GET_OPA4(a0) # a0 <- A+
GET_OPB(a1) # a1 <- B
GET_VREG(a3, a1) # a3 <- vB
- GET_VREG(a2, a0) # a2 <- vA
- beq a2, a3, 1f # branch to 1 if comparison failed
+ GET_VREG(a0, a0) # a0 <- vA
FETCH_S(rINST, 1) # rINST<- branch offset, in code units
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a2, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- bgez a2, .L_op_if_ne_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-
+ bne a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB)
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
/* ------------------------------ */
@@ -1850,9 +1693,8 @@
/* File: mips/op_if_lt.S */
/* File: mips/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1860,27 +1702,14 @@
GET_OPA4(a0) # a0 <- A+
GET_OPB(a1) # a1 <- B
GET_VREG(a3, a1) # a3 <- vB
- GET_VREG(a2, a0) # a2 <- vA
- bge a2, a3, 1f # branch to 1 if comparison failed
+ GET_VREG(a0, a0) # a0 <- vA
FETCH_S(rINST, 1) # rINST<- branch offset, in code units
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a2, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- bgez a2, .L_op_if_lt_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-
+ blt a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB)
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
/* ------------------------------ */
@@ -1889,9 +1718,8 @@
/* File: mips/op_if_ge.S */
/* File: mips/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1899,27 +1727,14 @@
GET_OPA4(a0) # a0 <- A+
GET_OPB(a1) # a1 <- B
GET_VREG(a3, a1) # a3 <- vB
- GET_VREG(a2, a0) # a2 <- vA
- blt a2, a3, 1f # branch to 1 if comparison failed
+ GET_VREG(a0, a0) # a0 <- vA
FETCH_S(rINST, 1) # rINST<- branch offset, in code units
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a2, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- bgez a2, .L_op_if_ge_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-
+ bge a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB)
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
/* ------------------------------ */
@@ -1928,9 +1743,8 @@
/* File: mips/op_if_gt.S */
/* File: mips/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1938,27 +1752,14 @@
GET_OPA4(a0) # a0 <- A+
GET_OPB(a1) # a1 <- B
GET_VREG(a3, a1) # a3 <- vB
- GET_VREG(a2, a0) # a2 <- vA
- ble a2, a3, 1f # branch to 1 if comparison failed
+ GET_VREG(a0, a0) # a0 <- vA
FETCH_S(rINST, 1) # rINST<- branch offset, in code units
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a2, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- bgez a2, .L_op_if_gt_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-
+ bgt a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB)
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
/* ------------------------------ */
@@ -1967,9 +1768,8 @@
/* File: mips/op_if_le.S */
/* File: mips/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1977,27 +1777,14 @@
GET_OPA4(a0) # a0 <- A+
GET_OPB(a1) # a1 <- B
GET_VREG(a3, a1) # a3 <- vB
- GET_VREG(a2, a0) # a2 <- vA
- bgt a2, a3, 1f # branch to 1 if comparison failed
+ GET_VREG(a0, a0) # a0 <- vA
FETCH_S(rINST, 1) # rINST<- branch offset, in code units
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a2, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
- bgez a2, .L_op_if_le_finish
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-
+ ble a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB)
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
/* ------------------------------ */
@@ -2006,35 +1793,19 @@
/* File: mips/op_if_eqz.S */
/* File: mips/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
GET_OPA(a0) # a0 <- AA
- GET_VREG(a2, a0) # a2 <- vAA
+ GET_VREG(a0, a0) # a0 <- vAA
FETCH_S(rINST, 1) # rINST <- branch offset, in code units
- bne a2, zero, 1f # branch to 1 if comparison failed
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a1, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 3f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-3:
+ beq a0, zero, MterpCommonTakenBranchNoFlags
+ li t0, JIT_CHECK_OSR # possible OSR re-entry?
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
@@ -2045,35 +1816,19 @@
/* File: mips/op_if_nez.S */
/* File: mips/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
GET_OPA(a0) # a0 <- AA
- GET_VREG(a2, a0) # a2 <- vAA
+ GET_VREG(a0, a0) # a0 <- vAA
FETCH_S(rINST, 1) # rINST <- branch offset, in code units
- beq a2, zero, 1f # branch to 1 if comparison failed
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a1, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 3f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-3:
+ bne a0, zero, MterpCommonTakenBranchNoFlags
+ li t0, JIT_CHECK_OSR # possible OSR re-entry?
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
@@ -2084,35 +1839,19 @@
/* File: mips/op_if_ltz.S */
/* File: mips/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
GET_OPA(a0) # a0 <- AA
- GET_VREG(a2, a0) # a2 <- vAA
+ GET_VREG(a0, a0) # a0 <- vAA
FETCH_S(rINST, 1) # rINST <- branch offset, in code units
- bge a2, zero, 1f # branch to 1 if comparison failed
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a1, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 3f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-3:
+ blt a0, zero, MterpCommonTakenBranchNoFlags
+ li t0, JIT_CHECK_OSR # possible OSR re-entry?
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
@@ -2123,35 +1862,19 @@
/* File: mips/op_if_gez.S */
/* File: mips/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
GET_OPA(a0) # a0 <- AA
- GET_VREG(a2, a0) # a2 <- vAA
+ GET_VREG(a0, a0) # a0 <- vAA
FETCH_S(rINST, 1) # rINST <- branch offset, in code units
- blt a2, zero, 1f # branch to 1 if comparison failed
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a1, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 3f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-3:
+ bge a0, zero, MterpCommonTakenBranchNoFlags
+ li t0, JIT_CHECK_OSR # possible OSR re-entry?
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
@@ -2162,35 +1885,19 @@
/* File: mips/op_if_gtz.S */
/* File: mips/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
GET_OPA(a0) # a0 <- AA
- GET_VREG(a2, a0) # a2 <- vAA
+ GET_VREG(a0, a0) # a0 <- vAA
FETCH_S(rINST, 1) # rINST <- branch offset, in code units
- ble a2, zero, 1f # branch to 1 if comparison failed
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a1, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 3f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-3:
+ bgt a0, zero, MterpCommonTakenBranchNoFlags
+ li t0, JIT_CHECK_OSR # possible OSR re-entry?
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
@@ -2201,35 +1908,19 @@
/* File: mips/op_if_lez.S */
/* File: mips/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
GET_OPA(a0) # a0 <- AA
- GET_VREG(a2, a0) # a2 <- vAA
+ GET_VREG(a0, a0) # a0 <- vAA
FETCH_S(rINST, 1) # rINST <- branch offset, in code units
- bgt a2, zero, 1f # branch to 1 if comparison failed
- b 2f
-1:
- li rINST, 2 # rINST- BYTE branch dist for not-taken
-2:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC()
- move a0, rSELF
- addu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- JAL(MterpProfileBranch) # (self, shadow_frame, offset)
- bnez v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- addu a1, rINST, rINST # convert to bytes
- FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST
- bgez a1, 3f
- lw ra, THREAD_FLAGS_OFFSET(rSELF)
- b MterpCheckSuspendAndContinue
-3:
+ ble a0, zero, MterpCommonTakenBranchNoFlags
+ li t0, JIT_CHECK_OSR # possible OSR re-entry?
+ beq rPROFILE, t0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST(2) # advance rPC, load rINST
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
@@ -7983,18 +7674,6 @@
.balign 4
artMterpAsmSisterStart:
-/* continuation for op_packed_switch */
-
-.Lop_packed_switch_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-
-/* continuation for op_sparse_switch */
-
-.Lop_sparse_switch_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-
/* continuation for op_cmpl_float */
.Lop_cmpl_float_nan:
@@ -8039,42 +7718,6 @@
GET_INST_OPCODE(t0) # extract opcode from rINST
SET_VREG_GOTO(rTEMP, rOBJ, t0) # vAA <- rTEMP
-/* continuation for op_if_eq */
-
-.L_op_if_eq_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-
-/* continuation for op_if_ne */
-
-.L_op_if_ne_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-
-/* continuation for op_if_lt */
-
-.L_op_if_lt_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-
-/* continuation for op_if_ge */
-
-.L_op_if_ge_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-
-/* continuation for op_if_gt */
-
-.L_op_if_gt_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-
-/* continuation for op_if_le */
-
-.L_op_if_le_finish:
- GET_INST_OPCODE(t0) # extract opcode from rINST
- GOTO_OPCODE(t0) # jump to next instruction
-
/* continuation for op_float_to_int */
/*
@@ -13089,20 +12732,110 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in lr.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * rPROFILE <= signed hotness countdown (expanded to 32 bits)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
*/
-MterpCheckSuspendAndContinue:
- lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF) # refresh rIBASE
+MterpCommonTakenBranchNoFlags:
+ bgtz rINST, .L_forward_branch # don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+#if JIT_CHECK_OSR != -1
+# error "JIT_CHECK_OSR must be -1."
+#endif
+ li t0, JIT_CHECK_OSR
+ beq rPROFILE, t0, .L_osr_check
+ blt rPROFILE, t0, .L_resume_backward_branch
+ subu rPROFILE, 1
+ beqz rPROFILE, .L_add_batch # counted down to zero - report
+.L_resume_backward_branch:
+ lw ra, THREAD_FLAGS_OFFSET(rSELF)
+ REFRESH_IBASE()
+ addu a2, rINST, rINST # a2<- byte offset
+ FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
and ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
- bnez ra, 1f
+ bnez ra, .L_suspend_request_pending
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
-1:
+
+.L_suspend_request_pending:
EXPORT_PC()
move a0, rSELF
JAL(MterpSuspendCheck) # (self)
bnez v0, MterpFallback
+ REFRESH_IBASE() # might have changed during suspend
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
+
+.L_no_count_backwards:
+ li t0, JIT_CHECK_OSR # check for possible OSR re-entry
+ bne rPROFILE, t0, .L_resume_backward_branch
+.L_osr_check:
+ move a0, rSELF
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rINST
+ EXPORT_PC()
+ JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ bnez v0, MterpOnStackReplacement
+ b .L_resume_backward_branch
+
+.L_forward_branch:
+ li t0, JIT_CHECK_OSR # check for possible OSR re-entry
+ beq rPROFILE, t0, .L_check_osr_forward
+.L_resume_forward_branch:
+ add a2, rINST, rINST # a2<- byte offset
+ FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST
+ GET_INST_OPCODE(t0) # extract opcode from rINST
+ GOTO_OPCODE(t0) # jump to next instruction
+
+.L_check_osr_forward:
+ move a0, rSELF
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rINST
+ EXPORT_PC()
+ JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ bnez v0, MterpOnStackReplacement
+ b .L_resume_forward_branch
+
+.L_add_batch:
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1)
+ lw a0, OFF_FP_METHOD(rFP)
+ move a2, rSELF
+ JAL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ move rPROFILE, v0 # restore new hotness countdown to rPROFILE
+ b .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ move a0, rSELF
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ li a2, 2
+ EXPORT_PC()
+ JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ bnez v0, MterpOnStackReplacement
+ FETCH_ADVANCE_INST(2)
GET_INST_OPCODE(t0) # extract opcode from rINST
GOTO_OPCODE(t0) # jump to next instruction
@@ -13149,6 +12882,26 @@
sw v1, 4(a2)
li v0, 1 # signal return to caller.
MterpDone:
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ blez rPROFILE, .L_pop_and_return # if > 0, we may have some counts to report.
+
+MterpProfileActive:
+ move rINST, v0 # stash return value
+ /* Report cached hotness counts */
+ lw a0, OFF_FP_METHOD(rFP)
+ addu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rSELF
+ sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1)
+ JAL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ move v0, rINST # restore return value
+
+.L_pop_and_return:
/* Restore from the stack and return. Frame size = STACK_SIZE */
STACK_LOAD_FULL()
jalr zero, ra
diff --git a/runtime/interpreter/mterp/out/mterp_mips64.S b/runtime/interpreter/mterp/out/mterp_mips64.S
index a17252b..29a12bf 100644
--- a/runtime/interpreter/mterp/out/mterp_mips64.S
+++ b/runtime/interpreter/mterp/out/mterp_mips64.S
@@ -58,16 +58,18 @@
s3 rINST first 16-bit code unit of current instruction
s4 rIBASE interpreted instruction base pointer, used for computed goto
s5 rREFS base of object references in shadow frame (ideally, we'll get rid of this later).
+ s6 rPROFILE jit profile hotness countdown
*/
/* During bringup, we'll use the shadow frame model instead of rFP */
/* single-purpose registers, given names for clarity */
-#define rPC s0
-#define rFP s1
-#define rSELF s2
-#define rINST s3
-#define rIBASE s4
-#define rREFS s5
+#define rPC s0
+#define rFP s1
+#define rSELF s2
+#define rINST s3
+#define rIBASE s4
+#define rREFS s5
+#define rPROFILE s6
/*
* This is a #include, not a %include, because we want the C pre-processor
@@ -87,7 +89,7 @@
#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
#define MTERP_PROFILE_BRANCHES 1
#define MTERP_LOGGING 0
@@ -128,6 +130,17 @@
.endm
/*
+ * Fetch the next instruction from an offset specified by _reg and advance xPC.
+ * xPC to point to the next instruction. "_reg" must specify the distance
+ * in bytes, *not* 16-bit code units, and may be a signed value. Must not set flags.
+ *
+ */
+.macro FETCH_ADVANCE_INST_RB reg
+ daddu rPC, rPC, \reg
+ FETCH_INST
+.endm
+
+/*
* Fetch the next instruction from the specified offset. Advances rPC
* to point to the next instruction.
*
@@ -274,7 +287,8 @@
#define STACK_OFFSET_S3 40
#define STACK_OFFSET_S4 48
#define STACK_OFFSET_S5 56
-#define STACK_SIZE 64
+#define STACK_OFFSET_S6 64
+#define STACK_SIZE 80 /* needs 16 byte alignment */
/* Constants for float/double_to_int/long conversions */
#define INT_MIN 0x80000000
@@ -344,6 +358,8 @@
.cfi_rel_offset 20, STACK_OFFSET_S4
sd s5, STACK_OFFSET_S5(sp)
.cfi_rel_offset 21, STACK_OFFSET_S5
+ sd s6, STACK_OFFSET_S6(sp)
+ .cfi_rel_offset 22, STACK_OFFSET_S6
/* Remember the return register */
sd a3, SHADOWFRAME_RESULT_REGISTER_OFFSET(a2)
@@ -364,6 +380,12 @@
/* Starting ibase */
REFRESH_IBASE
+ /* Set up for backwards branches & osr profiling */
+ ld a0, OFF_FP_METHOD(rFP)
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ jal MterpSetUpHotnessCountdown
+ move rPROFILE, v0 # Starting hotness countdown to rPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST
GET_INST_OPCODE v0
@@ -1100,24 +1122,9 @@
* double to get a byte offset.
*/
/* goto +AA */
- .extern MterpProfileBranch
srl rINST, rINST, 8
seb rINST, rINST # rINST <- offset (sign-extended AA)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1130,23 +1137,8 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
- .extern MterpProfileBranch
lh rINST, 2(rPC) # rINST <- offset (sign-extended AAAA)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1162,25 +1154,10 @@
* our "backward branch" test must be "<=0" instead of "<0".
*/
/* goto/32 +AAAAAAAA */
- .extern MterpProfileBranch
lh rINST, 2(rPC) # rINST <- aaaa (low)
lh a1, 4(rPC) # a1 <- AAAA (high)
ins rINST, a1, 16, 16 # rINST <- offset (sign-extended AAAAaaaa)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1206,21 +1183,7 @@
dlsa a0, a0, rPC, 1 # a0 <- PC + BBBBbbbb*2
jal MterpDoPackedSwitch # v0 <- code-unit branch offset
move rINST, v0
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1247,21 +1210,7 @@
dlsa a0, a0, rPC, 1 # a0 <- PC + BBBBbbbb*2
jal MterpDoSparseSwitch # v0 <- code-unit branch offset
move rINST, v0
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
@@ -1453,22 +1402,10 @@
lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC)
GET_VREG a0, a2 # a0 <- vA
GET_VREG a1, a3 # a1 <- vB
- beqc a0, a1, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ beqc a0, a1, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1492,22 +1429,10 @@
lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC)
GET_VREG a0, a2 # a0 <- vA
GET_VREG a1, a3 # a1 <- vB
- bnec a0, a1, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ bnec a0, a1, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1531,22 +1456,10 @@
lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC)
GET_VREG a0, a2 # a0 <- vA
GET_VREG a1, a3 # a1 <- vB
- bltc a0, a1, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ bltc a0, a1, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1570,22 +1483,10 @@
lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC)
GET_VREG a0, a2 # a0 <- vA
GET_VREG a1, a3 # a1 <- vB
- bgec a0, a1, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ bgec a0, a1, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1609,22 +1510,10 @@
lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC)
GET_VREG a0, a2 # a0 <- vA
GET_VREG a1, a3 # a1 <- vB
- bgtc a0, a1, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ bgtc a0, a1, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1648,22 +1537,10 @@
lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC)
GET_VREG a0, a2 # a0 <- vA
GET_VREG a1, a3 # a1 <- vB
- blec a0, a1, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ blec a0, a1, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1681,26 +1558,13 @@
* For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
- .extern MterpProfileBranch
srl a2, rINST, 8 # a2 <- AA
lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB)
GET_VREG a0, a2 # a0 <- vAA
- beqzc a0, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ beqzc a0, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1718,26 +1582,13 @@
* For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
- .extern MterpProfileBranch
srl a2, rINST, 8 # a2 <- AA
lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB)
GET_VREG a0, a2 # a0 <- vAA
- bnezc a0, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ bnezc a0, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1755,26 +1606,13 @@
* For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
- .extern MterpProfileBranch
srl a2, rINST, 8 # a2 <- AA
lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB)
GET_VREG a0, a2 # a0 <- vAA
- bltzc a0, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ bltzc a0, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1792,26 +1630,13 @@
* For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
- .extern MterpProfileBranch
srl a2, rINST, 8 # a2 <- AA
lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB)
GET_VREG a0, a2 # a0 <- vAA
- bgezc a0, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ bgezc a0, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1829,26 +1654,13 @@
* For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
- .extern MterpProfileBranch
srl a2, rINST, 8 # a2 <- AA
lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB)
GET_VREG a0, a2 # a0 <- vAA
- bgtzc a0, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ bgtzc a0, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -1866,26 +1678,13 @@
* For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
- .extern MterpProfileBranch
srl a2, rINST, 8 # a2 <- AA
lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB)
GET_VREG a0, a2 # a0 <- vAA
- blezc a0, 1f
- li rINST, 2 # offset if branch not taken
-1:
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- move a0, rSELF
- daddu a1, rFP, OFF_FP_SHADOWFRAME
- move a2, rINST
- jal MterpProfileBranch # (self, shadow_frame, offset)
- bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST
-#endif
- dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2
- lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue
- move a0, rINST # a0 <- offset
- FETCH_INST # load rINST
- bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch
+ blezc a0, MterpCommonTakenBranchNoFlags
+ li v0, JIT_CHECK_OSR # possible OSR re-entry?
+ beqc rPROFILE, v0, .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2 # advance rPC, load rINST
GET_INST_OPCODE v0 # extract opcode from rINST
GOTO_OPCODE v0 # jump to next instruction
@@ -12323,23 +12122,110 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in ra.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * rPROFILE <= signed hotness countdown (expanded to 64 bits)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
*/
- .extern MterpSuspendCheck
-MterpCheckSuspendAndContinue:
+MterpCommonTakenBranchNoFlags:
+ bgtzc rINST, .L_forward_branch # don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+ li v0, JIT_CHECK_OSR
+ beqc rPROFILE, v0, .L_osr_check
+ bltc rPROFILE, v0, .L_resume_backward_branch
+ dsubu rPROFILE, 1
+ beqzc rPROFILE, .L_add_batch # counted down to zero - report
+.L_resume_backward_branch:
+ lw ra, THREAD_FLAGS_OFFSET(rSELF)
REFRESH_IBASE
- and ra, ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
- bnez ra, check1
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
-check1:
+ daddu a2, rINST, rINST # a2<- byte offset
+ FETCH_ADVANCE_INST_RB a2 # update rPC, load rINST
+ and ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
+ bnezc ra, .L_suspend_request_pending
+ GET_INST_OPCODE v0 # extract opcode from rINST
+ GOTO_OPCODE v0 # jump to next instruction
+
+.L_suspend_request_pending:
EXPORT_PC
move a0, rSELF
- jal MterpSuspendCheck # (self)
- bnezc v0, MterpFallback # Something in the environment changed, switch interpreters
- GET_INST_OPCODE v0 # extract opcode from rINST
- GOTO_OPCODE v0 # jump to next instruction
+ jal MterpSuspendCheck # (self)
+ bnezc v0, MterpFallback
+ REFRESH_IBASE # might have changed during suspend
+ GET_INST_OPCODE v0 # extract opcode from rINST
+ GOTO_OPCODE v0 # jump to next instruction
+
+.L_no_count_backwards:
+ li v0, JIT_CHECK_OSR # check for possible OSR re-entry
+ bnec rPROFILE, v0, .L_resume_backward_branch
+.L_osr_check:
+ move a0, rSELF
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rINST
+ EXPORT_PC
+ jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset)
+ bnezc v0, MterpOnStackReplacement
+ b .L_resume_backward_branch
+
+.L_forward_branch:
+ li v0, JIT_CHECK_OSR # check for possible OSR re-entry
+ beqc rPROFILE, v0, .L_check_osr_forward
+.L_resume_forward_branch:
+ daddu a2, rINST, rINST # a2<- byte offset
+ FETCH_ADVANCE_INST_RB a2 # update rPC, load rINST
+ GET_INST_OPCODE v0 # extract opcode from rINST
+ GOTO_OPCODE v0 # jump to next instruction
+
+.L_check_osr_forward:
+ move a0, rSELF
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rINST
+ EXPORT_PC
+ jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset)
+ bnezc v0, MterpOnStackReplacement
+ b .L_resume_forward_branch
+
+.L_add_batch:
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1)
+ ld a0, OFF_FP_METHOD(rFP)
+ move a2, rSELF
+ jal MterpAddHotnessBatch # (method, shadow_frame, self)
+ move rPROFILE, v0 # restore new hotness countdown to rPROFILE
+ b .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ move a0, rSELF
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ li a2, 2
+ EXPORT_PC
+ jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset)
+ bnezc v0, MterpOnStackReplacement
+ FETCH_ADVANCE_INST 2
+ GET_INST_OPCODE v0 # extract opcode from rINST
+ GOTO_OPCODE v0 # jump to next instruction
/*
* On-stack replacement has happened, and now we've returned from the compiled method.
@@ -12395,6 +12281,28 @@
check2:
li v0, 1 # signal return to caller.
MterpDone:
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ blez rPROFILE, .L_pop_and_return # if > 0, we may have some counts to report.
+
+MterpProfileActive:
+ move rINST, v0 # stash return value
+ /* Report cached hotness counts */
+ ld a0, OFF_FP_METHOD(rFP)
+ daddu a1, rFP, OFF_FP_SHADOWFRAME
+ move a2, rSELF
+ sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1)
+ jal MterpAddHotnessBatch # (method, shadow_frame, self)
+ move v0, rINST # restore return value
+
+.L_pop_and_return:
+ ld s6, STACK_OFFSET_S6(sp)
+ .cfi_restore 22
ld s5, STACK_OFFSET_S5(sp)
.cfi_restore 21
ld s4, STACK_OFFSET_S4(sp)
@@ -12421,5 +12329,6 @@
.cfi_adjust_cfa_offset -STACK_SIZE
.cfi_endproc
+ .set reorder
.size ExecuteMterpImpl, .-ExecuteMterpImpl
