From dc4d80a2a73896e8509c196d917760b47cb09e6b Mon Sep 17 00:00:00 2001 From: Pavel Afremov Date: Wed, 14 Mar 2007 15:25:10 +0300 Subject: [PATCH] Fast path asm helpers for monitor enter / exit. Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch includes: • Enabling ssm helpers for x86-32 and x86-64 Linux platform • Improvement of x86-32 helpers. • First version of Linux x86-64 helpers. --- build/make/components/vm/jthread.xml | 31 ++ vm/port/src/lil/em64t/pim/m2n_em64t.cpp | 20 + vm/port/src/lil/em64t/pim/m2n_em64t_internal.h | 3 vm/thread/src/thread_helpers.cpp | 275 -------------------- vm/thread/src/thread_helpers_em64t.cpp | 254 ++++++++++++++++++ vm/thread/src/thread_helpers_ia32.cpp | 259 +++++++++++++++++++ vm/thread/src/thread_private.h | 20 + vm/vmcore/src/util/em64t/base/compile_em64t.cpp | 21 ++ .../util/em64t/base/jit_lock_rt_support_em64t.cpp | 269 ++++++++++++++++++++++ .../util/em64t/base/jit_runtime_support_em64t.cpp | 139 ++++++++++++ 10 files changed, 1008 insertions(+), 283 deletions(-) diff --git a/build/make/components/vm/jthread.xml b/build/make/components/vm/jthread.xml old mode 100644 new mode 100755 index 313001f..e8bad9f --- a/build/make/components/vm/jthread.xml +++ b/build/make/components/vm/jthread.xml @@ -78,7 +78,36 @@ vm.hythr" /> - + + + + + + + + + + + + diff --git a/vm/port/src/encoder/ia32_em64t/encoder.h b/vm/port/src/encoder/ia32_em64t/encoder.h old mode 100644 new mode 100755 diff --git a/vm/port/src/lil/em64t/pim/m2n_em64t.cpp b/vm/port/src/lil/em64t/pim/m2n_em64t.cpp index 8b5a316..f70a397 100644 --- a/vm/port/src/lil/em64t/pim/m2n_em64t.cpp +++ b/vm/port/src/lil/em64t/pim/m2n_em64t.cpp @@ -134,6 +134,12 @@ void * m2n_get_frame_base(M2nFrame * m2n /* Internal Interface */ +unsigned m2n_ts_to_register_size(unsigned num_std_need_to_save, + unsigned num_ret_need_to_save) { + return 13 + (6 * num_std_need_to_save) + + 3 + (6 * num_ret_need_to_save); +} + // rsp should point to the bottom of the activation frame since push may occur // inputs should be preserved outside if required since we do a call // num_std_need_to_save registers will be preserved @@ -264,6 +270,12 @@ char * m2n_gen_set_local_handles_imm(cha return buf; } +unsigned m2n_push_m2n_size(unsigned num_callee_saves, + unsigned num_std_need_to_save) { + return 82 - (5 * num_callee_saves) + + m2n_ts_to_register_size(num_std_need_to_save, 0); +} + // inputs should be preserved outside if required since we do a call // num_std_need_to_save registers will be preserved char * m2n_gen_push_m2n(char * buf, Method_Handle method, @@ -323,6 +335,11 @@ char * m2n_gen_push_m2n(char * buf, Meth return buf; } +unsigned m2n_pop_m2n_size(bool handles, unsigned num_callee_saves, unsigned preserve_ret) +{ + return 56 - 5*num_callee_saves + (preserve_ret ? 4: 0); +} + static void m2n_pop_local_handles() { assert(!hythread_is_suspend_enabled()); @@ -399,8 +416,9 @@ #endif size_64); bytes_to_m2n_bottom += LcgEM64TContext::GR_SIZE; } + return buf; -} +}//m2n_gen_pop_m2n // returns pointer to the registers used for jvmti PopFrame Registers* get_pop_frame_registers(M2nFrame* m2nf) { diff --git a/vm/port/src/lil/em64t/pim/m2n_em64t_internal.h b/vm/port/src/lil/em64t/pim/m2n_em64t_internal.h index ac11b7f..4f2e3b3 100644 --- a/vm/port/src/lil/em64t/pim/m2n_em64t_internal.h +++ b/vm/port/src/lil/em64t/pim/m2n_em64t_internal.h @@ -85,6 +85,7 @@ inline size_t m2n_get_size() { char * m2n_gen_ts_to_register(char * buf, const R_Opnd * reg, unsigned num_callee_saves_used, unsigned num_callee_saves_max, unsigned num_std_need_to_save, unsigned num_ret_need_to_save); +unsigned m2n_ts_to_register_size(unsigned num_std_need_to_save, unsigned num_ret_need_to_save); /** * Generate code to set the local handles of an M2nFrame. @@ -107,6 +108,7 @@ char * m2n_gen_set_local_handles_imm(cha */ char * m2n_gen_push_m2n(char * buf, Method_Handle method, frame_type current_frame_type, bool handles, unsigned num_callee_saves, unsigned num_std_need_to_save, int32 bytes_to_m2n_top); +unsigned m2n_push_m2n_size(unsigned num_callee_saves, unsigned num_std_need_to_save); /** * Generate code to pop an M2nFrame off the stack. @@ -119,6 +121,7 @@ char * m2n_gen_push_m2n(char * buf, Meth */ char * m2n_gen_pop_m2n(char * buf, bool handles, unsigned num_callee_saves, int32 bytes_to_m2n_bottom, unsigned preserve_ret); +unsigned m2n_pop_m2n_size(bool handles, unsigned num_callee_saves, unsigned preserve_ret); // returns top of the specified frame on the stack (it should point to return ip) void * m2n_get_frame_base(M2nFrame *); diff --git a/vm/thread/src/thread_helpers.cpp b/vm/thread/src/thread_helpers.cpp deleted file mode 100644 index 947eed1..0000000 --- a/vm/thread/src/thread_helpers.cpp +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file thread_helpers.cpp - * @brief Set of VM helpers - * - * This file contains the set of "VM helpers" which help to optimize monitors performance - * in the code generated by JIT compiler. Typically, these functions will be called by JIT, - * but VM also could also use them with care. - */ - -#include -#include -#include "thread_private.h" -#include "open/thread_externals.h" -#include "open/jthread.h" - -#include - - -/** - * Generates tmn_self() call. - * The code should not contains safepoint. - * The code uses and doesn't restore eax register. - * - * @return tm_self() in eax register - */ -char* gen_hythread_self_helper(char *ss) { -#ifdef FS14_TLS_USE - //ss = mov(ss, eax_opnd, M_Base_Opnd(fs_reg, 0x14)); - *ss++ = (char)0x64; - *ss++ = (char)0xa1; - *ss++ = (char)0x14; - *ss++ = (char)0x00; - *ss++ = (char)0x00; - *ss++ = (char)0x00; -#else - ss = call(ss, (char *)hythread_self); -#endif - return ss; -} - - -/** - * Generates fast path of monitor enter - * the code should not contains safepoint. - * - * @param[in] ss buffer to put the assembly code to - * @param[in] input_param1 register which should point to the object lockword. - * If input_param1 == ecx it reduces one register mov. - * the code use and do not restore ecx, edx, eax registers - * - * @return 0 if success in eax register - */ -char* gen_monitorenter_fast_path_helper(char *ss, const R_Opnd & input_param1) { - - if (&input_param1 != &ecx_opnd) { - ss = mov(ss, ecx_opnd, input_param1); - } -#ifdef ASM_MONITOR_HELPER - signed offset2; - //get self_id - ss = gen_hythread_self_helper(ss); - ss = mov(ss, edx_opnd, M_Base_Opnd(eax_reg, (uint32)&((HyThread *)0)->thread_id) ); -#ifdef LOCK_RESERVATION - //get lock_id - ss = mov(ss, eax_opnd, M_Base_Opnd(ecx_reg, 0)); - // move thread_id to AX - ss = shift(ss, ror_opc, eax_opnd, Imm_Opnd(16)); - - // test this recursion call - ss = alu(ss, cmp_opc, edx_opnd, eax_opnd, size_16); - ss = branch8(ss, Condition_Z, Imm_Opnd(size_8, 0)); - char *backpatch_address__recursion_inc = ((char *)ss) - 1; - - // test the lock is busy - ss = test(ss, eax_opnd, eax_opnd, size_16); - ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); - char *backpatch_address__inline_monitor_failed = ((char *)ss) - 1; -#else - ss = alu(ss, xor_opc, eax_opnd, eax_opnd); -#endif - // the lock is free or not reserved - ss = prefix(ss, lock_prefix); - ss = cmpxchg(ss, M_Base_Opnd(ecx_reg, 2), edx_opnd, size_16); - ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); - char *backpatch_address__inline_monitor_failed2 = ((char *)ss) - 1; - -#ifdef LOCK_RESERVATION - // if this is initial reservation also increase the recursion - ss = mov(ss, edx_opnd, eax_opnd); - // eax still ROR so ROR the mask - ss = alu(ss, and_opc, edx_opnd, Imm_Opnd(0x0400ffff)); - ss = test(ss, edx_opnd, edx_opnd); - ss = branch8(ss, Condition_Z, Imm_Opnd(size_8, 0)); - char *backpatch_address__recursion_inc2 = ((char *)ss) - 1; -#endif - ss = ret(ss, Imm_Opnd(4)); - -#ifdef LOCK_RESERVATION - // increase recursion branch - signed offset = (signed)ss - (signed)backpatch_address__recursion_inc - 1; - *backpatch_address__recursion_inc = (char)offset; - - // test recursion overflow - // eax still ROR so ROR the mask - ss = alu(ss, cmp_opc, eax_opnd, Imm_Opnd(0xf4000000)); - ss = branch8(ss, Condition_A, Imm_Opnd(size_8, 0)); - char *backpatch_address__inline_monitor_failed3 = ((char *)ss) - 1; - - offset2 = (signed)ss - (signed)backpatch_address__recursion_inc2 - 1; - *backpatch_address__recursion_inc2 = (char)offset2; - - // restore lock_id - ss = shift(ss, ror_opc, eax_opnd, Imm_Opnd(16)); - ss = alu(ss, add_opc, eax_opnd, Imm_Opnd(size_16, 0x800), size_16); - ss = mov(ss, M_Base_Opnd(ecx_reg, 0), eax_opnd, size_16); - - ss = ret(ss, Imm_Opnd(4)); - - offset = (signed)ss - (signed)backpatch_address__inline_monitor_failed - 1; - *backpatch_address__inline_monitor_failed = (char)offset; - offset = (signed)ss - (signed)backpatch_address__inline_monitor_failed3 - 1; - *backpatch_address__inline_monitor_failed3 = (char)offset; -#endif - offset2 = (signed)ss - (signed)backpatch_address__inline_monitor_failed2 - 1; - *backpatch_address__inline_monitor_failed2 = (char)offset2; - - -#endif //ASM_MONITOR_HELPER - // the second attempt to lock monitor - ss = push(ss, ecx_opnd); - ss = call(ss, (char *)hythread_thin_monitor_try_enter); - ss = alu(ss, add_opc, esp_opnd, Imm_Opnd(4)); // pop parameters - - return ss; -} - -/** - * Generates slow path of monitor enter. - * This code could block on monitor and contains safepoint. - * The appropriate m2n frame should be generated and - * - * @param[in] ss buffer to put the assembly code to - * @param[in] input_param1 register should point to the jobject(handle) - * If input_param1 == eax it reduces one register mov. - * the code use and do not restore ecx, edx, eax registers - * @return 0 if success in eax register - */ -char* gen_monitorenter_slow_path_helper(char *ss, const R_Opnd & input_param1) { - if (&input_param1 != &eax_opnd) { - ss = mov(ss, eax_opnd, input_param1); - } - - ss = push(ss, eax_opnd); // push the address of the handle - ss = call(ss, (char *)jthread_monitor_enter); - ss = alu(ss, add_opc, esp_opnd, Imm_Opnd(4)); // pop parameters - return ss; -} - -/** - * Generates monitor exit. - * The code should not contain safepoints. - * - * @param[in] ss buffer to put the assembly code to - * @param[in] input_param1 register should point to the lockword in object header. - * If input_param1 == ecx it reduce one register mov. - * The code use and do not restore eax registers. - * @return 0 if success in eax register - */ -char* gen_monitor_exit_helper(char *ss, const R_Opnd & input_param1) { - if (&input_param1 != &ecx_opnd) { - ss = mov(ss, ecx_opnd, input_param1); - } -#ifdef ASM_MONITOR_HELPER - ss = mov(ss, eax_opnd, M_Base_Opnd(ecx_reg, 0)); - ss = mov(ss, edx_opnd, eax_opnd); - ss = alu(ss, and_opc, eax_opnd, 0x8000f800); - ss = test(ss, eax_opnd, eax_opnd); - ss = branch8(ss, Condition_Z, Imm_Opnd(size_8, 0)); - char *backpatch_address__thin_monitor = ((char *)ss) - 1; - ss = alu(ss, and_opc, eax_opnd, 0x80000000); - ss = test(ss, eax_opnd, eax_opnd); - ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); - char *backpatch_address__fat_monitor = ((char *)ss) - 1; - - // recursion or reservation => dec recursion count - ss = alu(ss, sub_opc, edx_opnd, Imm_Opnd(0x800)); - ss = mov(ss, M_Base_Opnd(ecx_reg,0), edx_opnd); - ss = ret(ss, Imm_Opnd(4)); - - signed offset = (signed)ss - (signed)backpatch_address__thin_monitor - 1; - *backpatch_address__thin_monitor = (char)offset; - ss = mov(ss, M_Base_Opnd(ecx_reg, 2), Imm_Opnd(size_16, 0), size_16); - ss = ret(ss, Imm_Opnd(4)); - - - offset = (signed)ss - (signed)backpatch_address__fat_monitor - 1; - *backpatch_address__fat_monitor = (char)offset; - -#endif - - ss = push(ss, ecx_opnd); - ss = call(ss, (char *)hythread_thin_monitor_exit); - ss = alu(ss, add_opc, esp_opnd, Imm_Opnd(4)); // pop parameters - return ss; -} - -/** - * Generates slow path of monitor exit. - * This code could block on monitor and contains safepoint. - * The appropriate m2n frame should be generated and - * - * @param[in] ss buffer to put the assembly code to - * @param[in] input_param1 register should point to the jobject(handle) - * If input_param1 == eax it reduces one register mov. - * the code use and do not restore ecx, edx, eax registers - * @return 0 if success in eax register - */ -char* gen_monitorexit_slow_path_helper(char *ss, const R_Opnd & input_param1) { - if (&input_param1 != &eax_opnd) { - ss = mov(ss, eax_opnd, input_param1); - } - - ss = push(ss, eax_opnd); // push the address of the handle - ss = call(ss, (char *)jthread_monitor_exit); - ss = alu(ss, add_opc, esp_opnd, Imm_Opnd(4)); // pop parameters - return ss; -} - -/** - * Generates fast accessor to the TLS for the given key.
- * Example: - *

-  * get_thread_ptr = get_tls_helper(vm_thread_block_key);
-  * ...
-  * self = get_thread_ptr();
-  * 
- * - * @param[in] key TLS key - * @return fast accessor to key, if one exist - */ -fast_tls_func* get_tls_helper(hythread_tls_key_t key) { - // return tm_self_tls->thread_local_storage[key]; - unsigned key_offset = (unsigned)&(((hythread_t)(0))->thread_local_storage[key]); - - const int stub_size = 126; - char *stub = (char *)malloc(stub_size); - memset(stub, 0xcc /*int 3*/, stub_size); - - char *ss = stub; - - ss = gen_hythread_self_helper(ss); - ss = mov(ss, eax_opnd, M_Base_Opnd(eax_reg, key_offset)); - ss = ret(ss, Imm_Opnd(0)); - - assert((ss - stub) < stub_size); - - return (fast_tls_func*) stub; -} diff --git a/vm/thread/src/thread_helpers_em64t.cpp b/vm/thread/src/thread_helpers_em64t.cpp new file mode 100644 index 0000000..daa0d51 --- /dev/null +++ b/vm/thread/src/thread_helpers_em64t.cpp @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @author Artem Aliev + * @version $Revision: 1.1.2.11 $ + */ + +/** + * @file thread_helpers.cpp + * @brief Set of VM helpers + * + * This file contains the set of "VM helpers" which help to optimize monitors performance + * in the code generated by JIT compiler. Typically, these functions will be called by JIT, + * but VM also could also use them with care. + */ + +#include +#include +#include "thread_private.h" +#include "open/thread_externals.h" +#include "open/jthread.h" + +#include + +/** + * Generates tmn_self() call. + * The code should not contains safepoint. + * The code uses and doesn't restore eax register. + * + * @return tm_self() in eax register + */ +char* gen_hythread_self_helper(char *ss) { + ss = call(ss, (char *)hythread_self); + return ss; +} + + +/** + * Generates fast path of monitor enter + * the code should not contains safepoint. + * + * @param[in] ss buffer to put the assembly code to + * @param[in] input_param1 register which should point to the object lockword. + * If input_param1 == ecx it reduces one register mov. + * the code use and do not restore ecx, edx, eax registers + * + * @return 0 if success in eax register + */ +char* gen_monitorenter_fast_path_helper(char *ss, const R_Opnd & input_param1) { + + if (&input_param1 != &rdi_opnd) { + ss = mov(ss, rdi_opnd, input_param1); + } + + //get self_id + ss = push(ss, rdi_opnd); + ss = gen_hythread_self_helper(ss); + ss = pop(ss, rdi_opnd); + + ss = mov(ss, rdx_opnd, M_Base_Opnd(rax_reg, + (uint32)&((HyThread *)0)->thread_id) ); //mov rdx,dword [rax+off] + + ss = mov(ss, rax_opnd, M_Base_Opnd(rdi_reg, 2), size_16); // mov ax,word[ecx+2] + ss = alu(ss, cmp_opc, rdx_opnd, rax_opnd, size_16); // cmp dx,ax + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz check_zero + char *check_zero = ((char *)ss) - 1; + + //; ax==dx it's safe to do inc + ss = mov(ss, rax_opnd, M_Base_Opnd(rdi_reg, 1), size_8); // mov al, byte[rdi+1] + + //rec_inc: + ss = alu(ss, add_opc, rax_opnd, + Imm_Opnd(size_8, 0x8), size_8); // add al,0x8 + ss = branch8(ss, Condition_C, Imm_Opnd(size_8, 0)); // jc failed + char *failed1 = ((char *)ss) - 1; + + ss = mov(ss, M_Base_Opnd(rdi_reg, 1), rax_opnd, size_8); // mov byte[ecx+1],al + ss = alu(ss, add_opc, rsp_opnd, Imm_Opnd(size_8, 0x8)); // add rsp,0x8 + ss = ret(ss); // ret + + //check_zero: + signed offset = (signed)ss - (signed)check_zero - 1; + *check_zero = (char)offset; + + ss = test(ss, rax_opnd, rax_opnd, size_16); // test ax,ax + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz failed + char *failed2 = ((char *)ss) - 1; + + ss = prefix(ss, lock_prefix); //; here ax==0. + ss = cmpxchg(ss, M_Base_Opnd(rdi_reg, 2), rdx_opnd, size_16);// lock cmpxchg16 [ecx+2],dx + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz failed + char *failed3 = ((char *)ss) - 1; + +#ifdef LOCK_RESERVATION + ss = mov(ss, rax_opnd, M_Base_Opnd(rdi_reg, 1), size_8); // mov al, byte[ecx+1] + ss = test(ss, rax_opnd, Imm_Opnd(size_8, 0x4), size_8); // test al,0x4 + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz finish + char *finish = ((char *)ss) - 1; + + ss = alu(ss, add_opc, rax_opnd, Imm_Opnd(size_8, 8), size_8);// add al,0x8 + ss = mov(ss, M_Base_Opnd(rdi_reg, 1), rax_opnd, size_8); // mov byte[ecx+1],al + + //finish: + offset = (signed)ss - (signed)finish - 1; + *finish = (char)offset; +#endif + ss = alu(ss, add_opc, rsp_opnd, Imm_Opnd(size_8, 0x8)); // add rsp,0x8 + ss = ret(ss); // ret + + //failed: + offset = (signed)ss - (signed)failed1 - 1; + *failed1 = (char)offset; + offset = (signed)ss - (signed)failed2 - 1; + *failed2 = (char)offset; + offset = (signed)ss - (signed)failed3 - 1; + *failed3 = (char)offset; + + // the second attempt to lock monitor + ss = call(ss, (char *)hythread_thin_monitor_try_enter); + + return ss; +} + +/** + * Generates slow path of monitor enter. + * This code could block on monitor and contains safepoint. + * The appropriate m2n frame should be generated and + * + * @param[in] ss buffer to put the assembly code to + * @param[in] input_param1 register should point to the jobject(handle) + * If input_param1 == eax it reduces one register mov. + * the code use and do not restore ecx, edx, eax registers + * @return 0 if success in eax register + */ +char* gen_monitorenter_slow_path_helper(char *ss, const R_Opnd & input_param1) { + if (&input_param1 != &rdi_opnd) { + ss = mov(ss, rdi_opnd, input_param1); + } + + ss = call(ss, (char *)jthread_monitor_enter); + return ss; +} + +/** + * Generates monitor exit. + * The code should not contain safepoints. + * + * @param[in] ss buffer to put the assembly code to + * @param[in] input_param1 register should point to the lockword in object header. + * If input_param1 == ecx it reduce one register mov. + * The code use and do not restore eax registers. + * @return 0 if success in eax register + */ +char* gen_monitor_exit_helper(char *ss, const R_Opnd & input_param1) { + if (&input_param1 != &rdi_opnd) { + ss = mov(ss, rdi_opnd, input_param1); + } + +#ifdef ASM_MONITOR_HELPER + ss = mov(ss, rax_opnd, M_Base_Opnd(rdi_reg, 0)); // mov rax,dword[rdi] + ss = test(ss, rax_opnd, Imm_Opnd(0x80000000), size_32); // test rax,0x80000000 + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz fat + char *fat = ((char *)ss) - 1; + ss = mov(ss, rax_opnd, M_Base_Opnd(rdi_reg, 1), size_8); // mov al, byte[rdi+1] + + ss = alu(ss, sub_opc, rax_opnd, Imm_Opnd(size_8,0x8),size_8);// sub al, 0x8 + ss = branch8(ss, Condition_C, Imm_Opnd(size_8, 0)); // jc zero_rec + char *zero_rec = ((char *)ss) - 1; + ss = mov(ss, M_Base_Opnd(rdi_reg, 1), rax_opnd, size_8); // mov byte[rdi+1],al + ss = ret(ss); // ret + + //zero_rec: + signed offset = (signed)ss - (signed)zero_rec - 1; + *zero_rec = (char)offset; + + ss = mov(ss, M_Base_Opnd(rdi_reg, 2), + Imm_Opnd(size_16, 0), size_16); // mov word[rdi+2],0 + ss = ret(ss); // ret + + //fat: + offset = (signed)ss - (signed)fat - 1; + *fat = (char)offset; + +#endif + + ss = call(ss, (char *)hythread_thin_monitor_exit); + return ss; +} + +/** + * Generates slow path of monitor exit. + * This code could block on monitor and contains safepoint. + * The appropriate m2n frame should be generated and + * + * @param[in] ss buffer to put the assembly code to + * @param[in] input_param1 register should point to the jobject(handle) + * If input_param1 == eax it reduces one register mov. + * the code use and do not restore ecx, edx, eax registers + * @return 0 if success in eax register + */ +char* gen_monitorexit_slow_path_helper(char *ss, const R_Opnd & input_param1) { + if (&input_param1 != &rdi_opnd) { + ss = mov(ss, rdi_opnd, input_param1); + } + + ss = call(ss, (char *)jthread_monitor_exit); + return ss; +} + +/** + * Generates fast accessor to the TLS for the given key.
+ * Example: + *

+  * get_thread_ptr = get_tls_helper(vm_thread_block_key);
+  * ...
+  * self = get_thread_ptr();
+  * 
+ * + * @param[in] key TLS key + * @return fast accessor to key, if one exist + */ +fast_tls_func* get_tls_helper(hythread_tls_key_t key) { + // return tm_self_tls->thread_local_storage[key]; + unsigned key_offset = (unsigned)&(((hythread_t)(0))->thread_local_storage[key]); + + const int stub_size = 126; + char *stub = (char *)malloc(stub_size); + memset(stub, 0xcc /*int 3*/, stub_size); + + char *ss = stub; + + ss = gen_hythread_self_helper(ss); + ss = mov(ss, rax_opnd, M_Base_Opnd(rax_reg, key_offset)); + ss = ret(ss, Imm_Opnd(0)); + + assert((ss - stub) < stub_size); + + return (fast_tls_func*) stub; +} diff --git a/vm/thread/src/thread_helpers_ia32.cpp b/vm/thread/src/thread_helpers_ia32.cpp new file mode 100644 index 0000000..20be610 --- /dev/null +++ b/vm/thread/src/thread_helpers_ia32.cpp @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file thread_helpers.cpp + * @brief Set of VM helpers + * + * This file contains the set of "VM helpers" which help to optimize monitors performance + * in the code generated by JIT compiler. Typically, these functions will be called by JIT, + * but VM also could also use them with care. + */ + +#include +#include +#include "thread_private.h" +#include "open/thread_externals.h" +#include "open/jthread.h" + +#include + + +/** + * Generates tmn_self() call. + * The code should not contains safepoint. + * The code uses and doesn't restore eax register. + * + * @return tm_self() in eax register + */ +char* gen_hythread_self_helper(char *ss) { +#ifdef FS14_TLS_USE + //ss = mov(ss, eax_opnd, M_Base_Opnd(fs_reg, 0x14)); + *ss++ = (char)0x64; + *ss++ = (char)0xa1; + *ss++ = (char)0x14; + *ss++ = (char)0x00; + *ss++ = (char)0x00; + *ss++ = (char)0x00; +#else + ss = call(ss, (char *)hythread_self); +#endif + return ss; +} + + +/** + * Generates fast path of monitor enter + * the code should not contains safepoint. + * + * @param[in] ss buffer to put the assembly code to + * @param[in] input_param1 register which should point to the object lockword. + * If input_param1 == ecx it reduces one register mov. + * the code use and do not restore ecx, edx, eax registers + * + * @return 0 if success in eax register + */ +char* gen_monitorenter_fast_path_helper(char *ss, const R_Opnd & input_param1) { + + if (&input_param1 != &ecx_opnd) { + ss = mov(ss, ecx_opnd, input_param1); + } +#ifdef ASM_MONITOR_HELPER + + //get self_id + ss = gen_hythread_self_helper(ss); + ss = mov(ss, edx_opnd, M_Base_Opnd(eax_reg, (uint32)&((HyThread *)0)->thread_id) ); // mov edx,dword [eax+off] + + ss = mov(ss, eax_opnd, M_Base_Opnd(ecx_reg, 2), size_16); // mov ax,word[ecx+2] + ss = alu(ss, cmp_opc, edx_opnd, eax_opnd, size_16); // cmp dx,ax + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz check_zero + char *check_zero = ((char *)ss) - 1; + //; ax==dx it's safe to do inc + ss = mov(ss, eax_opnd, M_Base_Opnd(ecx_reg, 1), size_8); // mov al, byte[ecx+1] + //rec_inc: + + ss = alu(ss, add_opc, eax_opnd, Imm_Opnd(size_8, 0x8), size_8); // add al,0x8 + ss = branch8(ss, Condition_C, Imm_Opnd(size_8, 0)); // jc failed + char *failed1 = ((char *)ss) - 1; + + ss = mov(ss, M_Base_Opnd(ecx_reg, 1), eax_opnd, size_8); // mov byte[ecx+1],al + ss = ret(ss, Imm_Opnd(4)); // ret 4 + + signed offset = (signed)ss - (signed)check_zero - 1; + *check_zero = (char)offset; //check_zero: + + ss = test(ss, eax_opnd, eax_opnd, size_16); // test ax,ax + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz failed + char *failed2 = ((char *)ss) - 1; + + ss = prefix(ss, lock_prefix); //; here ax==0. + ss = cmpxchg(ss, M_Base_Opnd(ecx_reg, 2), edx_opnd, size_16); // lock cmpxchg16 [ecx+2],dx + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz failed + char *failed3 = ((char *)ss) - 1; + + +#ifdef LOCK_RESERVATION + ss = mov(ss, eax_opnd, M_Base_Opnd(ecx_reg, 1), size_8); // mov al, byte[ecx+1] + ss = test(ss, eax_opnd, Imm_Opnd(size_8, 0x4), size_8); // test al,0x4 + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz finish + char *finish = ((char *)ss) - 1; + + ss = alu(ss, add_opc, eax_opnd, Imm_Opnd(size_8, 0x8), size_8); // add al,0x8 + ss = mov(ss, M_Base_Opnd(ecx_reg, 1), eax_opnd, size_8); // mov byte[ecx+1],al + + offset = (signed)ss - (signed)finish - 1; + *finish = (char)offset; //finish: + +#endif + ss = ret(ss, Imm_Opnd(4)); // ret 4 + + offset = (signed)ss - (signed)failed1 - 1; + *failed1 = (char)offset; //failed: + + offset = (signed)ss - (signed)failed2 - 1; + *failed2 = (char)offset; + + offset = (signed)ss - (signed)failed3 - 1; + *failed3 = (char)offset; + +#endif //ASM_MONITOR_HELPER + // the second attempt to lock monitor + ss = push(ss, ecx_opnd); + ss = call(ss, (char *)hythread_thin_monitor_try_enter); + ss = alu(ss, add_opc, esp_opnd, Imm_Opnd(4)); // pop parameters + + return ss; +} + +/** + * Generates slow path of monitor enter. + * This code could block on monitor and contains safepoint. + * The appropriate m2n frame should be generated and + * + * @param[in] ss buffer to put the assembly code to + * @param[in] input_param1 register should point to the jobject(handle) + * If input_param1 == eax it reduces one register mov. + * the code use and do not restore ecx, edx, eax registers + * @return 0 if success in eax register + */ +char* gen_monitorenter_slow_path_helper(char *ss, const R_Opnd & input_param1) { + if (&input_param1 != &eax_opnd) { + ss = mov(ss, eax_opnd, input_param1); + } + + ss = push(ss, eax_opnd); // push the address of the handle + ss = call(ss, (char *)jthread_monitor_enter); + ss = alu(ss, add_opc, esp_opnd, Imm_Opnd(4)); // pop parameters + return ss; +} + +/** + * Generates monitor exit. + * The code should not contain safepoints. + * + * @param[in] ss buffer to put the assembly code to + * @param[in] input_param1 register should point to the lockword in object header. + * If input_param1 == ecx it reduce one register mov. + * The code use and do not restore eax registers. + * @return 0 if success in eax register + */ +char* gen_monitor_exit_helper(char *ss, const R_Opnd & input_param1) { + if (&input_param1 != &ecx_opnd) { + ss = mov(ss, ecx_opnd, input_param1); + } +#ifdef ASM_MONITOR_HELPER + ss = mov(ss, eax_opnd, M_Base_Opnd(ecx_reg, 0)); // mov eax,dword[ecx] + ss = test(ss, eax_opnd, Imm_Opnd(0x80000000), size_32); // test eax,0x80000000 + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); // jnz fat + char *fat = ((char *)ss) - 1; + ss = mov(ss, eax_opnd, M_Base_Opnd(ecx_reg, 1), size_8); // mov al, byte[ecx+1] + + ss = alu(ss, sub_opc, eax_opnd, Imm_Opnd(size_8,0x8),size_8); // sub al, 0x8 + ss = branch8(ss, Condition_C, Imm_Opnd(size_8, 0)); // jc zero_rec + char *zero_rec = ((char *)ss) - 1; + ss = mov(ss, M_Base_Opnd(ecx_reg, 1), eax_opnd, size_8); // mov byte[ecx+1],al + ss = ret(ss, Imm_Opnd(4)); // ret 4 + + signed offset = (signed)ss - (signed)zero_rec - 1; //zero_rec: + *zero_rec = (char)offset; + + ss = mov(ss, M_Base_Opnd(ecx_reg, 2), Imm_Opnd(size_16, 0), size_16);// mov word[ecx+2],0 + ss = ret(ss, Imm_Opnd(4)); // ret 4 + + offset = (signed)ss - (signed)fat - 1; //fat: + *fat = (char)offset; + +#endif + + ss = push(ss, ecx_opnd); + ss = call(ss, (char *)hythread_thin_monitor_exit); + ss = alu(ss, add_opc, esp_opnd, Imm_Opnd(4)); // pop parameters + return ss; +} + +/** + * Generates slow path of monitor exit. + * This code could block on monitor and contains safepoint. + * The appropriate m2n frame should be generated and + * + * @param[in] ss buffer to put the assembly code to + * @param[in] input_param1 register should point to the jobject(handle) + * If input_param1 == eax it reduces one register mov. + * the code use and do not restore ecx, edx, eax registers + * @return 0 if success in eax register + */ +char* gen_monitorexit_slow_path_helper(char *ss, const R_Opnd & input_param1) { + if (&input_param1 != &eax_opnd) { + ss = mov(ss, eax_opnd, input_param1); + } + + ss = push(ss, eax_opnd); // push the address of the handle + ss = call(ss, (char *)jthread_monitor_exit); + ss = alu(ss, add_opc, esp_opnd, Imm_Opnd(4)); // pop parameters + return ss; +} + +/** + * Generates fast accessor to the TLS for the given key.
+ * Example: + *

+  * get_thread_ptr = get_tls_helper(vm_thread_block_key);
+  * ...
+  * self = get_thread_ptr();
+  * 
+ * + * @param[in] key TLS key + * @return fast accessor to key, if one exist + */ +fast_tls_func* get_tls_helper(hythread_tls_key_t key) { + // return tm_self_tls->thread_local_storage[key]; + unsigned key_offset = (unsigned)&(((hythread_t)(0))->thread_local_storage[key]); + + const int stub_size = 126; + char *stub = (char *)malloc(stub_size); + memset(stub, 0xcc /*int 3*/, stub_size); + + char *ss = stub; + + ss = gen_hythread_self_helper(ss); + ss = mov(ss, eax_opnd, M_Base_Opnd(eax_reg, key_offset)); + ss = ret(ss, Imm_Opnd(0)); + + assert((ss - stub) < stub_size); + + return (fast_tls_func*) stub; +} diff --git a/vm/thread/src/thread_private.h b/vm/thread/src/thread_private.h index 32ddd13..124d7c4 100644 --- a/vm/thread/src/thread_private.h +++ b/vm/thread/src/thread_private.h @@ -52,9 +52,23 @@ #define SPIN_COUNT 5 #endif // !defined (_IPF_) -#if defined(WIN32) && !defined (_EM64T_) -//use optimized asm monitor enter and exit helpers -#define ASM_MONITOR_HELPER +#ifdef _EM64T_ +# ifdef _WIN64 + //don't use optimized asm monitor enter and exit helpers +# else + //use optimized asm monitor enter and exit helpers +# define ASM_MONITOR_HELPER +#endif +#else +#ifdef _IPF_ + //don't use optimized asm monitor enter and exit helpers +#else + //use optimized asm monitor enter and exit helpers +# define ASM_MONITOR_HELPER +#endif +#endif + +#ifdef WIN32 // FS14_TLS_USE define turns on windows specific TLS access optimization // We use free TIB slot with 14 offset, see following article for details // http://www.microsoft.com/msj/archive/S2CE.aspx diff --git a/vm/vmcore/src/util/em64t/base/compile_em64t.cpp b/vm/vmcore/src/util/em64t/base/compile_em64t.cpp index 1847071..076c157 100644 --- a/vm/vmcore/src/util/em64t/base/compile_em64t.cpp +++ b/vm/vmcore/src/util/em64t/base/compile_em64t.cpp @@ -142,6 +142,27 @@ #endif // _WIN64 } } +// Convert a reference, if null, from a managed null +// (represented by heap_base) to an unmanaged one (NULL/0). Uses %rdi. +char * gen_convert_managed_to_unmanaged_null_em64t(char * ss, + const R_Opnd & input_param1) { + if (&input_param1 != &rdi_opnd) { + ss = mov(ss, rdi_opnd, input_param1); + } + + if (VM_Global_State::loader_env->compress_references) { + ss = mov(ss, rcx_opnd, Imm_Opnd(size_64, (int64)VM_Global_State::loader_env->heap_base)); + ss = alu(ss, cmp_opc, rdi_opnd, rcx_opnd); + ss = branch8(ss, Condition_NE, Imm_Opnd(size_8, 0)); // not null, branch around the mov 0 + char *backpatch_address__not_managed_null = ((char *)ss) - 1; + ss = mov(ss, rdi_opnd, Imm_Opnd(0)); + signed offset = (signed)ss - (signed)backpatch_address__not_managed_null - 1; + *backpatch_address__not_managed_null = (char)offset; + } + return ss; +} + + /* BEGIN COMPILE-ME STUBS */ // compile_me stack frame diff --git a/vm/vmcore/src/util/em64t/base/jit_lock_rt_support_em64t.cpp b/vm/vmcore/src/util/em64t/base/jit_lock_rt_support_em64t.cpp index 09eea29..c165feb 100644 --- a/vm/vmcore/src/util/em64t/base/jit_lock_rt_support_em64t.cpp +++ b/vm/vmcore/src/util/em64t/base/jit_lock_rt_support_em64t.cpp @@ -26,10 +26,22 @@ #include #include "environment.h" #include "open/hythread_ext.h" +#include "open/jthread.h" +#include "open/thread_helpers.h" + +#include "open/vm_util.h" +#include "encoder.h" +#include "nogc.h" +#include "compile.h" + +#include "exceptions_jit.h" #include "lil.h" #include "lil_code_generator.h" -#include "jit_runtime_support.h" +#include "../m2n_em64t_internal.h" +#include "object_handles.h" #include "Class.h" +#include "jit_runtime_support.h" + #include "mon_enter_exit.h" #include "exceptions.h" #include "exceptions_jit.h" @@ -37,9 +49,262 @@ #include "exceptions_jit.h" #define LOG_DOMAIN "vm.helpers" #include "cxxlog.h" -#include "vm_stats.h" #include "dump.h" +#include "vm_stats.h" + +// Linix x86-64 fast helpers +char * gen_convert_managed_to_unmanaged_null_em64t(char * ss, + const R_Opnd & input_param1); +#define INPUT_ARG_OFFSET 8 +char * gen_setup_j2n_frame(char * s); +char * gen_pop_j2n_frame(char * s); + + +// patch_addr_null_arg_ptr is the address of a variable holding the +// address of a branch instruction byte to patch with the destination +// to be taken if the struct Class* argument is NULL. +static char * gen_convert_struct_class_to_object(char *ss, char **patch_addr_null_arg_ptr) +{ + // First make sure the struct Class* argument is non-NULL. + ss = test(ss, rdi_opnd, rdi_opnd); + ss = branch8(ss, Condition_Z, Imm_Opnd(size_8, 0)); + *patch_addr_null_arg_ptr = ((char *)ss) - 1; + + // Convert the struct Class* argument to the corresponding java_lang_Class reference. + ss = call(ss, (char *)struct_Class_to_java_lang_Class); + ss = mov(ss, rdi_opnd, rax_opnd); // overwrite the struct Class* with the raw java_lang_Class reference + return ss; +} //gen_convert_struct_class_to_object + +// Helper for monenter intstruction +static char * gen_restore_monitor_enter(char *ss, char *patch_addr_null_arg) +{ + // Obtain lockword offset for the given object + const unsigned header_offset = ManagedObject::header_offset(); + signed offset; + assert(header_offset); +#ifdef VM_STATS +// uint64* incr = &(VM_Statistics::get_vm_stats().num_monitor_enter); +// ss = inc(ss, M_Opnd((int64)incr)); +#endif + + ss = test(ss, rdi_opnd, rdi_opnd); + ss = branch8(ss, Condition_Z, Imm_Opnd(size_8, 0)); + char *backpatch_address__null_pointer = ((char *)ss) - 1; + + // Fast path + ss = push(ss, rdi_opnd); + ss = alu(ss, add_opc, rdi_opnd, Imm_Opnd(header_offset)); // pop parameters + ss = gen_monitorenter_fast_path_helper(ss, rdi_opnd); + ss = pop(ss, rdi_opnd); + + ss = test(ss, rax_opnd, rax_opnd); + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); + char *backpatch_address__fast_monitor_failed = ((char *)ss) - 1; + ss = ret(ss); + + // Slow path: happens when the monitor is busy (contention case) + offset = (int64)ss - (int64)backpatch_address__fast_monitor_failed - 1; + *backpatch_address__fast_monitor_failed = (char)offset; + + ss = gen_setup_j2n_frame(ss); + + ss = call(ss, (char *)oh_convert_to_local_handle); + ss = gen_monitorenter_slow_path_helper(ss, rax_opnd); + + ss = gen_pop_j2n_frame(ss); + + ss = ret(ss); + + // Handle NPE here + int64 npe_offset = (int64)ss - (int64)backpatch_address__null_pointer - 1; + *backpatch_address__null_pointer = (char)npe_offset; + if (patch_addr_null_arg != NULL) { + npe_offset = (int64)ss - (int64)patch_addr_null_arg - 1; + *patch_addr_null_arg = (char)npe_offset; + } + + // Object is null so throw a null pointer exception + ss = jump(ss, (char*)exn_get_rth_throw_null_pointer()); + + return ss; +} //gen_restore_monitor_enter + +void * getaddress__vm_monitor_enter_naked() +{ + static void *addr = NULL; + if (addr != NULL) { + return addr; + } + + const int stub_size = 160; + char *stub = (char *)malloc_fixed_code_for_jit(stub_size, DEFAULT_CODE_ALIGNMENT, CODE_BLOCK_HEAT_MAX/2, CAA_Allocate); +#ifdef _DEBUG + memset(stub, 0xcc /*int 3*/, stub_size); +#endif + char *ss = stub; + +#ifdef VM_STATS + int * value = VM_Statistics::get_vm_stats().rt_function_calls.lookup_or_add((void*)VM_RT_MONITOR_ENTER, 0, NULL); + ss = inc(ss, M_Opnd((int64)value)); +#endif + + ss = gen_restore_monitor_enter(ss, /*patch_addr_null_arg*/ NULL); + + addr = stub; + assert((ss - stub) < stub_size); + + compile_add_dynamic_generated_code_chunk("vm_monitor_enter_naked", stub, stub_size); + + // Put TI support here + DUMP_STUB(stub, "getaddress__vm_monitor_enter_naked", ss - stub); + + return addr; +} + +void * getaddress__vm_monitor_enter_static_naked() +{ + static void *addr = NULL; + if (addr != NULL) { + return addr; + } + + const int stub_size = 176; + char *stub = (char *)malloc_fixed_code_for_jit(stub_size, DEFAULT_CODE_ALIGNMENT, CODE_BLOCK_HEAT_MAX/2, CAA_Allocate); +#ifdef _DEBUG + memset(stub, 0xcc /*int 3*/, stub_size); +#endif + char *ss = stub; + +#ifdef VM_STATS + int * value = VM_Statistics::get_vm_stats().rt_function_calls.lookup_or_add((void*)VM_RT_MONITOR_ENTER_STATIC, 0, NULL); + ss = inc(ss, M_Opnd((int64)value)); +#endif + + char *patch_addr_null_arg; + ss = gen_convert_struct_class_to_object(ss, &patch_addr_null_arg); + ss = gen_restore_monitor_enter(ss, patch_addr_null_arg); + + addr = stub; + assert((ss - stub) < stub_size); + + compile_add_dynamic_generated_code_chunk("vm_monitor_enter_static_naked", stub, stub_size); + + if (VM_Global_State::loader_env->TI->isEnabled()) + jvmti_send_dynamic_code_generated_event("vm_monitor_enter_static_naked", stub, stub_size); + + DUMP_STUB(stub, "getaddress__vm_monitor_enter_static_naked", ss - stub); + + return addr; +} //getaddress__vm_monitor_enter_static_naked + +static char * gen_restore_monitor_exit(char *ss, char *patch_addr_null_arg) +{ + + const unsigned header_offset = ManagedObject::header_offset(); +#ifdef VM_STATS +// uint64* incr = &(VM_Statistics::get_vm_stats().num_monitor_exit); +// ss = inc(ss, M_Opnd((int64)incr)); +#endif + + ss = test(ss, rdi_opnd, rdi_opnd); + ss = branch8(ss, Condition_Z, Imm_Opnd(size_8, 0)); + char *backpatch_address__null_pointer = ((char *)ss) - 1; + + // Fast path only + ss = alu(ss, add_opc, rdi_opnd, Imm_Opnd(header_offset)); + ss = gen_monitor_exit_helper(ss, rdi_opnd); + + ss = test(ss, rax_opnd, rax_opnd); + ss = branch8(ss, Condition_NZ, Imm_Opnd(size_8, 0)); + char *backpatch_address__fast_monitor_failed = ((char *)ss) - 1; + ss = ret(ss); + + signed offset = (signed)ss - (signed)backpatch_address__fast_monitor_failed - 1; + *backpatch_address__fast_monitor_failed = (char)offset; + + // Monitor illegal state happend + ss = jump(ss, (char*)exn_get_rth_throw_illegal_state_exception()); + + offset = (signed)ss - (signed)backpatch_address__null_pointer - 1; + *backpatch_address__null_pointer = (char)offset; + if (patch_addr_null_arg != NULL) { + offset = (signed)ss - (signed)patch_addr_null_arg - 1; + *patch_addr_null_arg = (char)offset; + } + + // Object is null so throw a null pointer exception + ss = jump(ss, (char*)exn_get_rth_throw_null_pointer()); + + return ss; +} //gen_restore_monitor_exit + +void * getaddress__vm_monitor_exit_naked() +{ + static void *addr = NULL; + if (addr != NULL) { + return addr; + } + + const int stub_size = 112; + char *stub = (char *)malloc_fixed_code_for_jit(stub_size, DEFAULT_CODE_ALIGNMENT, CODE_BLOCK_HEAT_MAX/2, CAA_Allocate); + char *ss = stub; + +#ifdef VM_STATS + int * value = VM_Statistics::get_vm_stats().rt_function_calls.lookup_or_add((void*)VM_RT_MONITOR_EXIT, 0, NULL); + ss = inc(ss, M_Opnd((int64)value)); +#endif + + ss = gen_convert_managed_to_unmanaged_null_em64t((Emitter_Handle)ss, rdi_opnd); + ss = gen_restore_monitor_exit(ss, /*patch_addr_null_arg*/ NULL); + + addr = stub; + assert((ss - stub) < stub_size); + + compile_add_dynamic_generated_code_chunk("vm_monitor_exit_naked", stub, stub_size); + + if (VM_Global_State::loader_env->TI->isEnabled()) + jvmti_send_dynamic_code_generated_event("vm_monitor_exit_naked", stub, stub_size); + + DUMP_STUB(stub, "getaddress__vm_monitor_exit_naked", ss - stub); + + return addr; +} //getaddress__vm_monitor_exit_naked + +void * getaddress__vm_monitor_exit_static_naked() +{ + static void *addr = NULL; + if (addr != NULL) { + return addr; + } + + const int stub_size = 112; + char *stub = (char *)malloc_fixed_code_for_jit(stub_size, DEFAULT_CODE_ALIGNMENT, CODE_BLOCK_HEAT_MAX/2, CAA_Allocate); + char *ss = stub; + +#ifdef VM_STATS + int * value = VM_Statistics::get_vm_stats().rt_function_calls.lookup_or_add((void*)VM_RT_MONITOR_EXIT_STATIC, 0, NULL); + ss = inc(ss, M_Opnd((int64)value)); +#endif + + char *patch_addr_null_arg; + ss = gen_convert_struct_class_to_object(ss, &patch_addr_null_arg); + ss = gen_restore_monitor_exit(ss, patch_addr_null_arg); + + addr = stub; + assert((ss - stub) < stub_size); + + compile_add_dynamic_generated_code_chunk("vm_monitor_exit_static_naked", stub, stub_size); + + if (VM_Global_State::loader_env->TI->isEnabled()) + jvmti_send_dynamic_code_generated_event("vm_monitor_exit_static_naked", stub, stub_size); + + DUMP_STUB(stub, "getaddress__vm_monitor_exit_static_naked", ss - stub); + + return addr; +} //getaddress__vm_monitor_exit_static_naked +// Windows x86-64 helpers static LilCodeStub * rth_get_lil_monitor_enter_generic(LilCodeStub * cs) { if(VM_Global_State::loader_env->TI->isEnabled() && VM_Global_State::loader_env->TI->get_global_capability( diff --git a/vm/vmcore/src/util/em64t/base/jit_runtime_support_em64t.cpp b/vm/vmcore/src/util/em64t/base/jit_runtime_support_em64t.cpp index 249256b..875abaf 100644 --- a/vm/vmcore/src/util/em64t/base/jit_runtime_support_em64t.cpp +++ b/vm/vmcore/src/util/em64t/base/jit_runtime_support_em64t.cpp @@ -27,6 +27,8 @@ #include "open/gc.h" #include "port_general.h" #include "heap.h" #include "vm_threads.h" +#include "nogc.h" +#include "compile.h" #include "lil.h" #include "lil_code_generator.h" @@ -170,6 +172,118 @@ #endif return vm_rt_multianewarray_recursive(c, lens, dims); } +static void* getaddress__setup_java_to_native_frame() +{ + static void *addr = 0; + if (addr) { + return addr; + } + + const int stub_size = 32 + m2n_push_m2n_size(1, 0); + char *stub = (char *)malloc_fixed_code_for_jit(stub_size, DEFAULT_CODE_ALIGNMENT, CODE_BLOCK_HEAT_MAX/2, CAA_Allocate); + +#ifdef _DEBUG + memset(stub, 0xcc /*int 3*/, stub_size); +#endif + char *ss = stub; + + // Stack changes + // prev new + // + // ... ... + // -------- -------- ------------ + // ret ret + // -------- -------- + // ret r12 + // -------- -------- m2n frame + // + // ... + // + // -------- ------------ + // ret + // -------- + + ss = alu(ss, sub_opc, rsp_opnd, Imm_Opnd(m2n_sizeof_m2n_frame - 8)); + ss = mov(ss, r11_opnd, M_Base_Opnd(rsp_reg, m2n_sizeof_m2n_frame - 8)); + ss = mov(ss, M_Base_Opnd(rsp_reg, m2n_sizeof_m2n_frame - 8), r12_opnd); + ss = mov(ss, M_Base_Opnd(rsp_reg, 0), r11_opnd); + ss = mov(ss, r12_opnd, rdi_opnd); + + ss = m2n_gen_push_m2n(ss, NULL, FRAME_UNKNOWN, false, 1, 0, + m2n_sizeof_m2n_frame); + ss = mov(ss, rdi_opnd, r12_opnd); + ss = ret(ss); + + assert((ss - stub) <= stub_size); + addr = stub; + + compile_add_dynamic_generated_code_chunk("setup_java_to_native_frame", stub, stub_size); + + // Put TI support here. + DUMP_STUB(stub, "getaddress__setup_java_to_native_frame", ss - stub); + + return addr; +} //getaddress__setup_java_to_native_frame + +VMEXPORT char *gen_setup_j2n_frame(char *s) +{ + s = call(s, (char *)getaddress__setup_java_to_native_frame() ); + return s; +} //setup_j2n_frame + +static void m2n_free_local_handles() { + assert(!hythread_is_suspend_enabled()); + + if (exn_raised()) { + exn_rethrow(); + } + + M2nFrame * m2n = m2n_get_last_frame(); + free_local_object_handles3(m2n->local_object_handles); +} + +static void* getaddress__pop_java_to_native_frame() +{ + static void *addr = 0; + if (addr) { + return addr; + } + + const int stub_size = 32 + m2n_pop_m2n_size(false, 1, 0); + char *stub = (char *)malloc_fixed_code_for_jit(stub_size, DEFAULT_CODE_ALIGNMENT, CODE_BLOCK_HEAT_MAX/2, CAA_Allocate); +#ifdef _DEBUG + memset(stub, 0xcc /*int 3*/, stub_size); +#endif + char *ss = stub; + + ss = mov(ss, r12_opnd, rax_opnd); + + ss = m2n_gen_pop_m2n(ss, false, 1, 8, 0); + + ss = mov(ss, rax_opnd, r12_opnd); + ss = mov(ss, r11_opnd, M_Base_Opnd(rsp_reg, 0)); + ss = mov(ss, r12_opnd, M_Base_Opnd(rsp_reg, m2n_sizeof_m2n_frame - 8)); + ss = mov(ss, M_Base_Opnd(rsp_reg, m2n_sizeof_m2n_frame - 8), r11_opnd); + ss = alu(ss, add_opc, rsp_opnd, Imm_Opnd(m2n_sizeof_m2n_frame - 8)); + ss = ret(ss); + + assert((ss - stub) <= stub_size); + addr = stub; + + compile_add_dynamic_generated_code_chunk("pop_java_to_native_frame", stub, stub_size); + + // Put TI support here. + DUMP_STUB(stub, "getaddress__pop_java_to_native_frame", ss - stub); + + return addr; +} //getaddress__pop_java_to_native_frame + +VMEXPORT char *gen_pop_j2n_frame(char *s) +{ + s = call(s, (char *)getaddress__pop_java_to_native_frame() ); + return s; +} //setup_j2n_frame + // see jit_lock_rt_support.cpp for the implementation NativeCodePtr rth_get_lil_monitor_enter_static(); @@ -180,6 +294,11 @@ NativeCodePtr rth_get_lil_monitor_exit_s NativeCodePtr rth_get_lil_monitor_exit(); NativeCodePtr rth_get_lil_monitor_exit_non_null(); +void * getaddress__vm_monitor_enter_naked(); +void * getaddress__vm_monitor_enter_static_naked(); +void * getaddress__vm_monitor_exit_naked(); +void * getaddress__vm_monitor_exit_static_naked(); + void * vm_get_rt_support_addr(VM_RT_SUPPORT f) { #ifdef VM_STATS @@ -188,8 +307,9 @@ #endif // VM_STATS NativeCodePtr res = rth_get_lil_helper(f); if (res) return res; - + switch(f) { +#ifdef _WIN64 // Monitor enter runtime helpers case VM_RT_MONITOR_ENTER_STATIC: return rth_get_lil_monitor_enter_static(); @@ -205,6 +325,23 @@ #endif // VM_STATS return rth_get_lil_monitor_exit(); case VM_RT_MONITOR_EXIT_NON_NULL: return rth_get_lil_monitor_exit_non_null(); +#else + // Monitor enter runtime helpers + case VM_RT_MONITOR_ENTER_STATIC: + return getaddress__vm_monitor_enter_static_naked(); + case VM_RT_MONITOR_ENTER: + return getaddress__vm_monitor_enter_naked(); + case VM_RT_MONITOR_ENTER_NON_NULL: + return getaddress__vm_monitor_enter_naked(); + + // Monitor exit runtime helpers + case VM_RT_MONITOR_EXIT_STATIC: + return getaddress__vm_monitor_exit_static_naked(); + case VM_RT_MONITOR_EXIT: + return getaddress__vm_monitor_exit_naked(); + case VM_RT_MONITOR_EXIT_NON_NULL: + return getaddress__vm_monitor_exit_naked(); +#endif // Object creation helper case VM_RT_NEW_RESOLVED_USING_VTABLE_AND_SIZE: -- 1.4.2