Index: vm/include/open/ee_em_intf.h =================================================================== --- vm/include/open/ee_em_intf.h (revision 580018) +++ vm/include/open/ee_em_intf.h (working copy) @@ -98,6 +98,23 @@ */ JITEXPORT bool JIT_enable_profiling(JIT_Handle jit, PC_Handle pc, EM_JIT_PC_Role role); + + +/** +* Notifies JIT that profile is collected. +* +* EM uses this method to notify JIT that profile is collected. +* JIT could use this information to patch profiling counters. +* +* @param[in] jit - the JIT instance handle +* @param[in] pc - the handle of the profile collector instance +* @param[in] mh - the handle of the method with collected profile +* +* @note The given method is optional. Currently only JET supports this method. +*/ +JITEXPORT void JIT_profile_notification_callback(JIT_Handle jit, PC_Handle pc, Method_Handle mh); + + #ifdef __cplusplus } #endif Index: vm/include/open/em.h =================================================================== --- vm/include/open/em.h (revision 580018) +++ vm/include/open/em.h (working copy) @@ -57,7 +57,7 @@ */ typedef enum JIT_Result { /** - * The method compilation has finished successufuly. + * The method compilation has finished successfully. */ JIT_SUCCESS, /** Index: vm/em/src/DrlEMImpl.cpp =================================================================== --- vm/em/src/DrlEMImpl.cpp (revision 580018) +++ vm/em/src/DrlEMImpl.cpp (working copy) @@ -72,7 +72,6 @@ emInstance = NULL; } - static EM_PCTYPE get_pc_type(EM_Handle _this, PC_Handle pc) { assert(_this!=NULL); assert(pc!=NULL); @@ -86,7 +85,6 @@ return (PC_Handle)em->getProfileCollector(profile_type, jh, jit_role); } - static Method_Profile_Handle get_method_profile(EM_Handle _this, PC_Handle pch, Method_Handle mh) { assert(_this!=NULL); @@ -97,7 +95,6 @@ } - RStep::RStep(JIT_Handle _jit, const std::string& _jitName, RChain* _chain, apr_dso_handle_t* _libHandle) : jit(_jit), jitName(_jitName), catName(std::string(LOG_DOMAIN)+"."+_jitName), chain(_chain), loggingEnabled(false), enable_profiling(NULL), @@ -373,6 +370,9 @@ return false; } +static void profile_notification_callback_stub(JIT_Handle, PC_Handle, Method_Handle) { +} + bool DrlEMImpl::initJIT(const std::string& libName, apr_dso_handle_t* libHandle, RStep& step) { apr_dso_handle_sym_t fn = NULL; if (apr_dso_sym(&fn, libHandle, "JIT_init") != APR_SUCCESS) { @@ -395,6 +395,12 @@ step.enable_profiling = enable_profiling_stub; } + if (pcEnabled && apr_dso_sym(&fn, libHandle, "JIT_profile_notification_callback") == APR_SUCCESS) { + step.profile_notification_callback = (void(*)(JIT_Handle, PC_Handle, Method_Handle))fn; + } else { + step.profile_notification_callback = profile_notification_callback_stub; + } + return true; } @@ -749,6 +755,7 @@ hymutex_unlock(&recompilationLock); return; } + methodsInRecompile.insert((Method_Profile_Handle)mp); nMethodsRecompiled++; hymutex_unlock(&recompilationLock); @@ -764,10 +771,13 @@ for (RSteps::const_iterator sit = chain->steps.begin(), send = chain->steps.end(); sit!=send; ++sit) { RStep* step = *sit; if (step->jit == jit) { + + //notify JIT which has generated this profile that profile is ready. + step->profile_notification_callback(step->jit, mp->pc, mp->mh); + ++sit; RStep* nextStep = sit!=send ? *sit: NULL; if (nextStep != NULL) { - if (nextStep->loggingEnabled) { methodName = method_get_name(mp->mh); Class_Handle ch = method_get_class(mp->mh); Index: vm/em/src/DrlEMImpl.h =================================================================== --- vm/em/src/DrlEMImpl.h (revision 580018) +++ vm/em/src/DrlEMImpl.h (working copy) @@ -55,6 +55,7 @@ apr_dso_handle_t* libHandle; bool (*enable_profiling)(JIT_Handle, PC_Handle, EM_JIT_PC_Role); + void (*profile_notification_callback)(JIT_Handle, PC_Handle, Method_Handle); }; Index: vm/jitrino/config/ia32/client.emconf =================================================================== --- vm/jitrino/config/ia32/client.emconf (revision 580018) +++ vm/jitrino/config/ia32/client.emconf (working copy) @@ -36,13 +36,13 @@ CD_OPT.useProfile=EB_PROF -EB_PROF.mode=SYNC +EB_PROF.mode=ASYNC EB_PROF.entryThreshold=10000 EB_PROF.backedgeThreshold=100000 # these options are used only in ASYNC profiler mode -#EB_PROF.tbsTimeout=5 -#EB_PROF.tbsInitialTimeout=0 +EB_PROF.tbsTimeout=5 +EB_PROF.tbsInitialTimeout=5 Index: vm/jitrino/src/jet/cg.h =================================================================== --- vm/jitrino/src/jet/cg.h (revision 580317) +++ vm/jitrino/src/jet/cg.h (working copy) @@ -102,6 +102,32 @@ return slots; } +/** + * The structure contains information about counter for patching profiling counter after + * profile is ready + */ +struct ProfileCounterInfo { + //This field contains composite info on counter size (first byte) and offset (last 3 bytes) + uint32 offsetInfo; + //Link to the basic block to calculate counter's offset after code layout + BBInfo* bb; + ProfileCounterInfo() : offsetInfo(0), bb(NULL){} + + static uint32 getInstSize(uint32 offsetInfo) { return offsetInfo >> 24;} + static uint32 getInstOffset(uint32 offsetInfo) { return offsetInfo & 0x00FFFFFF;} + static uint32 createOffsetInfo(uint32 instSize, uint32 instOffset) { + assert(instSize<0xFF && instOffset<0xFFFFFF); + return (instSize<<24) | (instOffset); + } +}; + +/** + * Map contains patching information for all the counters in current method + */ +typedef std::vector ProfileCounterInfos; + + + /** the class is used by codegen internally to keep field information */ class FieldOpInfo { public: @@ -1288,6 +1314,8 @@ */ Compile_Handle m_compileHandle; + ProfileCounterInfos m_profileCountersMap; + }; Index: vm/jitrino/src/jet/mib.cpp =================================================================== --- vm/jitrino/src/jet/mib.cpp (revision 580317) +++ vm/jitrino/src/jet/mib.cpp (working copy) @@ -20,7 +20,7 @@ */ /** * @file - * @brief MethodInfoBlock implementaion. + * @brief MethodInfoBlock implementation. */ #include "mib.h" @@ -36,6 +36,9 @@ // rt_header = &m_header; rt_header->code_start = NULL; + num_profiler_counters = 0; + profiler_counters_map= NULL; + } void MethodInfoBlock::init(unsigned bc_size, unsigned stack_max, @@ -51,10 +54,10 @@ rt_header->m_in_slots = in_slots; rt_header->m_flags = flags; - // All the values must be initialized *before* get_dyn_size() ! - unsigned dyn_size = get_dyn_size(); - m_data = new char[dyn_size]; - memset(m_data, 0, dyn_size); + // All the values must be initialized *before* get_bcmap_size() ! + unsigned bcmap_size = get_bcmap_size(); + m_data = new char[bcmap_size]; + memset(m_data, 0, bcmap_size); rt_inst_addrs = (const char**)m_data; } @@ -62,8 +65,15 @@ { memset(to, 0, get_total_size()); memcpy(to, rt_header, get_hdr_size()); - memcpy(to + get_hdr_size(), m_data, get_dyn_size()); + memcpy(to + get_hdr_size(), m_data, get_bcmap_size()); rt_inst_addrs = (const char**)(to + get_hdr_size()); + + //store information about profiling counters for the method in MethodInfoBlock + uint32* countersInfo = (uint32*)(to + get_hdr_size() + get_bcmap_size()); + countersInfo[0]=num_profiler_counters; + if (num_profiler_counters > 0) { + memcpy(countersInfo+1, profiler_counters_map, num_profiler_counters * sizeof(uint32)); + } } const char * MethodInfoBlock::get_ip(unsigned pc) const Index: vm/jitrino/src/jet/cg_instr.cpp =================================================================== --- vm/jitrino/src/jet/cg_instr.cpp (revision 580317) +++ vm/jitrino/src/jet/cg_instr.cpp (working copy) @@ -31,6 +31,10 @@ namespace Jitrino { namespace Jet { +//Number of nops to put before counter instruction to be able to atomically +//replace beginning of the instruction with jump +#define NOPS_PER_COUNTER 1 + void CodeGen::gen_prof_be(void) { if (!is_set(JMF_PROF_ENTRY_BE)) { @@ -45,7 +49,17 @@ movp(addr, m_p_backedge_counter); int off = 0; #endif + uint32 offset = (uint32)(m_codeStream.ip() - m_codeStream.data() - m_bbinfo->ipoff); //store offsets inside of BB now. Fix it to method's offset after code layout + //put number of nops to align counter instruction + nop(NOPS_PER_COUNTER); alu(alu_add, Opnd(i32, addr, off), 1); + + //store information about profiling counters + uint32 new_offset = (uint32)(m_codeStream.ip() - m_codeStream.data() - m_bbinfo->ipoff); + ProfileCounterInfo info; + info.bb = m_bbinfo; + info.offsetInfo = ProfileCounterInfo::createOffsetInfo(new_offset - offset, offset); + m_profileCountersMap.push_back(info); } void CodeGen::gen_gc_safe_point() Index: vm/jitrino/src/jet/enc.h =================================================================== --- vm/jitrino/src/jet/enc.h (revision 580317) +++ vm/jitrino/src/jet/enc.h (working copy) @@ -893,7 +893,7 @@ * 'mov mem, fr' using FLD, FST and FXCH. * * @todo IPF support. The basic idea is to hide one or two registers from - * application and use them in Ecoder internally to emulate complex address + * application and use them in Encoder internally to emulate complex address * form and other operations that are not natively support in IPF's * instruction set. */ @@ -997,6 +997,16 @@ } alu_impl(alu, op0, op1); } + + /** + * Generates n-byte long NOP instruction. + */ + void nop(uint32 n) { + if (is_trace_on()) { + trace(string("nop"), to_str((int)n), string()); + } + nop_impl(n); + } /** * Performs bitwise NOT operation. @@ -1477,6 +1487,8 @@ void not_impl(const Opnd& op0); /// Implementation of alu(). void alu_impl(ALU op, const Opnd& op0, const Opnd& op1); + //Implementation of nop() + void nop_impl(uint32 n); /// Implementation of cmovcc(). void cmovcc_impl(COND c, const Opnd& op0, const Opnd& op1); /// Implementation of cmpxchg(). Index: vm/jitrino/src/jet/compiler.cpp =================================================================== --- vm/jitrino/src/jet/compiler.cpp (revision 580317) +++ vm/jitrino/src/jet/compiler.cpp (working copy) @@ -503,6 +503,19 @@ // // runtime data. must be initialized before code patching // + + //register profiler counters mapping info if present + if (!m_profileCountersMap.empty()) { + m_infoBlock.num_profiler_counters = m_profileCountersMap.size(); + m_infoBlock.profiler_counters_map = new uint32[m_infoBlock.num_profiler_counters]; + for (size_t i =0; iaddr - m_vmCode); + uint32 offsetInfo = ProfileCounterInfo::createOffsetInfo(ProfileCounterInfo::getInstSize(info.offsetInfo), offset); + m_infoBlock.profiler_counters_map[i]=offsetInfo; + } + } + unsigned data_size = m_infoBlock.get_total_size(); char * pdata; if (m_bEmulation) { Index: vm/jitrino/src/jet/rt.cpp =================================================================== --- vm/jitrino/src/jet/rt.cpp (revision 580317) +++ vm/jitrino/src/jet/rt.cpp (working copy) @@ -32,6 +32,7 @@ #include "jit_intf.h" #include "port_threadunsafe.h" +#include "EMInterface.h" #if !defined(_IPF_) #include "enc_ia32.h" @@ -592,5 +593,62 @@ return EXE_ERROR_NONE; } + +void rt_profile_notification_callback(JIT_Handle jit, PC_Handle pch, Method_Handle mh) +{ + JITInstanceContext* jitContext = JITInstanceContext::getContextForJIT(jit); + + //heck that profiler type is EB, counters are patched only for Entry Backage profile + if ((jitContext->getProfilingInterface())->getProfileType(pch) == EM_PCTYPE_ENTRY_BACKEDGE) { + //Get MethodInfoBlock of the method to be patched + char * pinfo = (char*)method_get_info_block_jit(mh, jit); + assert(MethodInfoBlock::is_valid_data(pinfo)); + MethodInfoBlock infoBlock(pinfo); + + if (infoBlock.get_flags() & DBG_TRACE_RT) { + const char* methodName = method_get_name(mh); + const char* className = class_get_name(method_get_class(mh)); + dbg_rt("rt.patch_eb_counters: patching method %s.%s\n", className, methodName); + } + + //Replace counters with nops in 3 steps: + //1. Atomically replace first 2 bytes of counter instruction with jump to the + //next instruction + //2. Replace all the remaining bytes of counter instruction with nops + //3. Atomically replace jump with 2 nops + Byte* methodAddr = method_get_code_block_addr_jit(mh, jit); + for (uint32 i = 0 ; i 0) { + profiler_counters_map = (uint32*)(countersInfo + sizeof(uint32)); + } + profiler_counters_map = (uint32*)(countersInfo + sizeof(uint32)); } /** @@ -391,7 +403,7 @@ } /** - * @brief Calculates and returns size occupied by variable part of + * @brief Calculates and returns size occupied by bc-map related variable part of * MethodInfoBlock. * * @note Obviously, all fields used to calculate the variable size @@ -400,11 +412,20 @@ * * @return size needed to store variable part of MethodInfoBlock. */ - unsigned get_dyn_size(void) const + unsigned get_bcmap_size(void) const { return sizeof(rt_inst_addrs[0])*rt_header->m_bc_size; } + /** + * @brief Calculates and returns size occupied by profiling counters info in + * MethodInfoBlock. + * + */ + unsigned get_profile_counters_map_size() const { + return sizeof (uint32) + num_profiler_counters * sizeof(uint32); + } + /** * @brief Disallows copying. */ @@ -527,9 +548,16 @@ * #init). During runtime, it points into the buffer provided to #load. */ const char ** rt_inst_addrs; // [bc_size] - char ** data_ips; // [m_num_datas] - unsigned* depths; // [m_num_datas] - unsigned* masks; // [m_num_datas*words(m_max_stack_depth)] + +public: + /** + /* Number of profile counters in the method + */ + uint32 num_profiler_counters; + /** + /* Profiling counters information for patching: counter size and offset + */ + uint32* profiler_counters_map; }; Index: vm/jitrino/src/jet/enc_ia32.cpp =================================================================== --- vm/jitrino/src/jet/enc_ia32.cpp (revision 580317) +++ vm/jitrino/src/jet/enc_ia32.cpp (working copy) @@ -680,6 +680,12 @@ ip(EncoderBase::encode(ip(), mn, args)); } +void Encoder::nop_impl(uint32 n) +{ + ip(EncoderBase::nops(ip(), n)); +} + + void Encoder::cmovcc_impl(COND c, const Opnd& op0, const Opnd& op1) { ConditionMnemonic cm = devirt(c); Index: vm/jitrino/src/jet/jet.h =================================================================== --- vm/jitrino/src/jet/jet.h (revision 580317) +++ vm/jitrino/src/jet/jet.h (working copy) @@ -80,6 +80,7 @@ void rt_unwind(JIT_Handle jit, Method_Handle method, JitFrameContext * context); + /** * @brief Enumerates root set for a given method. * @@ -247,6 +248,12 @@ */ OpenMethodExecutionParams get_exe_capabilities(); +/** +* @brief Notifies JET that profile is collected and counters could be removed +* now. +*/ +void rt_profile_notification_callback(JIT_Handle jit, PC_Handle pc, Method_Handle mh); + }}; // ~namespace Jitrino::Jet #endif // ~__JET_H_INCLUDED__ Index: vm/jitrino/src/vm/drl/DrlEMInterface.cpp =================================================================== --- vm/jitrino/src/vm/drl/DrlEMInterface.cpp (revision 580018) +++ vm/jitrino/src/vm/drl/DrlEMInterface.cpp (working copy) @@ -59,6 +59,9 @@ return profileAccessInterface->get_method_profile(emHandle, getPCHandle(type), md.getMethodHandle()); } +EM_PCTYPE ProfilingInterface::getProfileType(PC_Handle pch) const { + return profileAccessInterface->get_pc_type(emHandle, pch); +} bool ProfilingInterface::hasMethodProfile(ProfileType type, MethodDesc& md, JITProfilingRole role) const { Index: vm/jitrino/src/vm/drl/DrlJITInterface.cpp =================================================================== --- vm/jitrino/src/vm/drl/DrlJITInterface.cpp (revision 580018) +++ vm/jitrino/src/vm/drl/DrlJITInterface.cpp (working copy) @@ -151,6 +151,17 @@ extern "C" JITEXPORT void +JIT_profile_notification_callback(JIT_Handle jit, PC_Handle pc, Method_Handle mh) +{ + JITInstanceContext* jitContext = Jitrino::getJITInstanceContext(jit); + if (jitContext->isJet()) { + Jet::rt_profile_notification_callback(jit, pc, mh); + } //opt does not support counters patching today. +} + + +extern "C" +JITEXPORT void JIT_gc_start(JIT_Handle jit) { } Index: vm/jitrino/src/vm/EMInterface.h =================================================================== --- vm/jitrino/src/vm/EMInterface.h (revision 580018) +++ vm/jitrino/src/vm/EMInterface.h (working copy) @@ -107,6 +107,8 @@ // at run-time i.e. when there is no any memory managers available. Method_Profile_Handle getMethodProfileHandle(ProfileType type, MethodDesc& md) const; + EM_PCTYPE getProfileType(PC_Handle pc) const; + bool hasMethodProfile(ProfileType type, MethodDesc& md, JITProfilingRole role=JITProfilingRole_USE) const; bool enableProfiling(PC_Handle pc, JITProfilingRole role); bool isProfilingEnabled(ProfileType pcType, JITProfilingRole jitRole) const;