Index: vm/jitrino/config/em64t/server_static.emconf =================================================================== --- vm/jitrino/config/em64t/server_static.emconf (revision 627700) +++ vm/jitrino/config/em64t/server_static.emconf (working copy) @@ -32,7 +32,7 @@ -XX:jit.SS_OPT.path=opt_init,translator,optimizer,hir2lir,codegen --XX:jit.SS_OPT.path.optimizer=ssa,simplify,dce,uce,statprof,devirt_virtual,unguard,devirt_intf,hlo_api_magic,inline,purge,simplify,dce,uce,osr_path,escape_path,dce,uce,hvn,dce,uce,inline_helpers-,purge,simplify,uce,dce,uce,abce,lower,dce,uce,memopt,reassoc,dce,uce,hvn,dce,uce,gcm,dessa,statprof +-XX:jit.SS_OPT.path.optimizer=ssa,simplify,dce,uce,statprof,devirt_virtual,unguard,devirt_intf,hlo_api_magic,inline,purge,undevirt,simplify,dce,uce,osr_path,escape_path,dce,uce,hvn,dce,uce,inline_helpers-,purge,simplify,uce,dce,uce,abce,lower,dce,uce,memopt,reassoc,dce,uce,hvn,dce,uce,gcm,dessa,statprof -XX:jit.SS_OPT.path.osr_path=gcm,osr,simplify,dce,uce -XX:jit.SS_OPT.path.escape_path=hvn,simplify,dce,uce,escape -XX:jit.SS_OPT.path.abce=classic_abcd,dce,uce,dessa,statprof,peel,ssa,hvn,simplify,dce,uce,memopt,dce,uce,dessa,fastArrayFill,ssa,statprof,dabce,dce,uce Index: vm/jitrino/config/em64t/client.emconf =================================================================== --- vm/jitrino/config/em64t/client.emconf (revision 627700) +++ vm/jitrino/config/em64t/client.emconf (working copy) @@ -55,7 +55,7 @@ -XX:jit.CD_OPT.path=opt_init,translator,optimizer,hir2lir,codegen --XX:jit.CD_OPT.path.optimizer=ssa,devirt,inline,purge,simplify,dce,uce,memopt,simplify,dce,uce,lower,dessa,statprof +-XX:jit.CD_OPT.path.optimizer=ssa,devirt,inline,purge,undevirt,simplify,dce,uce,memopt,simplify,dce,uce,lower,dessa,statprof -XX:jit.CD_OPT.path.codegen=lock_method,bbp,gcpoints,cafl,dce1,i8l-,api_magic,light_jni-,early_prop-,itrace-,native,constraints,dce2,regalloc,spillgen,layout,copy,rce-,stack,break-,iprof-,emitter!,si_insts,gcmap,info,unlock_method -XX:jit.CD_OPT.path.dce1=cg_dce -XX:jit.CD_OPT.path.dce2=cg_dce Index: vm/jitrino/config/em64t/opt.emconf =================================================================== --- vm/jitrino/config/em64t/opt.emconf (revision 627700) +++ vm/jitrino/config/em64t/opt.emconf (working copy) @@ -24,7 +24,7 @@ -XX:jit.CS_OPT.path=opt_init,translator,optimizer,hir2lir,codegen --XX:jit.CS_OPT.path.optimizer=ssa,devirt,hlo_api_magic,inline,purge,simplify,dce,uce,osr_path-,escape_path,dce,uce,memopt,simplify,dce,uce,lower,dessa,statprof +-XX:jit.CS_OPT.path.optimizer=ssa,devirt,hlo_api_magic,inline,purge,undevirt,simplify,dce,uce,osr_path-,escape_path,dce,uce,memopt,simplify,dce,uce,lower,dessa,statprof -XX:jit.CS_OPT.path.osr_path=gcm,osr,simplify,dce,uce -XX:jit.CS_OPT.path.escape_path=hvn,simplify,dce,uce,escape -XX:jit.CS_OPT.path.codegen=lock_method,bbp,gcpoints,cafl,dce1,i8l-,api_magic,light_jni-,early_prop-,itrace-,native,constraints,dce2,regalloc,spillgen,layout,copy,rce-,stack,break-,iprof-,emitter!,si_insts,gcmap,info,unlock_method Index: vm/jitrino/config/em64t/server.emconf =================================================================== --- vm/jitrino/config/em64t/server.emconf (revision 627700) +++ vm/jitrino/config/em64t/server.emconf (working copy) @@ -78,7 +78,7 @@ #enable profiling of all virtual calls -XX:jit.SD1_OPT.arg.optimizer.vp_instrument.profile_abstract=true --XX:jit.SD2_OPT.path.optimizer=ssa,simplify,dce,uce,devirt_virtual,edge_annotate,unguard,devirt_intf,hlo_api_magic,inline,purge,simplify,dce,uce,osr_path,escape_path,dce,uce,hvn,dce,uce,inline_helpers,purge,simplify,uce,dce,uce,abce,lower,dce,uce,memopt,reassoc,dce,uce,hvn,dce,uce,gcm,dessa,statprof +-XX:jit.SD2_OPT.path.optimizer=ssa,simplify,dce,uce,devirt_virtual,edge_annotate,unguard,devirt_intf,hlo_api_magic,inline,purge,undevirt,simplify,dce,uce,osr_path,escape_path,dce,uce,hvn,dce,uce,inline_helpers,purge,simplify,uce,dce,uce,abce,lower,dce,uce,memopt,reassoc,dce,uce,hvn,dce,uce,gcm,dessa,statprof -XX:jit.SD2_OPT.path.osr_path=gcm,osr,simplify,dce,uce -XX:jit.SD2_OPT.path.escape_path=hvn,simplify,dce,uce,escape -XX:jit.SD2_OPT.path.abce=classic_abcd,dce,uce,dessa,statprof,peel,ssa,hvn,simplify,dce,uce,memopt,dce,uce,dessa,fastArrayFill,ssa,statprof,dabce,dce,uce Index: vm/jitrino/config/ia32/server_static.emconf =================================================================== --- vm/jitrino/config/ia32/server_static.emconf (revision 627700) +++ vm/jitrino/config/ia32/server_static.emconf (working copy) @@ -32,7 +32,7 @@ -XX:jit.SS_OPT.path=opt_init,translator,optimizer,hir2lir,codegen --XX:jit.SS_OPT.path.optimizer=ssa,simplify,dce,uce,devirt_virtual,statprof,unguard,devirt_intf,hlo_api_magic,inline,purge,simplify,dce,uce,osr_path,lazyexc,throwopt,escape_path,inline_helpers-,purge,simplify,uce,dce,uce,abce,lower,dce,uce,statprof,unroll,ssa,simplify,dce,uce,memopt,reassoc,dce,uce,hvn,dce,uce,gcm,dessa,statprof +-XX:jit.SS_OPT.path.optimizer=ssa,simplify,dce,uce,devirt_virtual,statprof,unguard,devirt_intf,hlo_api_magic,inline,purge,undevirt,simplify,dce,uce,osr_path,lazyexc,throwopt,escape_path,inline_helpers-,purge,simplify,uce,dce,uce,abce,lower,dce,uce,statprof,unroll,ssa,simplify,dce,uce,memopt,reassoc,dce,uce,hvn,dce,uce,gcm,dessa,statprof -XX:jit.SS_OPT.path.osr_path=gcm,osr,simplify,dce,uce -XX:jit.SS_OPT.path.escape_path=hvn,simplify,dce,uce,escape -XX:jit.SS_OPT.path.abce=classic_abcd,dce,uce,dessa,statprof,peel,ssa,hvn,simplify,dce,uce,memopt,dce,uce,dessa,fastArrayFill,ssa,statprof,dabce,dce,uce Index: vm/jitrino/config/ia32/client.emconf =================================================================== --- vm/jitrino/config/ia32/client.emconf (revision 627700) +++ vm/jitrino/config/ia32/client.emconf (working copy) @@ -53,7 +53,7 @@ -XX:jit.CD_OPT.path=opt_init,lock_method,translator,optimizer,hir2lir,codegen,unlock_method --XX:jit.CD_OPT.path.optimizer=ssa,devirt,hlo_api_magic,inline,purge,simplify,dce,uce,lazyexc,throwopt,memopt,simplify,dce,uce,lower,statprof,unroll,ssa,simplify,dce,uce,dessa,statprof +-XX:jit.CD_OPT.path.optimizer=ssa,devirt,hlo_api_magic,inline,purge,undevirt,simplify,dce,uce,lazyexc,throwopt,memopt,simplify,dce,uce,lower,statprof,unroll,ssa,simplify,dce,uce,dessa,statprof -XX:jit.CD_OPT.path.codegen=bbp,btr,gcpoints,cafl,dce1,i8l,api_magic,light_jni-,early_prop,peephole,itrace-,native,constraints,dce2,regalloc,spillgen,copy,i586,layout,rce+,stack,break-,iprof-,peephole,emitter!,si_insts,gcmap,info -XX:jit.CD_OPT.path.dce1=cg_dce -XX:jit.CD_OPT.path.dce2=cg_dce Index: vm/jitrino/config/ia32/opt.emconf =================================================================== --- vm/jitrino/config/ia32/opt.emconf (revision 627700) +++ vm/jitrino/config/ia32/opt.emconf (working copy) @@ -24,7 +24,7 @@ -XX:jit.CS_OPT.path=opt_init,lock_method,translator,optimizer,hir2lir,codegen,unlock_method --XX:jit.CS_OPT.path.optimizer=ssa,devirt,hlo_api_magic,inline,purge,simplify,dce,uce,osr_path-,lazyexc,throwopt,escape_path,dce,uce,memopt,simplify,dce,uce,lower,statprof,unroll,ssa,simplify,dce,uce,dessa,statprof +-XX:jit.CS_OPT.path.optimizer=ssa,devirt,hlo_api_magic,inline,purge,undevirt,simplify,dce,uce,osr_path-,lazyexc,throwopt,escape_path,dce,uce,memopt,simplify,dce,uce,lower,statprof,unroll,ssa,simplify,dce,uce,dessa,statprof -XX:jit.CS_OPT.path.osr_path=gcm,osr,simplify,dce,uce -XX:jit.CS_OPT.path.escape_path=hvn,simplify,dce,uce,escape -XX:jit.CS_OPT.path.codegen=bbp,btr,gcpoints,cafl,dce1,i8l,api_magic,light_jni-,early_prop,peephole,itrace-,native,constraints,dce2,regalloc,spillgen,copy,i586,layout,rce+,stack,break-,iprof-,peephole,emitter!,si_insts,gcmap,info Index: vm/jitrino/config/ia32/server.emconf =================================================================== --- vm/jitrino/config/ia32/server.emconf (revision 627700) +++ vm/jitrino/config/ia32/server.emconf (working copy) @@ -80,7 +80,7 @@ -XX:jit.SD2_OPT.path=opt_init,translator,optimizer,hir2lir,codegen --XX:jit.SD2_OPT.path.optimizer=ssa,simplify,dce,uce,devirt_virtual,edge_annotate,unguard,devirt_intf,hlo_api_magic,inline,purge,simplify,dce,uce,osr_path,lazyexc,throwopt,escape_path,inline_helpers,purge,simplify,uce,dce,uce,abce,lower,dce,uce,statprof,unroll,ssa,simplify,dce,uce,memopt,reassoc,dce,uce,hvn,dce,uce,gcm,dessa,statprof +-XX:jit.SD2_OPT.path.optimizer=ssa,simplify,dce,uce,devirt_virtual,edge_annotate,unguard,devirt_intf,hlo_api_magic,inline,purge,undevirt,simplify,dce,uce,osr_path,lazyexc,throwopt,escape_path,inline_helpers,purge,simplify,uce,dce,uce,abce,lower,dce,uce,statprof,unroll,ssa,simplify,dce,uce,memopt,reassoc,dce,uce,hvn,dce,uce,gcm,dessa,statprof -XX:jit.SD2_OPT.path.osr_path=gcm,osr,simplify,dce,uce -XX:jit.SD2_OPT.path.escape_path=hvn,simplify,dce,uce,escape -XX:jit.SD2_OPT.path.abce=classic_abcd,dce,uce,dessa,statprof,peel,ssa,hvn,simplify,dce,uce,memopt,dce,uce,dessa,fastArrayFill,ssa,statprof,dabce,dce,uce Index: vm/jitrino/src/optimizer/devirtualizer.h =================================================================== --- vm/jitrino/src/optimizer/devirtualizer.h (revision 627700) +++ vm/jitrino/src/optimizer/devirtualizer.h (working copy) @@ -31,11 +31,16 @@ class Devirtualizer { public: + enum UnguardMode { + DirectCallCold, // Remove guard if the direct call is cold + DirectCallNotInlined // Remove guard if the direct call is not inlined + }; + Devirtualizer(IRManager& irm, SessionAction* sa = NULL); void guardCallsInRegion(IRManager& irm, DominatorTree* tree); - void unguardCallsInRegion(IRManager& irm); + void unguardCallsInRegion(IRManager& irm, UnguardMode mode); static bool isGuardableVirtualCall(Inst* inst, MethodInst*& methodInst, Opnd*& base, Opnd* & tauNullChecked, Opnd*&tauTypesChecked, uint32 &argOffset, bool &isIntfCall); Index: vm/jitrino/src/optimizer/devirtualizer.cpp =================================================================== --- vm/jitrino/src/optimizer/devirtualizer.cpp (revision 627700) +++ vm/jitrino/src/optimizer/devirtualizer.cpp (working copy) @@ -39,14 +39,22 @@ pass.guardCallsInRegion(irm, dominatorTree); } -DEFINE_SESSION_ACTION(GuardRemovalPass, unguard, "Removal of Cold Guards") +DEFINE_SESSION_ACTION(UnguardColdDCPass, unguard, "Removal Guards if the Direct Call is cold") void -GuardRemovalPass::_run(IRManager& irm) { +UnguardColdDCPass::_run(IRManager& irm) { Devirtualizer pass(irm, this); - pass.unguardCallsInRegion(irm); + pass.unguardCallsInRegion(irm, Devirtualizer::UnguardMode::DirectCallCold); } +DEFINE_SESSION_ACTION(UnguardNonInlinedDCPass, undevirt, "Removal of Guards if the Direct Call was not Inlined") + +void +UnguardNonInlinedDCPass::_run(IRManager& irm) { + Devirtualizer pass(irm, this); + pass.unguardCallsInRegion(irm, Devirtualizer::UnguardMode::DirectCallNotInlined); +} + Devirtualizer::Devirtualizer(IRManager& irm, SessionAction* sa) : _hasProfileInfo(irm.getFlowGraph().hasEdgeProfile()), _typeManager(irm.getTypeManager()), @@ -223,8 +231,8 @@ // // Reconnect graph with new nodes. Connect to merge point later. // - regionFG.addEdge(guard, directCallBlock)->setEdgeProb(1.0); - regionFG.addEdge(guard, virtualCallBlock); + regionFG.addEdge(guard, directCallBlock, 1.0); + regionFG.addEdge(guard, virtualCallBlock, 0.0); regionFG.addEdge(directCallBlock, dispatch); regionFG.addEdge(virtualCallBlock, dispatch); directCallBlock->setExecCount(guard->getExecCount()); @@ -524,9 +532,9 @@ } void -Devirtualizer::unguardCallsInRegion(IRManager& regionIRM) { +Devirtualizer::unguardCallsInRegion(IRManager& regionIRM, UnguardMode mode) { ControlFlowGraph ®ionFG = regionIRM.getFlowGraph(); - if(!regionFG.hasEdgeProfile()) { + if(!regionFG.hasEdgeProfile() && (mode == UnguardMode::DirectCallCold)) { if (Log::isEnabled()) { Log::out()<<"No edge profile, skipping unguard pass"<getOpcode() != Op_TauLdVirtFunAddrSlot || callimem->getOpcode() != Op_IndirectMemoryCall) continue; - // - // A guard - fold based on profile results. - // - - bool fold = dCallNode->getExecCount() < (node->getExecCount() * _directCallPercent / 100); - fold = fold || (dCallNode->getExecCount() < (regionFG.getEntryNode()->getExecCount() * _directCallPercientOfEntry / 100)); - - if(fold) { + bool unguard = false; + switch(mode) { + case DirectCallCold: + // + // A guard - fold based on profile results. + // + { + unguard = dCallNode->getExecCount() < (node->getExecCount() * _directCallPercent / 100); + unguard = unguard || (dCallNode->getExecCount() < (regionFG.getEntryNode()->getExecCount() * _directCallPercientOfEntry / 100)); + + } + break; + case DirectCallNotInlined: + { + Inst* labelInst = (Inst*)dCallNode->getFirstInst(); + assert(labelInst && labelInst->isLabel()); + Inst* directCallCandidateInst = (Inst*)labelInst->getNextInst(); + assert(directCallCandidateInst); + if (directCallCandidateInst->asMethodCallInst() + && (directCallCandidateInst->getOpcode() == Op_DirectCall) + && (directCallCandidateInst->getNextInst() == NULL)) { + unguard = true; + } + } + break; + default: + assert(0); + } + if(unguard) { // // A compile time is compared. Later simplification will fold branch appropriately. // if (Log::isEnabled()) { Log::out()<<"Unguarding: instId="<getId()<setSrc(1, src0); branch->setComparisonModifier(Cmp_NE_Un); + + // Restore original profile info on the virtual call + vCallNode->setExecCount(node->getExecCount()); + node->getFalseEdge()->setEdgeProb(1.0); + } - //regionFG.removeEdge(node->getTrueEdge()); - // branch->unlink(); - } } } }