Index: vm/jitrino/src/optimizer/helper_inliner.cpp =================================================================== --- vm/jitrino/src/optimizer/helper_inliner.cpp (revision 566097) +++ vm/jitrino/src/optimizer/helper_inliner.cpp (working copy) @@ -24,6 +24,7 @@ #include "inliner.h" #include "LoopTree.h" #include "Dominator.h" +#include "StlPriorityQueue.h" namespace Jitrino { @@ -35,7 +36,7 @@ #define DECLARE_STANDARD_HELPER_FLAGS(name) \ bool name##_doInlining;\ - int name##_hotnessPercentToInline;\ + uint32 name##_hotnessPercentToInline;\ const char* name##_className;\ const char* name##_methodName;\ const char* name##_signature;\ @@ -114,9 +115,9 @@ class HelperInliner { public: - HelperInliner(HelperInlinerSession* _sessionAction, MemoryManager& tmpMM, CompilationContext* _cc, Inst* _inst) + HelperInliner(HelperInlinerSession* _sessionAction, MemoryManager& tmpMM, CompilationContext* _cc, Inst* _inst, uint32 _hotness) : flags(((HelperInlinerAction*)_sessionAction->getAction())->getFlags()), localMM(tmpMM), - cc(_cc), inst(_inst), session(_sessionAction), method(NULL) + cc(_cc), inst(_inst), session(_sessionAction), method(NULL), hotness(_hotness) { irm = cc->getHIRManager(); instFactory = &irm->getInstFactory(); @@ -128,6 +129,9 @@ virtual ~HelperInliner(){}; virtual void run()=0; + + uint32 hotness; + protected: MethodDesc* ensureClassIsResolvedAndInitialized(const char* className, const char* methodName, const char* signature); virtual void doInline() = 0; @@ -147,13 +151,19 @@ OpndManager* opndManager; TypeManager* typeManager; ControlFlowGraph* cfg; +}; +class HelperInlinerCompare { +public: + bool operator()(const HelperInliner* hi1, const HelperInliner* hi2) { return hi1->hotness < hi2->hotness; } }; + + #define DECLARE_HELPER_INLINER(name, flagPrefix)\ class name : public HelperInliner {\ public:\ - name (HelperInlinerSession* session, MemoryManager& tmpMM, CompilationContext* cc, Inst* inst)\ - : HelperInliner(session, tmpMM, cc, inst){}\ + name (HelperInlinerSession* session, MemoryManager& tmpMM, CompilationContext* cc, Inst* inst, uint32 hotness)\ + : HelperInliner(session, tmpMM, cc, inst, hotness){}\ \ virtual void run() { \ if (Log::isEnabled()) {\ @@ -188,53 +198,57 @@ //finding all helper calls ControlFlowGraph& fg = irm.getFlowGraph(); double entryExecCount = fg.hasEdgeProfile() ? fg.getEntryNode()->getExecCount(): 1; - StlVector helperInliners(tmpMM); + uint32 maxNodeCount = irm.getOptimizerFlags().hir_node_threshold; + StlPriorityQueue, HelperInlinerCompare> *helperInlineCandidates = + new (tmpMM) StlPriorityQueue, HelperInlinerCompare>(tmpMM); const Nodes& nodes = fg.getNodesPostOrder();//process checking only reachable nodes. for (Nodes::const_iterator it = nodes.begin(), end = nodes.end(); it!=end; ++it) { Node* node = *it; - int nodePercent = fg.hasEdgeProfile() ? (int)(node->getExecCount()*100/entryExecCount) : 0; + double execCount = node->getExecCount(); + assert (execCount >= 0); + uint32 nodePercent = fg.hasEdgeProfile() ? (uint32)(execCount*100/entryExecCount) : 0; if (node->isBlockNode()) { for (Inst* inst = (Inst*)node->getFirstInst(); inst!=NULL; inst = inst->getNextInst()) { Opcode opcode = inst->getOpcode(); switch(opcode) { case Op_NewObj: if (flags.newObj_doInlining && nodePercent >= flags.newObj_hotnessPercentToInline) { - helperInliners.push_back(new (tmpMM) NewObjHelperInliner(this, tmpMM, cc, inst)); + helperInlineCandidates->push(new (tmpMM) NewObjHelperInliner(this, tmpMM, cc, inst, nodePercent)); } break; case Op_NewArray: if (flags.newArray_doInlining && nodePercent >= flags.newArray_hotnessPercentToInline) { - helperInliners.push_back(new (tmpMM) NewArrayHelperInliner(this, tmpMM, cc, inst)); + helperInlineCandidates->push(new (tmpMM) NewArrayHelperInliner(this, tmpMM, cc, inst, nodePercent)); } break; case Op_TauMonitorEnter: if (flags.objMonEnter_doInlining && nodePercent >= flags.objMonEnter_hotnessPercentToInline) { - helperInliners.push_back(new (tmpMM) ObjMonitorEnterHelperInliner(this, tmpMM, cc, inst)); + helperInlineCandidates->push(new (tmpMM) ObjMonitorEnterHelperInliner(this, tmpMM, cc, inst, nodePercent)); } break; case Op_TauMonitorExit: if (flags.objMonExit_doInlining && nodePercent >= flags.objMonExit_hotnessPercentToInline) { - helperInliners.push_back(new (tmpMM) ObjMonitorExitHelperInliner(this, tmpMM, cc, inst)); + helperInlineCandidates->push(new (tmpMM) ObjMonitorExitHelperInliner(this, tmpMM, cc, inst, nodePercent)); } break; case Op_TauStRef: if (flags.wb_doInlining && nodePercent >= flags.wb_hotnessPercentToInline) { - helperInliners.push_back(new (tmpMM) WriteBarrierHelperInliner(this, tmpMM, cc, inst)); + helperInlineCandidates->push(new (tmpMM) WriteBarrierHelperInliner(this, tmpMM, cc, inst, nodePercent)); } break; case Op_TauLdIntfcVTableAddr: if (flags.ldInterface_doInlining && nodePercent >= flags.ldInterface_hotnessPercentToInline) { - helperInliners.push_back(new (tmpMM) LdInterfaceHelperInliner(this, tmpMM, cc, inst)); + helperInlineCandidates->push(new (tmpMM) LdInterfaceHelperInliner(this, tmpMM, cc, inst, nodePercent)); } break; case Op_TauCheckCast: if (flags.checkCast_doInlining && nodePercent >= flags.checkCast_hotnessPercentToInline) { - helperInliners.push_back(new (tmpMM) CheckCastHelperInliner(this, tmpMM, cc, inst)); + helperInlineCandidates->push(new (tmpMM) CheckCastHelperInliner(this, tmpMM, cc, inst, nodePercent)); } break; case Op_TauInstanceOf: if (flags.instanceOf_doInlining && nodePercent >= flags.instanceOf_hotnessPercentToInline) { - helperInliners.push_back(new (tmpMM) InstanceOfHelperInliner(this, tmpMM, cc, inst)); + helperInlineCandidates->push(new (tmpMM) InstanceOfHelperInliner(this, tmpMM, cc, inst, nodePercent)); } break; default: break; @@ -244,10 +258,10 @@ } //running all inliners - //TODO: set inline limit! - for (StlVector::const_iterator it = helperInliners.begin(), end = helperInliners.end(); it!=end; ++it) { - HelperInliner* inliner = *it; + while(!helperInlineCandidates->empty() && (fg.getNodeCount() < maxNodeCount)) { + HelperInliner* inliner = helperInlineCandidates->top(); inliner->run(); + helperInlineCandidates->pop(); } } Index: vm/jitrino/src/optimizer/inliner.h =================================================================== --- vm/jitrino/src/optimizer/inliner.h (revision 566097) +++ vm/jitrino/src/optimizer/inliner.h (working copy) @@ -61,19 +61,22 @@ class InlineNode : public TreeNode { public: - InlineNode(IRManager& irm, Inst *callInst, Node *callNode) : _irm(irm), _callInst(callInst), _callNode(callNode) {} + InlineNode(IRManager& irm, Inst *callInst, Node *callNode, bool forced = false) + : _irm(irm), _callInst(callInst), _callNode(callNode), _forceInline(forced) {} InlineNode* getChild() {return (InlineNode*) child;} InlineNode* getSiblings() {return (InlineNode*) siblings;} InlineNode* getParent() {return (InlineNode*) parent;} IRManager& getIRManager() { return _irm; } Inst* getCallInst() { return _callInst; } - Node* getCallNode() { return _callNode; } + Node* getCallNode() { return _callNode; } + bool isForced() { return _forceInline; } void print(::std::ostream& os); void printTag(::std::ostream& os); private: IRManager& _irm; Inst* _callInst; - Node* _callNode; + Node* _callNode; + bool _forceInline; }; class InlineTree : public Tree { @@ -193,7 +196,9 @@ int32 _inlineRecursionPenalty; int32 _inlineExactArgBonus; int32 _inlineExactAllBonus; - + + uint32 _inlineMaxNodeThreshold; + bool _inlineSkipExceptionPath; bool _inlineSkipApiMagicMethods; Method_Table* _inlineSkipMethodTable; Index: vm/jitrino/src/optimizer/optimizer.h =================================================================== --- vm/jitrino/src/optimizer/optimizer.h (revision 566097) +++ vm/jitrino/src/optimizer/optimizer.h (working copy) @@ -39,6 +39,11 @@ //global optimizer flags bool dumpdot; + // Max number of nodes HIR would grow up to during various inlinings + uint32 hir_node_threshold; + // A share of node limit inlining may use (in percents), the remaining share is for helper inlining + uint32 inline_node_quota; + bool cse_final; uint32 hash_init_factor; Index: vm/jitrino/src/optimizer/inliner.cpp =================================================================== --- vm/jitrino/src/optimizer/inliner.cpp (revision 566097) +++ vm/jitrino/src/optimizer/inliner.cpp (working copy) @@ -109,6 +109,8 @@ _inlineExactArgBonus = argSource->getIntArg("exact_single_parameter_bonus", INLINE_EXACT_ARG_BONUS); _inlineExactAllBonus = argSource->getIntArg("exact_all_parameter_bonus", INLINE_EXACT_ALL_BONUS); + _inlineMaxNodeThreshold = irm.getOptimizerFlags().hir_node_threshold * irm.getOptimizerFlags().inline_node_quota / 100; + _inlineSkipExceptionPath = argSource->getBoolArg("skip_exception_path", INLINE_SKIP_EXCEPTION_PATH); #if defined (_EM64T_) || defined (_IPF_) _inlineSkipApiMagicMethods = false; @@ -158,7 +160,7 @@ if (inlinePragma!=NULL && methodDesc.hasAnnotation(inlinePragma)) { //methods marked with inline pragma processed separately and are always inlined //regardless of it benefits and size limitations. - return -1; + assert(0); } if (_inlineBonusMethodTable!=NULL && _inlineBonusMethodTable->accept_this_method(methodDesc)) { benefit+=1000; @@ -897,7 +899,7 @@ call = ((Inst*)callNode->getLastInst())->asMethodCallInst(); assert(call != NULL); methodDesc = call->getMethodDesc(); - bool isPragmaInline = benefit == PRAGMA_INLINE_BENEFIT; + bool isPragmaInline = methodDesc->hasAnnotation(inlinePragma);; // If candidate would cause top level method to exceed size threshold, throw away. @@ -941,7 +943,8 @@ MethodDesc *methodDesc = call->getMethodDesc(); IRManager* inlinedIRM = new (_tmpMM) IRManager(_tmpMM, _toplevelIRM, *methodDesc, NULL); // Augment inline tree - InlineNode *inlineNode = new (_tmpMM) InlineNode(*inlinedIRM, call, call->getNode()); + bool forceInline = methodDesc->hasAnnotation(inlinePragma); + InlineNode *inlineNode = new (_tmpMM) InlineNode(*inlinedIRM, call, call->getNode(), forceInline); inlineCC.setHIRManager(inlinedIRM); @@ -990,6 +993,7 @@ assert(size > 0); Log::out() << "Inline benefit " << methodDesc->getParentType()->getName() << "." << methodDesc->getName() << " == " << (int) benefit << ::std::endl; if(0 < size && benefit > _minBenefitThreshold) { + assert(benefit < PRAGMA_INLINE_BENEFIT); // Inline candidate Log::out() << "Add to queue" << std::endl; _inlineCandidates.push(CallSite(benefit, node, inlineNode)); @@ -1152,6 +1156,10 @@ } //inline current region inlineRegion(regionNode); + // Limit inlining by node count. All @Inline methods still must be inlined. + if (!regionNode->isForced() && _toplevelIRM.getFlowGraph().getNodeCount() > _inlineMaxNodeThreshold) { + break; + } } while (true); } Index: vm/jitrino/src/optimizer/optimizer.cpp =================================================================== --- vm/jitrino/src/optimizer/optimizer.cpp (revision 566097) +++ vm/jitrino/src/optimizer/optimizer.cpp (working copy) @@ -63,6 +63,9 @@ namespace Jitrino { +#define HIR_NODE_THRESHOLD 1300 +#define INLINE_NODE_QUOTA 60 // percents + class OptInitAction : public Action { public: void init() {readFlags();} @@ -105,6 +108,8 @@ memset( &optimizerFlags, 0, sizeof(OptimizerFlags)); optimizerFlags.dumpdot= getBoolArg("dumpdot", false); + optimizerFlags.hir_node_threshold = getIntArg("hir_node_threshold", HIR_NODE_THRESHOLD); + optimizerFlags.inline_node_quota = getIntArg("inline_node_quota", INLINE_NODE_QUOTA); optimizerFlags.cse_final = getBoolArg("cse_final", true); @@ -196,6 +201,7 @@ void showFlags(std::ostream& os) { os << "\n"< - max number of CFG nodes for inlining" << std::endl; os << " elim_cmp3[={ON|off}] - eliminate cmp3 tests" << std::endl; os << " elim_checks[={ON|off}] - try to eliminate some checks using branch conditions" << std::endl; os << " use_mulhi{ON|off}] - use MulHi opcode" << std::endl; Index: vm/jitrino/src/translator/java/JavaLabelPrepass.cpp =================================================================== --- vm/jitrino/src/translator/java/JavaLabelPrepass.cpp (revision 566097) +++ vm/jitrino/src/translator/java/JavaLabelPrepass.cpp (working copy) @@ -1950,7 +1950,7 @@ << "][index=" << index << "] is empty" << ::std::endl; } else { Log::out() << "localVars[offset=" << offset << "][index=" << index << "].var->declT = "; - (*iter).second->getDeclaredType()->print(::std::cout); + (*iter).second->getDeclaredType()->print(Log::out()); Log::out() << ::std::endl; } }