Index: jitrino/src/codegenerator/CodeGenIntfc.h =================================================================== --- jitrino/src/codegenerator/CodeGenIntfc.h (revision 582116) +++ jitrino/src/codegenerator/CodeGenIntfc.h (working copy) @@ -152,15 +152,6 @@ }; }; -class IntrinsicCallOp { -public: - enum Id { - CharArrayCopy, - ArrayCopyDirect, - ArrayCopyReverse - }; -}; - class JitHelperCallOp { public: enum Id { @@ -169,7 +160,10 @@ SaveThisState, ReadThisState, LockedCompareAndExchange, - AddValueProfileValue + AddValueProfileValue, + ArrayCopyDirect, + ArrayCopyReverse, + ArrayCopyFast }; }; @@ -258,8 +252,6 @@ NamedType* vtableType) = 0; virtual CG_OpndHandle* call(uint32 numArgs, CG_OpndHandle** args, Type* retType, MethodDesc *desc) = 0; - virtual CG_OpndHandle* arraycopyReverse(uint32 numArgs, CG_OpndHandle** args) = 0; - virtual CG_OpndHandle* arraycopy(uint32 numArgs, CG_OpndHandle** args) = 0; virtual CG_OpndHandle* tau_call(uint32 numArgs, CG_OpndHandle** args, Type* retType, MethodDesc *desc, CG_OpndHandle *tauNullChecked, @@ -272,10 +264,6 @@ CG_OpndHandle* methodPtr, CG_OpndHandle* tauNullChecked, CG_OpndHandle* tauTypesChecked) = 0; - virtual CG_OpndHandle* tau_callintr(uint32 numArgs, CG_OpndHandle** args, Type* retType, - IntrinsicCallOp::Id callId, - CG_OpndHandle* tauNullChecked, - CG_OpndHandle* tauTypesChecked) = 0; virtual CG_OpndHandle* callhelper(uint32 numArgs, CG_OpndHandle** args, Type* retType, JitHelperCallOp::Id callId) = 0; virtual CG_OpndHandle* callvmhelper(uint32 numArgs, CG_OpndHandle** args, Type* retType, Index: jitrino/src/codegenerator/ia32/Ia32Encoder.cpp =================================================================== --- jitrino/src/codegenerator/ia32/Ia32Encoder.cpp (revision 582116) +++ jitrino/src/codegenerator/ia32/Ia32Encoder.cpp (working copy) @@ -177,8 +177,7 @@ } } for (uint32 i = 0, n = fi.opndCount; i < n; i++) { - uint32 idx = fi.isExtended ? og->extendedToNativeMap[i] : i; - Constraint co=fi.opndConstraints[idx]; + Constraint co=fi.opndConstraints[i]; if (any) { co = Constraint(OpndKind_Any, co.getSize()); } @@ -483,4 +482,4 @@ og.properties |= Inst::Properties_MemoryOpndConditional; } -}}; //namespace Ia32 +}}; //namespace Ia32 \ No newline at end of file Index: jitrino/src/codegenerator/ia32/Ia32GCSafePoints.cpp =================================================================== --- jitrino/src/codegenerator/ia32/Ia32GCSafePoints.cpp (revision 582116) +++ jitrino/src/codegenerator/ia32/Ia32GCSafePoints.cpp (working copy) @@ -532,9 +532,9 @@ fromOpnd = inst->getOpnd(useIndex1); break; case Mnemonic_ADD: { -#ifdef _EM64T_ +//#ifdef _EM64T_ case Mnemonic_SUB: -#endif +//#endif fromOpnd = opnd; Opnd* offsetOpnd = inst->getOpnd(useIndex1); Opnd* immOffset = findImmediateSource(offsetOpnd); @@ -932,3 +932,6 @@ }} //namespace + + + Index: jitrino/src/codegenerator/ia32/Ia32I8Lowerer.cpp =================================================================== --- jitrino/src/codegenerator/ia32/Ia32I8Lowerer.cpp (revision 582116) +++ jitrino/src/codegenerator/ia32/Ia32I8Lowerer.cpp (working copy) @@ -528,9 +528,9 @@ // to regalloc and constraint resolver. // However, this seems does not change anything currently, // so leaving as-is. - //test low, low - //setns hi ; if lo is positive, then load 1 into hi - //sub hi, 1 ; if lo is positive, then hi is now '0'. otherwise, it's -1 + //test low, low + //setns hi ; if lo is positive, then load 1 into hi + //sub hi, 1 ; if lo is positive, then hi is now '0'. otherwise, it's -1 irManager->newInstEx(Mnemonic_CDQ, 1, dst_2, dst_1)->insertBefore(inst); } else { //fill upper word with 0 @@ -1521,3 +1521,8 @@ }} + + + + + Index: jitrino/src/codegenerator/ia32/Ia32Inst.h =================================================================== --- jitrino/src/codegenerator/ia32/Ia32Inst.h (revision 582116) +++ jitrino/src/codegenerator/ia32/Ia32Inst.h (working copy) @@ -715,6 +715,9 @@ m==Mnemonic_MOVS32 || m==Mnemonic_STD || m==Mnemonic_CLD || +#ifdef _HAVE_MMX_ + m==Mnemonic_EMMS || +#endif m==Mnemonic_POPFD || m==Mnemonic_PUSHFD || m==Mnemonic_POP || Index: jitrino/src/codegenerator/ia32/Ia32InstCodeSelector.cpp =================================================================== --- jitrino/src/codegenerator/ia32/Ia32InstCodeSelector.cpp (revision 582116) +++ jitrino/src/codegenerator/ia32/Ia32InstCodeSelector.cpp (working copy) @@ -156,6 +156,32 @@ return; } +void __stdcall memory_copy_direct(uint32 op1, uint32 op2, uint32 op3, uint32 op4, uint32 op5) { + + Jitrino::crash("Illegal internal helper was called.\n Please enable FastMemoryCopy optimization in the code generator path"); + assert(0); + + return; +} + +void __stdcall memory_copy_reverse(uint32 op1, uint32 op2, uint32 op3, uint32 op4, uint32 op5) { + + Jitrino::crash("Illegal internal helper was called.\n Please enable FastMemoryCopy optimization in the code generator path"); + assert(0); + + return; +} + +void __stdcall memory_copy_fast(uint32 op1, uint32 op2, uint32 op3, uint32 op4, uint32 op5) { + + Jitrino::crash("Illegal internal helper was called.\n Please enable FastMemoryCopy optimization in the code generator path"); + assert(0); + + return; +} + + + //_______________________________________________________________________________________________________________ uint32 InstCodeSelector::_tauUnsafe; @@ -178,6 +204,9 @@ irManager.registerInternalHelperInfo("add_value_profile_value", IRManager::InternalHelperInfo((void*)&add_value_profile_value,&CallingConvention_STDCALL)); irManager.registerInternalHelperInfo("fill_array_with_const", IRManager::InternalHelperInfo((void*)&fill_array_with_const,&CallingConvention_STDCALL)); + irManager.registerInternalHelperInfo("memory_copy_direct", IRManager::InternalHelperInfo((void*)&memory_copy_direct,&CallingConvention_STDCALL)); + irManager.registerInternalHelperInfo("memory_copy_reverse", IRManager::InternalHelperInfo((void*)&memory_copy_reverse,&CallingConvention_STDCALL)); + irManager.registerInternalHelperInfo("memory_copy_fast", IRManager::InternalHelperInfo((void*)&memory_copy_fast,&CallingConvention_STDCALL)); } //_______________________________________________________________________________________________________________ @@ -472,7 +501,7 @@ CG_OpndHandle* InstCodeSelector::convToInt(ConvertToIntOp::Types opType, bool isSigned, - bool isZeroExtend, + bool isZeroExtend, ConvertToIntOp::OverflowMod ovfMod, Type* dstType, CG_OpndHandle* src) @@ -486,16 +515,16 @@ sizeType=isSigned?typeManager.getInt16Type():typeManager.getUInt16Type(); break; -#ifdef _IA32_ +#ifdef _IA32_ case ConvertToIntOp::I: -#endif +#endif case ConvertToIntOp::I4: sizeType=isSigned?typeManager.getInt32Type():typeManager.getUInt32Type(); break; #ifdef _EM64T_ case ConvertToIntOp::I: -#endif +#endif case ConvertToIntOp::I8: sizeType=isSigned?typeManager.getInt64Type():typeManager.getUInt64Type(); break; @@ -1798,7 +1827,7 @@ uint32 elemSize = 0; if (elemType->isReference() - && Type::isCompressedReference(elemType->tag, compilationInterface) + && Type::isCompressedReference(elemType->tag, compilationInterface) && !elemType->isCompressedReference()) { elemSize = 4; } else { @@ -2440,90 +2469,6 @@ } //_______________________________________________________________________________________________________________ -// reverse copying with 'rep move' instruction -// start indexes (args[1] and args[3] must be prepared respectively) - -CG_OpndHandle* InstCodeSelector::arraycopyReverse(uint32 numArgs, - CG_OpndHandle** args) -{ - - - appendInsts(irManager.newInst(Mnemonic_PUSHFD)); - appendInsts(irManager.newInst(Mnemonic_STD)); - - arraycopy(numArgs,args); - - appendInsts(irManager.newInst(Mnemonic_POPFD)); - - return NULL; -} - -//_______________________________________________________________________________________________________________ -// Transforming System::arraycopy call into 'rep move' - -CG_OpndHandle* InstCodeSelector::arraycopy(uint32 numArgs, - CG_OpndHandle** args) -{ - assert(numArgs == 5); - -#ifdef _EM64T_ - RegName counterRegName = RegName_RCX; - RegName srcAddrRegName = RegName_RSI; - RegName dstAddrRegName = RegName_RDI; -#else - RegName counterRegName = RegName_ECX; - RegName srcAddrRegName = RegName_ESI; - RegName dstAddrRegName = RegName_EDI; -#endif - - // prepare counter - Type* counterType = typeManager.getIntPtrType(); - Opnd* counter = irManager.newRegOpnd(counterType,counterRegName); - copyOpnd(counter,(Opnd*)args[4]); - - // prepare src position - Opnd* srcAddr = (Opnd*)addElemIndexWithLEA(NULL,args[0],args[1]); - Opnd* srcAddrReg = irManager.newRegOpnd(srcAddr->getType(),srcAddrRegName); - copyOpnd(srcAddrReg,srcAddr); - - // prepare dst position - Opnd* dstAddr = (Opnd*)addElemIndexWithLEA(NULL,args[2],args[3]); - Opnd* dstAddrReg = irManager.newRegOpnd(dstAddr->getType(),dstAddrRegName); - copyOpnd(dstAddrReg,dstAddr); - - // double counter if elem type is 64 bits long - PtrType* srcAddrType = srcAddr->getType()->asPtrType(); - assert(srcAddrType); - Type::Tag tag = srcAddrType->getPointedToType()->tag; - Mnemonic mn = Mnemonic_NULL; - OpndSize typeSize = IRManager::getTypeSize(tag); - switch(typeSize) { - case OpndSize_8: mn = Mnemonic_MOVS8; break; - case OpndSize_16: mn = Mnemonic_MOVS16; break; - case OpndSize_32: mn = Mnemonic_MOVS32; break; - case OpndSize_64: - { - /** - * FIXME - * Currently JIT erroneously supposes that compressed mode is always on. - * So if type is object, it is actually compressed (32-bit sized). - * But IRManager::getTypeSize() "correctly" returns OpndSize_64. - */ - if (!srcAddrType->getPointedToType()->isObject()) { - appendInsts(irManager.newInst(Mnemonic_SHL, counter, irManager.newImmOpnd(counterType, (int32)1))); - } - mn = Mnemonic_MOVS32; - } - break; - default: assert(0); mn = Mnemonic_MOVS32; break; - } - Inst* copyInst = irManager.newInst(mn,dstAddrReg,srcAddrReg,counter); - copyInst->setPrefix(InstPrefix_REP); - appendInsts(copyInst); - return NULL; -} - -//_______________________________________________________________________________________________________________ // Direct call to the method. Depending on the code generator flags we // either expand direct call into loading method address into a register and // indirect call or true direct call that will be patched if method is recompiled. @@ -2574,32 +2519,6 @@ } //_______________________________________________________________________________________________________________ -// Intrinsic call - -CG_OpndHandle* InstCodeSelector::callintr(uint32 numArgs, - CG_OpndHandle** args, - Type* retType, - IntrinsicCallOp::Id callId) -{ - ICS_ASSERT(0); - return 0; -} - -//_______________________________________________________________________________________________________________ -// Intrinsic call - -CG_OpndHandle* InstCodeSelector::tau_callintr(uint32 numArgs, - CG_OpndHandle** args, - Type* retType, - IntrinsicCallOp::Id callId, - CG_OpndHandle* tauNullsChecked, - CG_OpndHandle* tauTypesChecked) -{ - ICS_ASSERT(0); - return 0; -} - -//_______________________________________________________________________________________________________________ // JIT helper call CG_OpndHandle* InstCodeSelector::callhelper(uint32 numArgs, @@ -2688,6 +2607,28 @@ appendInsts(irManager.newInternalRuntimeHelperCallInst("fill_array_with_const", numArgs, newArgs, dstOpnd)); break; } + case ArrayCopyDirect: + { + assert(numArgs == 5); + Opnd * newArgs[5] = {(Opnd *)args[0], (Opnd *)args[1], (Opnd *)args[2], (Opnd *)args[3], (Opnd *)args[4]}; + appendInsts(irManager.newInternalRuntimeHelperCallInst("memory_copy_direct", numArgs, newArgs, dstOpnd)); + break; + } + case ArrayCopyReverse: + { + assert(numArgs == 5); + Opnd * newArgs[5] = {(Opnd *)args[0], (Opnd *)args[1], (Opnd *)args[2], (Opnd *)args[3], (Opnd *)args[4]}; + appendInsts(irManager.newInternalRuntimeHelperCallInst("memory_copy_reverse", numArgs, newArgs, dstOpnd)); + break; + } + case ArrayCopyFast: + { + assert(numArgs == 5); + Opnd * newArgs[5] = {(Opnd *)args[0], (Opnd *)args[1], (Opnd *)args[2], (Opnd *)args[3], (Opnd *)args[4]}; + appendInsts(irManager.newInternalRuntimeHelperCallInst("memory_copy_fast", numArgs, newArgs, dstOpnd)); + break; + } + default: { assert(0); Index: jitrino/src/codegenerator/ia32/Ia32InstCodeSelector.h =================================================================== --- jitrino/src/codegenerator/ia32/Ia32InstCodeSelector.h (revision 582116) +++ jitrino/src/codegenerator/ia32/Ia32InstCodeSelector.h (working copy) @@ -211,14 +211,10 @@ CG_OpndHandle* methodPtr, CG_OpndHandle* nonNullFirstArgTau, CG_OpndHandle* tauTypesChecked); CG_OpndHandle* call(uint32 numArgs, CG_OpndHandle** args, Type* retType, MethodDesc *desc); - CG_OpndHandle* arraycopyReverse(uint32 numArgs, CG_OpndHandle** args); - CG_OpndHandle* arraycopy(uint32 numArgs, CG_OpndHandle** args); CG_OpndHandle* tau_call(uint32 numArgs, CG_OpndHandle** args, Type* retType, MethodDesc *desc, CG_OpndHandle *nonNullFirstArgTau, CG_OpndHandle *tauTypesChecked); CG_OpndHandle* tau_callvirt(uint32 numArgs,CG_OpndHandle** args, Type* retType, MethodDesc *desc, CG_OpndHandle* tauNullChecked, CG_OpndHandle* tauTypesChecked); - CG_OpndHandle* callintr(uint32 numArgs, CG_OpndHandle** args, Type* retType,IntrinsicCallOp::Id callId); - CG_OpndHandle* tau_callintr(uint32 numArgs, CG_OpndHandle** args, Type* retType,IntrinsicCallOp::Id callId, CG_OpndHandle *tauNullsChecked, CG_OpndHandle *tauTypesChecked); CG_OpndHandle* callhelper(uint32 numArgs, CG_OpndHandle** args, Type* retType,JitHelperCallOp::Id callId); CG_OpndHandle* callvmhelper(uint32 numArgs, CG_OpndHandle** args, Type* retType, CompilationInterface::RuntimeHelperId callId); @@ -321,6 +317,7 @@ bool isIntegerType(Type * type) { return type->isInteger()||type->isBoolean()||type->isChar(); } + void copyOpnd(Opnd *dst, Opnd *src); void copyOpndTrivialOrTruncatingConversion(Opnd *dst, Opnd *src); Index: jitrino/src/codegenerator/ia32/IA32MemoryCopy.cpp =================================================================== --- jitrino/src/codegenerator/ia32/IA32MemoryCopy.cpp (revision 0) +++ jitrino/src/codegenerator/ia32/IA32MemoryCopy.cpp (revision 0) @@ -0,0 +1,363 @@ +#include "Ia32IRManager.h" + + +namespace Jitrino +{ +namespace Ia32 { + +class MemoryCopyOptimizer +{ +public: + MemoryCopyOptimizer(IRManager* irmanager, MemoryManager& mm); + void OptimizeMemoryCopy(); +private: + void genFastMemoryCopy(uint32 numArgs, Opnd** args, Node* currNode, Node* exitNode); + void genDirectMemoryCopy(uint32 numArgs, Opnd** args, Node* currNode); + void genReverseMemoryCopy(uint32 numArgs, Opnd** args, Node* currNode); + Opnd* genLoadIndexWithLEA(Opnd* array, Opnd* index, RegName addrReg, Node* currNode); + + typedef StlList Insts; + + IRManager* irm; + MemoryManager& mcmm; + TypeManager* typeManager; + ControlFlowGraph* cfg; + + Insts* exam2Insts; +}; + +MemoryCopyOptimizer::MemoryCopyOptimizer(IRManager* irmanager, MemoryManager& mm) +: mcmm(mm), irm(irmanager) +{ + typeManager = &irmanager->getTypeManager(); + cfg = irmanager->getFlowGraph(); + exam2Insts = new (mcmm) Insts(mcmm); +} + +//_______________________________________________________________________________________________________________ +// Compute address of the array element given +// address of the first element and index +// using 'LEA' instruction + +Opnd* MemoryCopyOptimizer::genLoadIndexWithLEA(Opnd* array, Opnd* index, RegName addrReg, Node* currNode) +{ + ArrayType * arrayType=array->getType()->asArrayType(); + Type * elemType=arrayType->getElementType(); + Type * dstType=irm->getManagedPtrType(elemType); + +#ifdef _EM64T_ + Type * indexType = typeManager->getInt64Type(); + Type * offType = typeManager->getInt64Type(); +#else + Type * indexType = typeManager->getInt32Type(); + Type * offType = typeManager->getInt32Type(); +#endif + + uint32 elemSize = 0; + if (elemType->isReference() + && Type::isCompressedReference(elemType->tag, irm->getCompilationInterface()) + && !elemType->isCompressedReference()) { + elemSize = 4; + } else { + elemSize = getByteSize(irm->getTypeSize(elemType)); + } + Opnd * elemSizeOpnd = irm->newImmOpnd(indexType, elemSize); + + Opnd * indexOpnd = index; + + if ( indexOpnd->isPlacedIn(OpndKind_Imm) ) { + // we need to put index operand on a register to satisfy LEA constraint + int64 immValue = indexOpnd->getImmValue(); + if (immValue == 0) { + indexOpnd = NULL; + elemSizeOpnd = NULL; + } else { + Constraint indexRegConstr(OpndKind_GPReg, irm->getTypeSize(indexType)); + indexOpnd = irm->newOpnd(indexType, indexRegConstr); + currNode->appendInst(irm->newInstEx(Mnemonic_MOV, 1, indexOpnd, index)); + } + } + + Opnd * addr = irm->newMemOpnd(dstType, array, indexOpnd, elemSizeOpnd, + irm->newImmOpnd(offType, arrayType->getArrayElemOffset()) + ); + + Opnd * dst = irm->newRegOpnd(dstType, addrReg); + + currNode->appendInst(irm->newInstEx(Mnemonic_LEA, 1, dst, addr)); + return dst; +} + +void MemoryCopyOptimizer::genFastMemoryCopy(uint32 numArgs, Opnd** args, Node* currNode, Node* exitNode) +{ +#ifdef _EM64T_ + RegName counterRegName = RegName_RCX; + RegName srcAddrRegName = RegName_RSI; + RegName dstAddrRegName = RegName_RDI; +#else + RegName counterRegName = RegName_ECX; + RegName srcAddrRegName = RegName_ESI; + RegName dstAddrRegName = RegName_EDI; +#endif + +#ifdef _EM64T_ + Constraint reg64Constr(OpndKind_GPReg, OpndSize_64); +#endif + Constraint reg32Constr(OpndKind_GPReg, OpndSize_32); + Constraint reg16Constr(OpndKind_GPReg, OpndSize_16); + Constraint reg8Constr(OpndKind_GPReg, OpndSize_8); + + Type* int32Type = typeManager->getInt32Type(); + Type* int16Type = typeManager->getInt16Type(); + Type* int8Type = typeManager->getInt8Type(); + + // prepare counter + Type* counterType = typeManager->getIntPtrType(); + +#ifdef _EM64T_ + Opnd* counter = irm->newOpnd(counterType, reg64Constr); +#else + Opnd* counter = irm->newOpnd(counterType, reg32Constr); +#endif + + currNode->appendInst(irm->newCopyPseudoInst(Mnemonic_MOV, counter, args[4])); + + // prepare src position + Opnd* srcAddr = genLoadIndexWithLEA(args[0],args[1], srcAddrRegName, currNode); + + // prepare dst position + Opnd* dstAddr = genLoadIndexWithLEA(args[2],args[3], dstAddrRegName, currNode); + + Opnd* one8bit = irm->newImmOpnd(int8Type, (int32)1); + Opnd* one32bit = irm->newImmOpnd(int32Type, (int32)1); + Opnd* two8bit = irm->newImmOpnd(int8Type, (int32)2); + Opnd* two32bit = irm->newImmOpnd(int32Type, (int32)2); + Opnd* three8bit = irm->newImmOpnd(int8Type, (int32)3); + Opnd* three32bit = irm->newImmOpnd(int32Type, (int32)3); + Opnd* four8bit = irm->newImmOpnd(int8Type, (int32)4); + Opnd* four32bit = irm->newImmOpnd(int32Type, (int32)4); + + PtrType* srcAddrType = srcAddr->getType()->asPtrType(); + Type::Tag tag = srcAddrType->getPointedToType()->tag; + OpndSize typeSize = IRManager::getTypeSize(tag); + switch(typeSize) { + case OpndSize_8: + break; + case OpndSize_16: + currNode->appendInst(irm->newInstEx(Mnemonic_SHL, 1, counter, counter, one8bit)); // shl ecx, 1 + break; + case OpndSize_32: + currNode->appendInst(irm->newInstEx(Mnemonic_SHL, 1, counter, counter, two8bit)); // shl ecx, 2 + break; + case OpndSize_64: + { + /** + * FIXME + * Currently JIT erroneously supposes that compressed mode is always on. + * So if type is object, it is actually compressed (32-bit sized). + * But IRManager::getTypeSize() "correctly" returns OpndSize_64. + */ + if (!srcAddrType->getPointedToType()->isObject()) { + currNode->appendInst(irm->newInstEx(Mnemonic_SHL, 1, counter, counter, three8bit)); // shl ecx, 3 + } + else + currNode->appendInst(irm->newInstEx(Mnemonic_SHL, 1, counter, counter, two8bit)); // shl ecx, 2 + } + break; + default: assert(0); + break; + } + Opnd* edx = irm->newOpnd(int32Type, reg32Constr); + Opnd* ebx = irm->newOpnd(int32Type, reg32Constr); + Opnd* bx = irm->newOpnd(int16Type, reg16Constr); + Opnd* bl = irm->newOpnd(int8Type, reg8Constr); + + Node* mainLoop = cfg->createBlockNode(); + Node* copy2BytesEndTest = cfg->createBlockNode(); + Node* copy2BytesEnd = cfg->createBlockNode(); + Node* copy1ByteEndTest = cfg->createBlockNode(); + Node* copy1ByteEnd = cfg->createBlockNode(); + + Edge* outEdge = currNode->getOutEdge(Edge::Kind_Unconditional); + cfg->removeEdge(outEdge); + cfg->addEdge(currNode, copy2BytesEndTest, 0.4); + cfg->addEdge(currNode, mainLoop, 0.6); + cfg->addEdge(mainLoop, mainLoop, 0.85); + cfg->addEdge(mainLoop, copy2BytesEndTest, 0.15); + cfg->addEdge(copy2BytesEndTest, copy2BytesEnd, 0.6); + cfg->addEdge(copy2BytesEndTest, copy1ByteEndTest, 0.4); + cfg->addEdge(copy2BytesEnd, copy1ByteEndTest); + cfg->addEdge(copy1ByteEndTest, copy1ByteEnd, 0.6); + cfg->addEdge(copy1ByteEndTest, exitNode, 0.4); + cfg->addEdge(copy1ByteEnd, exitNode); + + currNode->appendInst(irm->newInstEx(Mnemonic_MOV, 1, edx, four32bit)); // mov edx, 4 + currNode->appendInst(irm->newInstEx(Mnemonic_SUB, 1, srcAddr, srcAddr, four8bit)); // sub esi, 4 + currNode->appendInst(irm->newInstEx(Mnemonic_SUB, 1, dstAddr, dstAddr, four8bit)); // sub edi, 4 + currNode->appendInst(irm->newInstEx(Mnemonic_CMP, 0, counter, four8bit)); // cmp ecx, 4 + currNode->appendInst(irm->newBranchInst(Mnemonic_JL, copy2BytesEndTest, mainLoop)); // jl copy2BytesEndTest + + mainLoop->appendInst(irm->newInstEx(Mnemonic_MOV, 1, ebx, irm->newMemOpnd(int32Type, srcAddr, edx, 0, 0))); // mov ebx, [esi + edx] + mainLoop->appendInst(irm->newInstEx(Mnemonic_MOV, 1, irm->newMemOpnd(int32Type, dstAddr, edx, 0, 0), ebx)); // mov [edi + edx], ebx + mainLoop->appendInst(irm->newInstEx(Mnemonic_ADD, 1, edx, edx, four8bit)); // add edx, 4 + mainLoop->appendInst(irm->newInstEx(Mnemonic_CMP, 0, edx, counter)); // cmp edx, ecx + mainLoop->appendInst(irm->newBranchInst(Mnemonic_JLE, mainLoop, copy2BytesEndTest)); // jle mainLoop + + copy2BytesEndTest->appendInst(irm->newInstEx(Mnemonic_TEST, 0, counter, two32bit)); // test ecx, 2 + copy2BytesEndTest->appendInst(irm->newBranchInst(Mnemonic_JZ, copy1ByteEndTest, copy2BytesEnd)); // jz copy1ByteEndTest + + copy2BytesEnd->appendInst(irm->newInstEx(Mnemonic_MOV, 1, bx, irm->newMemOpnd(int16Type, srcAddr, edx, 0, 0))); // mov bx, [esi + edx] + copy2BytesEnd->appendInst(irm->newInstEx(Mnemonic_MOV, 1, irm->newMemOpnd(int16Type, dstAddr, edx, 0, 0), bx)); // mov [edi + edx], bx + copy2BytesEnd->appendInst(irm->newInstEx(Mnemonic_ADD, 1, edx, edx, two8bit)); // add edx, 2 + + copy1ByteEndTest->appendInst(irm->newInstEx(Mnemonic_TEST, 0, counter, one32bit)); // test ecx, 1 + copy1ByteEndTest->appendInst(irm->newBranchInst(Mnemonic_JZ, exitNode, copy1ByteEnd)); // jz exitNode + + copy1ByteEnd->appendInst(irm->newInstEx(Mnemonic_MOV, 1, bl, irm->newMemOpnd(int8Type, srcAddr, edx, 0, 0))); // mov bl, [esi + edx] + copy1ByteEnd->appendInst(irm->newInstEx(Mnemonic_MOV, 1, irm->newMemOpnd(int8Type, dstAddr, edx, 0, 0), bl)); // mov [edi + edx], bl +} + +void MemoryCopyOptimizer::genDirectMemoryCopy(uint32 numArgs, Opnd** args, Node* currNode) +{ +#ifdef _EM64T_ + RegName counterRegName = RegName_RCX; + RegName srcAddrRegName = RegName_RSI; + RegName dstAddrRegName = RegName_RDI; +#else + RegName counterRegName = RegName_ECX; + RegName srcAddrRegName = RegName_ESI; + RegName dstAddrRegName = RegName_EDI; +#endif + + // prepare counter + Type* counterType = typeManager->getIntPtrType(); + Opnd* counter = irm->newRegOpnd(counterType,counterRegName); + currNode->appendInst(irm->newCopyPseudoInst(Mnemonic_MOV, counter, args[4])); + + // prepare src position + Opnd* srcAddr = genLoadIndexWithLEA(args[0],args[1], srcAddrRegName, currNode); + + // prepare dst position + Opnd* dstAddr = genLoadIndexWithLEA(args[2],args[3], dstAddrRegName, currNode); + + Opnd* one = irm->newImmOpnd(counterType, (int32)1); + + PtrType* srcAddrType = srcAddr->getType()->asPtrType(); + Type::Tag tag = srcAddrType->getPointedToType()->tag; + Mnemonic mn = Mnemonic_NULL; + OpndSize typeSize = IRManager::getTypeSize(tag); + switch(typeSize) { + case OpndSize_8: mn = Mnemonic_MOVS8; break; + case OpndSize_16: mn = Mnemonic_MOVS16; break; + case OpndSize_32: mn = Mnemonic_MOVS32; break; + case OpndSize_64: + { + /** + * FIXME + * Currently JIT erroneously supposes that compressed mode is always on. + * So if type is object, it is actually compressed (32-bit sized). + * But IRManager::getTypeSize() "correctly" returns OpndSize_64. + */ + if (!srcAddrType->getPointedToType()->isObject()) { + currNode->appendInst(irm->newInstEx(Mnemonic_SHL, 1, counter, counter, one)); + } + mn = Mnemonic_MOVS32; + } + break; + default: assert(0); mn = Mnemonic_MOVS32; break; + } + + Inst* copyInst = irm->newInst(mn,dstAddr,srcAddr,counter); + copyInst->setPrefix(InstPrefix_REP); + currNode->appendInst(copyInst); +} + +void MemoryCopyOptimizer::genReverseMemoryCopy(uint32 numArgs, Opnd** args, Node* currNode) +{ + currNode->appendInst(irm->newInst(Mnemonic_PUSHFD)); + currNode->appendInst(irm->newInst(Mnemonic_STD)); + + genDirectMemoryCopy(numArgs,args,currNode); + + currNode->appendInst(irm->newInst(Mnemonic_POPFD)); +} + +void +MemoryCopyOptimizer::OptimizeMemoryCopy() +{ + const Nodes& nodes = cfg->getNodes(); + for (Nodes::const_iterator it = nodes.begin(),end = nodes.end();it!=end; ++it) { + Node* bb = *it; + if (!bb->isBlockNode()) { + continue; + } + if(bb->isEmpty()) { + continue; + } + + Inst * inst = (Inst*)bb->getLastInst(); + if (inst->getMnemonic() != Mnemonic_CALL) { + continue; + } + + Opnd::RuntimeInfo * rt = inst->getOpnd(((ControlTransferInst*)inst)->getTargetOpndIndex())->getRuntimeInfo(); + + if (!rt || rt->getKind() != Opnd::RuntimeInfo::Kind_InternalHelperAddress) { + continue; + } + + std::string str = std::string((const char*)rt->getValue(0)); + if (str != "memory_copy_direct" && str!= "memory_copy_reverse" && str!= "memory_copy_fast") { + continue; + } + exam2Insts->push_back(inst); + } + + for (Insts::iterator it = exam2Insts->begin(), end = exam2Insts->end(); it!=end; ++it) + { + Inst* inst = *it; + Node* currNode = inst->getNode(); + inst->unlink(); + + Edge * outEdge = currNode->getOutEdge(Edge::Kind_Unconditional); + assert(outEdge!= NULL); + Node * nextNode = outEdge->getTargetNode(); + + Inst::Opnds opnds(inst, Inst::OpndRole_Use|Inst::OpndRole_Auxilary); + Inst::Opnds::iterator ito = opnds.begin(); + + Opnd* args[5] = {inst->getOpnd(ito++), inst->getOpnd(ito++), inst->getOpnd(ito++), inst->getOpnd(ito++), inst->getOpnd(ito++)}; + + Opnd::RuntimeInfo * rt = inst->getOpnd(((ControlTransferInst*)inst)->getTargetOpndIndex())->getRuntimeInfo(); + std::string str = std::string((const char*)rt->getValue(0)); + + Edge* dispEdge = currNode->getOutEdge(Edge::Kind_Dispatch); + if (dispEdge!= NULL) + cfg->removeEdge(dispEdge); + + if (str == "memory_copy_fast") + genFastMemoryCopy(5, args, currNode, nextNode); + else + if (str == "memory_copy_direct") + genDirectMemoryCopy(5, args, currNode); + else + genReverseMemoryCopy(5, args, currNode); + } +} + +class FastMemoryCopy: public SessionAction { +public: + void runImpl(); +}; + +void FastMemoryCopy::runImpl() +{ + MemoryManager mcMemManager("Memorycopyoptimizer mm"); + MemoryCopyOptimizer memcpyopt(irManager, mcMemManager); + + memcpyopt.OptimizeMemoryCopy(); +} + +static ActionFactory _fmemcpy("FastMemoryCopy"); + +}} \ No newline at end of file Index: jitrino/src/dynopt/EdgeProfiler.cpp =================================================================== --- jitrino/src/dynopt/EdgeProfiler.cpp (revision 582116) +++ jitrino/src/dynopt/EdgeProfiler.cpp (working copy) @@ -666,7 +666,7 @@ } Inst* last = (Inst*)node->getLastInst(); // This method is not a leaf method. - if( last->getOpcode() >= Op_DirectCall && last->getOpcode() <= Op_IntrinsicCall ){ + if( last->getOpcode() >= Op_DirectCall && last->getOpcode() <= Op_VMHelperCall ){ return false; } Edges::const_iterator eiter; Index: jitrino/src/dynopt/StaticProfiler.cpp =================================================================== --- jitrino/src/dynopt/StaticProfiler.cpp (revision 582116) +++ jitrino/src/dynopt/StaticProfiler.cpp (working copy) @@ -295,6 +295,7 @@ void StaticProfilerPass::_run(IRManager& irm) { + const char* name = irm.getMethodDesc().getName(); OptPass::computeDominatorsAndLoops(irm, false); StaticProfiler::estimateGraph(irm, DEFAULT_ENTRY_NODE_FREQUENCY); } @@ -346,7 +347,7 @@ double dprob = (*fn)(c); assert(dprob>0 && dprob<1); if (dprob!=PROB_HEURISTIC_FAIL) { - double newprob = prob * dprob / (prob*dprob + (1-prob)*(1-dprob)); + double newprob = prob*dprob / (prob*dprob + (1-prob)*(1-dprob)); assert(newprob>0 && newprob<1); prob = newprob; } @@ -415,8 +416,8 @@ static double callHeuristic(const StaticProfilerContext* c) { Node* node1 = c->edge1->getTargetNode(); Node* node2 = c->edge2->getTargetNode(); - bool node1HasCall = findInst(node1, Op_DirectCall, Op_IntrinsicCall)!=NULL; - bool node2HasCall = findInst(node2, Op_DirectCall, Op_IntrinsicCall)!=NULL; + bool node1HasCall = findInst(node1, Op_DirectCall, Op_VMHelperCall)!=NULL; + bool node2HasCall = findInst(node2, Op_DirectCall, Op_VMHelperCall)!=NULL; if (!node1HasCall && !node2HasCall) { return PROB_HEURISTIC_FAIL; Index: jitrino/src/dynopt/ValueProfiler.cpp =================================================================== --- jitrino/src/dynopt/ValueProfiler.cpp (revision 582116) +++ jitrino/src/dynopt/ValueProfiler.cpp (working copy) @@ -110,7 +110,7 @@ Opnd* valueOpnd = vtableInst->getDst(); const uint32 numArgs = 2; Opnd* args[numArgs] = {indexOpnd, valueOpnd}; - Inst* addValueInst = instFactory.makeJitHelperCall(opndManager.getNullOpnd(), AddValueProfileValue, numArgs, args); + Inst* addValueInst = instFactory.makeJitHelperCall(opndManager.getNullOpnd(), AddValueProfileValue, NULL, NULL, numArgs, args); addValueInst->setBCOffset(call->getBCOffset()); ((CFGInst *)addValueInst)->insertBefore(call); ((CFGInst *)loadIndexInst)->insertBefore(addValueInst); Index: jitrino/src/optimizer/arraycopyoptimize.cpp =================================================================== --- jitrino/src/optimizer/arraycopyoptimize.cpp (revision 0) +++ jitrino/src/optimizer/arraycopyoptimize.cpp (revision 0) @@ -0,0 +1,599 @@ +#include "optpass.h" +#include "IRManager.h" + +namespace Jitrino { + +class ArrayCopyOptimizer { +public: + ArrayCopyOptimizer(IRManager& irmanager, MemoryManager& mm); + void OptimizeArrayCopy(IRManager& irm); + +private: + bool methodIsArraycopy(MethodDesc * methodDesc); + bool arraycopyOptimizable(MethodDesc * methodDesc, uint32 numArgs, Opnd ** srcOpnds); + void genOptimizedArrayCopy(uint32 numArgs, Opnd** srcOpnds, Node* firstNode, Node* lastNode, uint16 BCoffset); + + Opnd* createOpnd(Type* type); + Opnd* genTauCheckNull(Opnd* base, Node* currNode, uint16 BCoffset); + Opnd* genAnd(Type* dstType, Opnd* src1, Opnd* src2, Node* currNode, uint16 BCoffset); + Opnd* genTauAnd(Opnd *src1, Opnd *src2, Node* currNode, uint16 BCoffset); + Opnd* genAdd(Type* dstType, Modifier mod, Opnd* src1, Opnd* src2, Node* currNode, uint16 BCoffset); + Opnd* genLdConstant(int32 val, Node* currNode, uint16 BCoffset); + Opnd* genArrayLen(Type* dstType, Type::Tag type, Opnd* array, Opnd* tauNonNull, Node* currNode, uint16 BCoffset); + Opnd* genTauArrayLen(Type* dstType, Type::Tag type, Opnd* array, + Opnd* tauNullChecked, Opnd *tauTypeChecked, Node* currNode, uint16 BCoffset); + Opnd* genCmp3(Type* dstType, Type::Tag instType, + ComparisonModifier mod, Opnd* src1, Opnd* src2, Node* currNode, uint16 BCoffset); + Opnd* genCmp(Type* dstType, Type::Tag instType, + ComparisonModifier mod, Opnd* src1, Opnd* src2, Node* currNode, uint16 BCoffset); + Opnd* genTauSafe(Node* currNode, uint16 BCoffset); + Opnd* genTauCheckBounds(Opnd* array, Opnd* index, Opnd *tauNullChecked, Node* currNode, uint16 BCoffset); + Opnd* genTauCheckBounds(Opnd* ub, Opnd *index, Node* currNode, uint16 BCoffset); + Opnd* genTauHasType(Opnd *src, Type *castType, Node* currNode, uint16 BCoffset); + Opnd* genJitHelperCall(JitHelperCallId helperId, Type* returnType, Opnd* tauNullCheckedRefArgs, Opnd* tauTypesChecked, + uint32 numArgs, Opnd* args[], Node* currNode, uint16 BCoffset); + + typedef StlList Insts; + +private: + IRManager* irm; + InstFactory* instFactory; + OpndManager* opndManager; + TypeManager* typeManager; + ControlFlowGraph* cfg; + MemoryManager& acmm; + + Insts *exam2Insts; +}; + + + +ArrayCopyOptimizer::ArrayCopyOptimizer(IRManager& irmanager, MemoryManager& mm) +: acmm(mm) +{ + irm = &irmanager; + instFactory = &irm->getInstFactory(); + opndManager = &irm->getOpndManager(); + typeManager = &irm->getTypeManager(); + cfg = &irm->getFlowGraph(); + exam2Insts = new (acmm) Insts(acmm); +} + +bool +ArrayCopyOptimizer::methodIsArraycopy(MethodDesc * methodDesc) { + + return (strcmp(methodDesc->getName(),"arraycopy") == 0 && + strcmp(methodDesc->getParentType()->getName(),"java/lang/System") ==0); +} + + +bool +ArrayCopyOptimizer::arraycopyOptimizable(MethodDesc * methodDesc, + uint32 numArgs, + Opnd ** srcOpnds) { + + // + // an ArrayStoreException is thrown and the destination is not modified: + // + // - The src argument refers to an object that is not an array. + // - The dest argument refers to an object that is not an array. + // - The src argument and dest argument refer to arrays whose component types are different primitive types. + // - The src argument refers to an array with a primitive component type and the dest argument + // refers to an array with a reference component type. + // - The src argument refers to an array with a reference component type and the dest argument + // refers to an array with a primitive component type. + // + assert(numArgs == 5); + Opnd * src = srcOpnds[0]; + Type * srcType = src->getType(); + Opnd * dst = srcOpnds[2]; + Type * dstType = dst->getType(); + assert(srcType->isObject() && + srcOpnds[1]->getType()->isInt4() && // 1 - srcPos + dstType->isObject() && + srcOpnds[3]->getType()->isInt4() && // 3 - dstPos + srcOpnds[4]->getType()->isInt4()); // 4 - length + + bool throwsASE = false; + bool srcIsArray = srcType->isArray() && !srcType->isUnresolvedType(); + bool dstIsArray = dstType->isArray() && !dstType->isUnresolvedType(); + ArrayType* srcAsArrayType = srcType->asArrayType(); + ArrayType* dstAsArrayType = dstType->asArrayType(); + bool srcIsArrOfPrimitive = srcIsArray && VMInterface::isArrayOfPrimitiveElements(srcAsArrayType->getVMTypeHandle()); + bool dstIsArrOfPrimitive = dstIsArray && VMInterface::isArrayOfPrimitiveElements(dstAsArrayType->getVMTypeHandle()); + if ( !(srcIsArray && dstIsArray) ) { + throwsASE = true; + } else if ( srcIsArrOfPrimitive ) { + if( !dstIsArrOfPrimitive || srcType != dstType ) + throwsASE = true; + } else if( dstIsArrOfPrimitive ) { + throwsASE = true; + } else { // the both are of objects + // Here is some inaccuracy. If src is a subclass of dst there is no ASE for sure. + // If it is not, we should check the assignability of each element being copied. + // To avoid this we just reject the inlining of System::arraycopy call in this case. + NamedType* srcElemType = srcAsArrayType->getElementType(); + NamedType* dstElemType = dstAsArrayType->getElementType(); + throwsASE = srcElemType->getVMTypeHandle() != dstElemType->getVMTypeHandle(); + } + if ( throwsASE ) + return false; + else + return true; +} + + +void +ArrayCopyOptimizer::genOptimizedArrayCopy(uint32 numArgs, Opnd** srcOpnds, Node* firstNode, Node* lastNode, uint16 BCoffset) +{ + Opnd * src = srcOpnds[0]; + Opnd * srcPos = srcOpnds[1]; + Type * srcPosType = srcPos->getType(); + Opnd * dst = srcOpnds[2]; + Opnd * dstPos = srcOpnds[3]; + Type * dstPosType = dstPos->getType(); + Opnd * len = srcOpnds[4]; + + // + // Generate exception condition checks: + // chknull src + // chknull dst + // cmpbr srcPos < 0, boundsException + // cmpbr dstPos < 0, boundsException + // cmpbr len < 0, boundsException + // srcEnd = add srcPos, len + // srcLen = src.length + // cmpbr srcEnd > srcLen, boundsException + // dstEnd = add dstPos, len + // dstLen = dst.length + // cmpbr dstEnd > dstLen, boundsException + // Skip trivial: + // cmpbr (src == dst) && (dstPos == srcPos), Exit + // + // Choose method: + // cmpbr (src != dst), Fast + // Choose a direction: + // cmpbr (dstPos > srcPos), Reverse + // + // Direct: + // JitHelperCall id=ArrayCopyDirect + // goto Exit + // Reverse: + // srcPos = srcPos + len - 1 + // dstPos = dstPos + len - 1 + // JitHelperCall id=ArrayCopyReverse + // goto Exit + // + // Fast: + // JitHelperCall id=ArrayCopyFast + // goto Exit + // boundsException: + // chkbounds -1, src + // Exit: + // + + Node *currNode, *prevNode; + Node* dispatch = cfg->getUnwindNode(); + + + // Referenced nodes creation + LabelInst * reverseCopying = (LabelInst*)instFactory->makeLabel(); + LabelInst * fastCopying = (LabelInst*)instFactory->makeLabel(); + LabelInst * boundsException = (LabelInst*)instFactory->makeLabel(); + LabelInst * Exit = (LabelInst*)instFactory->makeLabel(); + + Node *exitNode = cfg->createBlockNode(Exit); + Node *BExcNode = cfg->createBlockNode(boundsException); + Node *RevCopyNode = cfg->createBlockNode(reverseCopying); + Node *FastCopyNode = cfg->createBlockNode(fastCopying); + + //Other nodes creation and filling + Opnd *tauSrcNullChecked = genTauCheckNull(src, firstNode, BCoffset); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(firstNode, currNode); + cfg->addEdge(currNode, dispatch, 0); + prevNode = currNode; + Opnd *tauDstNullChecked = genTauCheckNull(dst, currNode, BCoffset); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + prevNode = currNode; + Opnd *tauNullCheckedRefArgs = genTauAnd(tauSrcNullChecked, tauDstNullChecked, currNode, BCoffset); + + Type * intType = typeManager->getInt32Type(); + Type::Tag intTag = intType->tag; + Type * voidType = typeManager->getVoidType(); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + cfg->addEdge(currNode, BExcNode, 0); + prevNode = currNode; + + Opnd * zero = genLdConstant((int32)0, currNode, BCoffset); + Opnd * minusone = genLdConstant((int32)-1, currNode, BCoffset); + Inst* srcBExc = instFactory->makeBranch(Cmp_GT,intTag,zero,srcPos,boundsException); + srcBExc->setBCOffset(BCoffset); + currNode->appendInst(srcBExc); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + cfg->addEdge(currNode, BExcNode, 0); + prevNode = currNode; + + Inst* dstBExc = instFactory->makeBranch(Cmp_GT,intTag,zero,dstPos,boundsException); + dstBExc->setBCOffset(BCoffset); + currNode->appendInst(dstBExc); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + cfg->addEdge(currNode, BExcNode, 0); + prevNode = currNode; + + Inst* lenBExc = instFactory->makeBranch(Cmp_GT,intTag,zero,len,boundsException); + lenBExc->setBCOffset(BCoffset); + currNode->appendInst(lenBExc); + + Modifier mod = Modifier(Overflow_None)|Modifier(Exception_Never)|Modifier(Strict_No); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + cfg->addEdge(currNode, BExcNode, 0); + prevNode = currNode; + + Opnd * srcLen = genArrayLen(intType, intTag, src, tauSrcNullChecked, currNode, BCoffset); + Opnd * srcEnd = genAdd(intType,mod,srcPos,len, currNode, BCoffset); + + Inst* srcEndBExc = instFactory->makeBranch(Cmp_GT,intTag,srcEnd,srcLen,boundsException); + srcEndBExc->setBCOffset(BCoffset); + currNode->appendInst(srcEndBExc); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + cfg->addEdge(currNode, BExcNode, 0); + prevNode = currNode; + + Opnd * dstEnd = genAdd(intType,mod,dstPos,len, currNode, BCoffset); + Opnd * dstLen = genArrayLen(intType, intTag, dst, tauDstNullChecked, currNode, BCoffset); + Inst* dstEndBExc = instFactory->makeBranch(Cmp_GT,intTag,dstEnd,dstLen,boundsException); + dstEndBExc->setBCOffset(BCoffset); + currNode->appendInst(dstEndBExc); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + cfg->addEdge(currNode, exitNode, 0); + prevNode = currNode; + + // The case of same arrays and same positions + Opnd * diff = genCmp3(intType,intTag,Cmp_GT,dstPos,srcPos, currNode, BCoffset); + Opnd * sameArrays = genCmp(intType,Type::IntPtr,Cmp_EQ,src,dst, currNode, BCoffset); + Opnd * zeroDiff = genCmp(intType,intTag,Cmp_EQ,diff,zero, currNode, BCoffset); + Opnd * nothingToCopy = genAnd(intType,sameArrays,zeroDiff, currNode, BCoffset); + Inst* nothingCopy = instFactory->makeBranch(Cmp_GT,intTag,nothingToCopy,zero,Exit); + nothingCopy->setBCOffset(BCoffset); + currNode->appendInst(nothingCopy); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + cfg->addEdge(currNode, FastCopyNode); + prevNode = currNode; + + Opnd* tauTypesChecked = genTauSafe(currNode, BCoffset); + + // Checking for opportunity to use optimized method + Inst* fCopy = instFactory->makeBranch(Cmp_NE_Un,Type::Object,src,dst,fastCopying); + fCopy->setBCOffset(BCoffset); + currNode->appendInst(fCopy); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode, 0); + cfg->addEdge(currNode, RevCopyNode, 0); + prevNode = currNode; + + // Choosing direction + Inst* revCopy = instFactory->makeBranch(Cmp_GT,intTag,dstPos,srcPos,reverseCopying); + revCopy->setBCOffset(BCoffset); + currNode->appendInst(revCopy); + + // node + currNode = cfg->createBlockNode(instFactory->makeLabel()); + cfg->addEdge(prevNode, currNode); + cfg->addEdge(currNode, exitNode); + cfg->addEdge(currNode, dispatch, 0); + + { // Direct Copying + genJitHelperCall(ArrayCopyDirect,voidType, + tauNullCheckedRefArgs, + tauTypesChecked, + numArgs,srcOpnds, currNode, BCoffset); + } // End of Direct Copying + + + // Referenced nodes filling + // Reverse Copy + cfg->addEdge(RevCopyNode, exitNode); + cfg->addEdge(RevCopyNode, dispatch, 0); + { // Reverse Copying + + Opnd* lastSrcIdx = genAdd(srcPosType,mod,srcEnd,minusone, RevCopyNode, BCoffset); + Opnd* lastDstIdx = genAdd(dstPosType,mod,dstEnd,minusone, RevCopyNode, BCoffset); + + Opnd* reverseArgs [5];// = new (acmm) Opnd*[numArgs]; + + reverseArgs[0] = srcOpnds[0]; // src + reverseArgs[1] = lastSrcIdx; // srcPos+len-1 + reverseArgs[2] = srcOpnds[2]; // dst + reverseArgs[3] = lastDstIdx; // dstPos+len-1 + reverseArgs[4] = srcOpnds[4]; // len + genJitHelperCall(ArrayCopyReverse,voidType, + tauNullCheckedRefArgs, + tauTypesChecked, + numArgs,reverseArgs, RevCopyNode, BCoffset); + } // End of Reverse Copying + + // Fast Copy + cfg->addEdge(FastCopyNode, exitNode); + cfg->addEdge(FastCopyNode, dispatch, 0); + { // Fast Copying + genJitHelperCall(ArrayCopyFast,voidType, + tauNullCheckedRefArgs, + tauTypesChecked, + numArgs,srcOpnds, FastCopyNode, BCoffset); + } // End of Fast Copying + + // Bounds Exception + cfg->addEdge(BExcNode, exitNode); + cfg->addEdge(BExcNode, dispatch, 0); + genTauCheckBounds(src,minusone,tauSrcNullChecked, BExcNode, BCoffset); + + // Exit + cfg->addEdge(exitNode, lastNode); +} + + +Opnd* +ArrayCopyOptimizer::createOpnd(Type* type) { + if (type->tag == Type::Void) + return OpndManager::getNullOpnd(); + return opndManager->createSsaTmpOpnd(type); +} + +Opnd* +ArrayCopyOptimizer::genTauCheckNull(Opnd* base, Node* currNode, uint16 BCoffset) +{ + Opnd* dst = createOpnd(typeManager->getTauType()); + Inst* inst = instFactory->makeTauCheckNull(dst, base); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genAnd(Type* dstType, Opnd* src1, Opnd* src2, Node* currNode, uint16 BCoffset) { + Opnd* dst = createOpnd(dstType); + Inst* inst = instFactory->makeAnd(dst, src1, src2); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genTauAnd(Opnd *src1, Opnd *src2, Node* currNode, uint16 BCoffset) { + if (src1->getId() > src2->getId()) { + Opnd *tmp = src1; + src1 = src2; + src2 = tmp; + } + Opnd* dst = createOpnd(typeManager->getTauType()); + Opnd* srcs[2] = { src1, src2 }; + Inst* inst = instFactory->makeTauAnd(dst, 2, srcs); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + + return dst; +} + +Opnd* +ArrayCopyOptimizer::genAdd(Type* dstType, Modifier mod, Opnd* src1, Opnd* src2, Node* currNode, uint16 BCoffset) { + Opnd* dst = createOpnd(dstType); + Inst *inst = instFactory->makeAdd(mod, dst, src1, src2); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + + +Opnd* +ArrayCopyOptimizer::genLdConstant(int32 val, Node* currNode, uint16 BCoffset) { + Opnd* dst = createOpnd(typeManager->getInt32Type()); + Inst* inst = instFactory->makeLdConst(dst, val); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genArrayLen(Type* dstType, Type::Tag type, Opnd* array, Opnd* tauNonNull, Node* currNode, uint16 BCoffset) { + Opnd *tauIsArray = genTauHasType(array, array->getType(), currNode, BCoffset); + + return genTauArrayLen(dstType, type, array, tauNonNull, tauIsArray, currNode, BCoffset); +} + +Opnd* +ArrayCopyOptimizer::genTauArrayLen(Type* dstType, Type::Tag type, Opnd* array, + Opnd* tauNullChecked, Opnd *tauTypeChecked, Node* currNode, uint16 BCoffset) { + Opnd* dst = createOpnd(dstType); + Inst* inst = instFactory->makeTauArrayLen(dst, type, array, tauNullChecked, + tauTypeChecked); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genCmp3(Type* dstType, + Type::Tag instType, // source type for inst + ComparisonModifier mod, + Opnd* src1, + Opnd* src2, + Node* currNode, uint16 BCoffset) { + // result of comparison is always a 32-bit int + Opnd* dst = createOpnd(dstType); + Inst* inst = instFactory->makeCmp3(mod, instType, dst, src1, src2); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genCmp(Type* dstType, + Type::Tag instType, // source type for inst + ComparisonModifier mod, + Opnd* src1, + Opnd* src2, + Node* currNode, uint16 BCoffset) { + // result of comparison is always a 32-bit int + Opnd* dst = createOpnd(dstType); + Inst *inst = instFactory->makeCmp(mod, instType, dst, src1, src2); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genTauSafe(Node* currNode, uint16 BCoffset) { + Opnd* dst = createOpnd(typeManager->getTauType()); + Inst* inst = instFactory->makeTauSafe(dst); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genTauCheckBounds(Opnd* array, Opnd* index, Opnd *tauNullChecked, Node* currNode, uint16 BCoffset) { + Opnd *tauArrayTypeChecked = genTauHasType(array, array->getType(), currNode, BCoffset); + Opnd* arrayLen = genTauArrayLen(typeManager->getInt32Type(), Type::Int32, array, + tauNullChecked, tauArrayTypeChecked, currNode, BCoffset); + + Opnd* dst = genTauCheckBounds(arrayLen, index, currNode, BCoffset); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genTauCheckBounds(Opnd* ub, Opnd *index, Node* currNode, uint16 BCoffset) { + Opnd* dst = createOpnd(typeManager->getTauType()); + Inst* inst = instFactory->makeTauCheckBounds(dst, ub, index); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genTauHasType(Opnd *src, Type *castType, Node* currNode, uint16 BCoffset) { + Opnd* dst = createOpnd(typeManager->getTauType()); + Inst* inst = instFactory->makeTauHasType(dst, src, castType); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +Opnd* +ArrayCopyOptimizer::genJitHelperCall(JitHelperCallId helperId, + Type* returnType, + Opnd* tauNullCheckedRefArgs, + Opnd* tauTypesChecked, + uint32 numArgs, + Opnd* args[], + Node* currNode, uint16 BCoffset) { + Opnd * dst = createOpnd(returnType); + Inst* inst = instFactory->makeJitHelperCall(dst, helperId, + tauNullCheckedRefArgs, + tauTypesChecked, + numArgs, args); + inst->setBCOffset(BCoffset); + currNode->appendInst(inst); + return dst; +} + +void +ArrayCopyOptimizer::OptimizeArrayCopy(IRManager& irm) +{ + Opnd *srcOpnds[7]; + + const Nodes& nodes = irm.getFlowGraph().getNodes(); + for (Nodes::const_iterator it = nodes.begin(), end = nodes.end(); it!=end; ++it) + { + Node* node = *it; + if (node->isBlockNode()) { + for (Inst* inst = (Inst*)node->getFirstInst(); inst!=NULL; inst = inst->getNextInst()) { + if (inst->getOpcode() == Op_DirectCall) + { + MethodCallInst * call = (MethodCallInst *)inst; + MethodDesc * methDesc = call->getMethodDesc(); + if ( methodIsArraycopy(methDesc) ) + { + uint32 numArgs = call->getNumSrcOperands(); + + for (uint32 i = 0; igetSrc(i+2); + } + + if (arraycopyOptimizable(methDesc, numArgs-2, srcOpnds)) + { + exam2Insts->push_back(inst); + } + } + } + } + } + } + + for (Insts::iterator it = exam2Insts->begin(), end = exam2Insts->end(); it!=end; ++it) + { + Inst* inst = *it; + MethodCallInst * call = (MethodCallInst *)inst; + MethodDesc * methDesc = call->getMethodDesc(); + + uint32 numArgs = call->getNumSrcOperands(); + + for (uint32 i = 0; igetSrc(i+2); + } + + Node* currNode = inst->getNode(); + Edge* outEdge = currNode->getOutEdge(Edge::Kind_Unconditional); + Node* nextNode = outEdge->getTargetNode(); + cfg->removeEdge(outEdge); + uint16 BCoffset = inst->getBCOffset(); + inst->unlink(); + + genOptimizedArrayCopy(numArgs-2, srcOpnds, currNode, nextNode, BCoffset); + } +} + + +DEFINE_SESSION_ACTION(ArrayCopyOptimizePass, arrcpy, "Array Copy Optimization") + +void +ArrayCopyOptimizePass::_run(IRManager& irm) +{ + MemoryManager acMemManager("Arraycopyoptimizer mm"); + ArrayCopyOptimizer arrcpyopt(irm, acMemManager); + + if (!irm.getCompilationInterface().needWriteBarriers()) + arrcpyopt.OptimizeArrayCopy(irm); +} + +} //namespace Jitrino \ No newline at end of file Index: jitrino/src/optimizer/codelowerer.h =================================================================== --- jitrino/src/optimizer/codelowerer.h (revision 582116) +++ jitrino/src/optimizer/codelowerer.h (working copy) @@ -136,8 +136,6 @@ Inst* caseIndirectMemoryCall(CallInst* inst) {return caseDefault(inst);} - Inst* caseIntrinsicCall(IntrinsicCallInst* inst) {return caseDefault(inst);} - Inst* caseJitHelperCall(JitHelperCallInst* inst) {return caseDefault(inst);} Inst* caseVMHelperCall(VMHelperCallInst* inst) {return caseDefault(inst);} Index: jitrino/src/optimizer/CodeSelectors.cpp =================================================================== --- jitrino/src/optimizer/CodeSelectors.cpp (revision 582116) +++ jitrino/src/optimizer/CodeSelectors.cpp (working copy) @@ -388,19 +388,6 @@ return ConvertToIntOp::NoOvf; // to keep the compiler quiet } -// -// Maps intrinsic id -// -IntrinsicCallOp::Id _BlockCodeSelector::convertIntrinsicId(IntrinsicCallId callId) { - switch(callId) { - case CharArrayCopy: return IntrinsicCallOp::CharArrayCopy; - case ArrayCopyDirect: return IntrinsicCallOp::ArrayCopyDirect; - case ArrayCopyReverse: return IntrinsicCallOp::ArrayCopyReverse; - } - assert(0); - return IntrinsicCallOp::CharArrayCopy; // to keep compiler quiet -} - JitHelperCallOp::Id _BlockCodeSelector::convertJitHelperId(JitHelperCallId callId) { switch(callId) { case InitializeArray: return JitHelperCallOp::InitializeArray; @@ -409,6 +396,10 @@ case LockedCompareAndExchange: return JitHelperCallOp::LockedCompareAndExchange; case AddValueProfileValue: return JitHelperCallOp::AddValueProfileValue; case FillArrayWithConst: return JitHelperCallOp::FillArrayWithConst; + case ArrayCopyDirect: return JitHelperCallOp::ArrayCopyDirect; + case ArrayCopyReverse: return JitHelperCallOp::ArrayCopyReverse; + case ArrayCopyFast: return JitHelperCallOp::ArrayCopyFast; + } assert(0); return JitHelperCallOp::InitializeArray; // to keep compiler quiet @@ -852,7 +843,7 @@ getCGInst(tauTypesChecked)); } break; - case Op_IntrinsicCall: +/* case Op_IntrinsicCall: { assert(inst->getNumSrcOperands() >= 2); Opnd *tauNullChecked = inst->getSrc(0); @@ -888,11 +879,18 @@ } break; - case Op_JitHelperCall: +*/ case Op_JitHelperCall: { JitHelperCallInst* call = inst->asJitHelperCallInst(); JitHelperCallId callId = call->getJitHelperId(); - cgInst = + if (callId == ArrayCopyDirect || callId == ArrayCopyReverse || callId == ArrayCopyFast) + cgInst = + instructionCallback.callhelper(inst->getNumSrcOperands()-2, // omit taus + genCallArgs(call,2), // omit taus + inst->getDst()->getType(), + convertJitHelperId(callId)); + else + cgInst = instructionCallback.callhelper(inst->getNumSrcOperands(), genCallArgs(call,0), inst->getDst()->getType(), Index: jitrino/src/optimizer/CodeSelectors.h =================================================================== --- jitrino/src/optimizer/CodeSelectors.h (revision 582116) +++ jitrino/src/optimizer/CodeSelectors.h (working copy) @@ -104,9 +104,6 @@ // Maps instruction to ConvertToIntOp::OverflowMod ConvertToIntOp::OverflowMod mapToIntConvertOvfMod(Inst *inst); - // Maps intrinsic id - IntrinsicCallOp::Id convertIntrinsicId(IntrinsicCallId callId); - JitHelperCallOp::Id convertJitHelperId(JitHelperCallId callId); CG_OpndHandle ** genCallArgs(Inst * call, uint32 arg0Pos); Index: jitrino/src/optimizer/escanalyzer.cpp =================================================================== --- jitrino/src/optimizer/escanalyzer.cpp (revision 582116) +++ jitrino/src/optimizer/escanalyzer.cpp (working copy) @@ -404,7 +404,7 @@ } } break; - case Op_JitHelperCall: // calljithelper + /*case Op_JitHelperCall: // calljithelper if (method_ea_level == 0) { switch(inst->asJitHelperCallInst()->getJitHelperId()) { case InitializeArray: @@ -413,13 +413,17 @@ case ReadThisState: case LockedCompareAndExchange: case AddValueProfileValue: + case ArrayCopyDirect: + case ArrayCopyReverse: + case ArrayCopyFast: + break; default: assert(0); } } break; - + */ case Op_TauStInd: // stind { type=inst->getSrc(0)->getType(); @@ -502,7 +506,7 @@ } break; - case Op_IntrinsicCall: // callintr + case Op_JitHelperCall: // calljithelper if (!inst->getDst()->isNull()) { assert(0); } @@ -884,26 +888,6 @@ } break; - case Op_IntrinsicCall: // callintr - n=inst->getNumSrcOperands(); - switch(inst->asIntrinsicCallInst()->getIntrinsicId()) { - case ArrayCopyDirect: - case ArrayCopyReverse: - cgnode = findCnGNode_op(inst->getSrc(4)->getId()); - assert(cgnode!=NULL); - cgn_src = findCnGNode_op(inst->getSrc(2)->getId()); - assert(cgn_src!=NULL); - addEdge(cgnode,cgn_src,ET_DEFER,inst); - if (verboseLog) { - Log::out() << "need to add edge from "<< cgnode->cngNodeId << " - " << cgnode->opndId << " to " - << cgn_src->cngNodeId << " - " << cgn_src->opndId << std::endl; - } - break; - default: - assert(0); - } - break; - case Op_StVar: // stvar type = inst->getDst()->getType(); if (type->isObject()) { @@ -1239,19 +1223,6 @@ } return; } - if (inst->getOpcode() == Op_IntrinsicCall) { //callintr - uint32 iid = inst->asIntrinsicCallInst()->getIntrinsicId(); - switch(iid) { - case ArrayCopyDirect: - case ArrayCopyReverse: - if (verboseLog) { - Log::out() << "++++ addEdge: callintr" << std::endl; - } - return; - default: - assert(0); - } - } if (inst->getOpcode() == Op_LdFieldAddr) { FieldDesc* fd=inst->asFieldAccessInst()->getFieldDesc(); if (fd->getParentType()->isSystemString()&&strcmp(fd->getName(),"value")==0) { @@ -2689,8 +2660,6 @@ fd->printFullName(os); os << std::endl; } - if (inst->asIntrinsicCallInst()) - os << " IntrinsicCallInst" << std::endl; if (inst->asMethodCallInst()) os << " MethodCallInst" << std::endl; if (inst->asMultiSrcInst()) @@ -3535,7 +3504,7 @@ Opnd** args = NULL; InstFactory& instfactory = irManager.getInstFactory(); Inst* jhcinst = instfactory.makeJitHelperCall( - stThis, ReadThisState, 0, args); + stThis, ReadThisState, NULL, NULL, 0, args); jhcinst->insertAfter(inst_after); newBlock = fg.splitNodeAtInstruction(jhcinst,true, false,instfactory.makeLabel()); fg.addEdge(oldBlock,fg.getUnwindNode()); @@ -3646,7 +3615,7 @@ Opnd* args[1] = {stVal}; InstFactory& instfactory = irManager.getInstFactory(); Inst* jhcinst = instfactory.makeJitHelperCall( - OpndManager::getNullOpnd(), SaveThisState, 1, args); + OpndManager::getNullOpnd(), SaveThisState, NULL, NULL, 1, args); // insert jit helper if (inst_before->getNode()->getFirstInst() == inst_before) { jhcinst->insertAfter(inst_before); @@ -3742,7 +3711,7 @@ os_sc << " null "; os_sc << std::endl; } else { - printCnGNode(stnode,os_sc); os_sc << std::endl; + printCnGNode(stnode,os_sc); os_sc << std::endl; } } } @@ -3842,7 +3811,7 @@ if ((*it)->nodeType == NT_OBJECT && getOutEscaped(*it) == 0 && !((*it)->nInst->getOpcode()==Op_LdRef)) { if ((*it)->nInst->getNode() == NULL && getEscState(*it)==NO_ESCAPE) { - continue; // already scalarized + continue; // already scalarized } ob_ref_type = (*it)->nodeRefType; // object ref type if (ob_ref_type != NR_REF) { @@ -3878,7 +3847,7 @@ os_sc << " null "; os_sc << std::endl; } else { - printCnGNode(stnode,os_sc); os_sc << std::endl; + printCnGNode(stnode,os_sc); os_sc << std::endl; } } @@ -5534,17 +5503,17 @@ EscAnalyzer::CnGNode* EscAnalyzer::getLObj(CnGNode* vval) { - CnGNode* n = vval; - while (n->nodeType != NT_LDOBJ) { - if (n->nodeType!=NT_VARVAL) - return NULL; - if (n->outEdges == NULL) - return NULL; - if ((n->outEdges)->size()!=1) - return NULL; - n=(n->outEdges)->front()->cngNodeTo; - } - return n; + CnGNode* n = vval; + while (n->nodeType != NT_LDOBJ) { + if (n->nodeType!=NT_VARVAL) + return NULL; + if (n->outEdges == NULL) + return NULL; + if ((n->outEdges)->size()!=1) + return NULL; + n=(n->outEdges)->front()->cngNodeTo; + } + return n; } //getLObj(CnGNode* vval) @@ -5869,3 +5838,8 @@ } //namespace Jitrino + + + + + Index: jitrino/src/optimizer/escapeanalyzer.cpp =================================================================== --- jitrino/src/optimizer/escapeanalyzer.cpp (revision 582116) +++ jitrino/src/optimizer/escapeanalyzer.cpp (working copy) @@ -71,7 +71,6 @@ switch (inst->getOpcode()) { case Op_DirectCall: case Op_TauVirtualCall: case Op_IndirectCall: case Op_IndirectMemoryCall: - case Op_IntrinsicCall: case Op_Return: case Op_Throw: case Op_TauStInd: case Op_TauStRef: case Op_TauStField: case Op_TauStElem: case Op_TauStStatic: @@ -107,7 +106,7 @@ if (srcIndex == 0) return false; break; - case Op_DirectCall: case Op_TauVirtualCall: case Op_IntrinsicCall: + case Op_DirectCall: case Op_TauVirtualCall: break; // // return & throw @@ -182,7 +181,7 @@ // // calls should already be marked as escaping // - case Op_DirectCall: case Op_TauVirtualCall: case Op_IntrinsicCall: + case Op_DirectCall: case Op_TauVirtualCall: case Op_IndirectCall: case Op_IndirectMemoryCall: break; // @@ -339,7 +338,6 @@ break; case Op_DirectCall: case Op_TauVirtualCall: - case Op_IntrinsicCall: { if (isRefOrPtrType(inst->getDst())) { // this instruction creates a free ptr/ref Index: jitrino/src/optimizer/FastArrayFilling.cpp =================================================================== --- jitrino/src/optimizer/FastArrayFilling.cpp (revision 582116) +++ jitrino/src/optimizer/FastArrayFilling.cpp (working copy) @@ -362,7 +362,7 @@ // insert the helper. // this helper should be expanded in the code generator phase Inst* initInst = irManager.getInstFactory().makeJitHelperCall( - OpndManager::getNullOpnd(), FillArrayWithConst, 4, args); + OpndManager::getNullOpnd(), FillArrayWithConst, NULL, NULL, 4, args); prepNode->appendInst(initInst); fg.addEdge(prepNode, outEdge->getTargetNode()); @@ -372,3 +372,6 @@ } + + + Index: jitrino/src/optimizer/hashvaluenumberer.cpp =================================================================== --- jitrino/src/optimizer/hashvaluenumberer.cpp (revision 582116) +++ jitrino/src/optimizer/hashvaluenumberer.cpp (working copy) @@ -198,8 +198,6 @@ Inst* caseIndirectMemoryCall(CallInst* inst) { return caseDefault(inst); } - Inst* caseIntrinsicCall(IntrinsicCallInst* inst) { return caseDefault(inst); } - Inst* caseJitHelperCall(JitHelperCallInst* inst) {return caseDefault(inst);} Inst* caseVMHelperCall(VMHelperCallInst* inst) {return caseDefault(inst);} @@ -2312,7 +2310,7 @@ case Op_TauCheckDivOpnds: break; case Op_DirectCall: case Op_TauVirtualCall: case Op_IndirectCall: - case Op_IndirectMemoryCall: case Op_IntrinsicCall: case Op_InitType: + case Op_IndirectMemoryCall: case Op_JitHelperCall: case Op_InitType: break; case Op_TauMonitorExit: break; Index: jitrino/src/optimizer/inliner.cpp =================================================================== --- jitrino/src/optimizer/inliner.cpp (revision 582116) +++ jitrino/src/optimizer/inliner.cpp (working copy) @@ -133,7 +133,7 @@ _inlineSkipMethodTable->add_method_record("java/lang/Integer", "numberOfLeadingZeros", "(I)I", des, false); _inlineSkipMethodTable->add_method_record("java/lang/Integer", "numberOfTrailingZeros", "(I)I", des, false); _inlineSkipMethodTable->add_method_record("java/lang/Long", "numberOfLeadingZeros", "(J)I", des, false); - _inlineSkipMethodTable->add_method_record("java/lang/Long", "numberOfTrailingZeros", "(J)I", des, false); + _inlineSkipMethodTable->add_method_record("java/lang/Long", "numberOfTrailingZeros", "(J)I", des, false); #endif } } Index: jitrino/src/optimizer/Inst.cpp =================================================================== --- jitrino/src/optimizer/Inst.cpp (revision 582116) +++ jitrino/src/optimizer/Inst.cpp (working copy) @@ -440,26 +440,6 @@ } } -void IntrinsicCallInst::handlePrintEscape(::std::ostream& os, char code) const { - switch(code) { - case 'd': - switch(intrinsicId) { - case CharArrayCopy: - os << "charArrayCopy"; break; - case ArrayCopyDirect: - os << "ArrayCopyDirect"; break; - case ArrayCopyReverse: - os << "ArrayCopyReverse"; break; - default: - assert(0); break; - } - break; - default: - Inst::handlePrintEscape(os, code); - break; - } -} - void JitHelperCallInst::handlePrintEscape(::std::ostream& os, char code) const { switch(code) { case 'd': @@ -476,6 +456,13 @@ os << "AddValueProfileValue"; break; case FillArrayWithConst: os << "FillArrayWithConst"; break; + case ArrayCopyDirect: + os << "ArrayCopyDirect"; break; + case ArrayCopyReverse: + os << "ArrayCopyReverse"; break; + case ArrayCopyFast: + os << "ArrayCopyFast"; break; + default: assert(0); break; } @@ -618,9 +605,6 @@ void accept(CatchLabelInst* inst) { clone = instFactory.makeClone(inst, opndManager, renameTable); } - void accept(IntrinsicCallInst* inst) { - clone = instFactory.makeClone(inst, opndManager, renameTable); - } void accept(JitHelperCallInst* inst) { clone = instFactory.makeClone(inst, opndManager, renameTable); } @@ -1038,25 +1022,6 @@ return newInst; } -IntrinsicCallInst* -InstFactory::makeClone(IntrinsicCallInst* inst, - OpndManager& opndManager, - OpndRenameTable& table) { - uint32 nsrcs = inst->getNumSrcOperands(); - Opnd** newArgs = new (memManager) Opnd*[nsrcs]; - for (uint32 i=0; igetSrc(i)); - IntrinsicCallInst *newInst = makeIntrinsicCallInst(inst->getOpcode(), - inst->getModifier(), - inst->getType(), - table.duplicate(opndManager, inst->getDst()), - nsrcs, - newArgs, - inst->getIntrinsicId()); - newInst->setPersistentInstructionId(inst->getPersistentInstructionId()); - return newInst; -} - JitHelperCallInst* InstFactory::makeClone(JitHelperCallInst* inst, OpndManager& opndManager, @@ -1572,19 +1537,6 @@ return inst; } -IntrinsicCallInst* -InstFactory::makeIntrinsicCallInst(Opcode op, Modifier mod, - Type::Tag type, - Opnd* dst, - uint32 nArgs, - Opnd** args_, - IntrinsicCallId id) { - IntrinsicCallInst * inst = - new (memManager) IntrinsicCallInst(op, mod, type, dst, nArgs, args_, id); - inst->id = numInsts++; - return inst; -} - JitHelperCallInst* InstFactory::makeJitHelperCallInst(Opcode op, Modifier mod, @@ -2020,26 +1972,20 @@ } Inst* -InstFactory::makeIntrinsicCall(Opnd* dst, - IntrinsicCallId id, - Opnd* tauNullChecked, - Opnd* tauTypesChecked, - uint32 numArgs, - Opnd** args) { - assert(tauNullChecked->getType()->tag == Type::Tau); - assert(tauTypesChecked->getType()->tag == Type::Tau); +InstFactory::makeJitHelperCall(Opnd* dst, JitHelperCallId id, Opnd* tauNullChecked, Opnd* tauTypesChecked, uint32 numArgs, Opnd** args) { Type::Tag returnType = dst->isNull()? Type::Void : dst->getType()->tag; - args = copyOpnds(tauNullChecked, tauTypesChecked, args, numArgs); - return makeIntrinsicCallInst(Op_IntrinsicCall, Modifier(Exception_Sometimes), + if (id == ArrayCopyDirect || id == ArrayCopyReverse || id == ArrayCopyFast) + { + args = copyOpnds(tauNullChecked, tauTypesChecked, args, numArgs); + return makeJitHelperCallInst(Op_JitHelperCall, Modifier(Exception_Sometimes), returnType, dst, numArgs+2, args, id); -} - -Inst* -InstFactory::makeJitHelperCall(Opnd* dst, JitHelperCallId id, uint32 numArgs, Opnd** args) { - Type::Tag returnType = dst->isNull()? Type::Void : dst->getType()->tag; - args = copyOpnds(args, numArgs); - return makeJitHelperCallInst(Op_JitHelperCall, Modifier(Exception_Sometimes), + } + else + { + args = copyOpnds(args, numArgs); + return makeJitHelperCallInst(Op_JitHelperCall, Modifier(Exception_Sometimes), returnType, dst, numArgs, args, id); + } } Inst* @@ -2687,7 +2633,6 @@ case Op_TauVirtualCall: return caseTauVirtualCall(inst->asMethodCallInst()); case Op_IndirectCall: return caseIndirectCall(inst->asCallInst()); case Op_IndirectMemoryCall: return caseIndirectMemoryCall(inst->asCallInst()); - case Op_IntrinsicCall: return caseIntrinsicCall(inst->asIntrinsicCallInst()); case Op_JitHelperCall: return caseJitHelperCall(inst->asJitHelperCallInst()); case Op_VMHelperCall: return caseVMHelperCall(inst->asVMHelperCallInst()); case Op_Return: return caseReturn(inst); Index: jitrino/src/optimizer/Inst.h =================================================================== --- jitrino/src/optimizer/Inst.h (revision 582116) +++ jitrino/src/optimizer/Inst.h (working copy) @@ -69,7 +69,6 @@ class TokenInst; class LinkingExcInst; class CallInst; -class IntrinsicCallInst; class JitHelperCallInst; class VMHelperCallInst; class PhiInst; @@ -89,7 +88,6 @@ virtual void accept(CatchLabelInst*) = 0; virtual void accept(ConstInst*) = 0; virtual void accept(DispatchLabelInst*) = 0; - virtual void accept(IntrinsicCallInst*) = 0; virtual void accept(JitHelperCallInst*) = 0; virtual void accept(VMHelperCallInst*) = 0; virtual void accept(FieldAccessInst*) = 0; @@ -199,9 +197,6 @@ FieldAccessInst* asFieldAccessInst() const { if (isFieldAccess()) return (FieldAccessInst*)this; else return NULL; } - IntrinsicCallInst* asIntrinsicCallInst() const { - if (isIntrinsicCallInst()) return (IntrinsicCallInst*) this; else return NULL; - } JitHelperCallInst* asJitHelperCallInst() const { if (isJitHelperCallInst()) return (JitHelperCallInst*) this; else return NULL; } @@ -256,7 +251,6 @@ virtual bool isConst() const { return false; }; virtual bool isDispatchLabel() const { return false; }; virtual bool isFieldAccess() const { return false; }; - virtual bool isIntrinsicCallInst() const { return false; }; virtual bool isJitHelperCallInst() const { return false; }; virtual bool isVMHelperCallInst() const { return false; }; virtual bool isMethodCall() const { return false; }; @@ -939,40 +933,6 @@ Opnd** args; }; -// intrinsic calls -class IntrinsicCallInst : public Inst { -public: - void visit(InstFormatVisitor& visitor) {visitor.accept(this);} - bool isIntrinsicCallInst() const { return true; } - IntrinsicCallId getIntrinsicId() const {return intrinsicId;} -private: - virtual void handlePrintEscape(::std::ostream&, char code) const; - friend class InstFactory; - IntrinsicCallInst(Opcode op, Modifier mod, - Type::Tag type, - Opnd* dst, - uint32 nArgs, - Opnd** args_, - IntrinsicCallId intr) : Inst(op, mod, type, dst, nArgs), - intrinsicId(intr) { - args = NULL; - switch (nArgs) { - default: args = args_ + MAX_INST_SRCS; - case 2: srcs[1] = args_[1]; - case 1: srcs[0] = args_[0]; - case 0: break; - } - } - Opnd* getSrcExtended(uint32 srcIndex) const { - return args[srcIndex - MAX_INST_SRCS]; - } - void setSrcExtended(uint32 srcIndex, Opnd* src) { - args[srcIndex - MAX_INST_SRCS] = src; - } - Opnd** args; - IntrinsicCallId intrinsicId; -}; - // JIT helper calls class JitHelperCallInst : public Inst { public: @@ -1133,10 +1093,7 @@ Opnd *tauNullCheckedFirstArg, Opnd *tauTypesChecked, uint32 numArgs, Opnd** args); - Inst* makeIntrinsicCall(Opnd* dst, IntrinsicCallId id, - Opnd* tauNullChecked, Opnd* tauTypesChecked, - uint32 numArgs, Opnd** args); - Inst* makeJitHelperCall(Opnd* dst, JitHelperCallId id, uint32 numArgs, Opnd** args); + Inst* makeJitHelperCall(Opnd* dst, JitHelperCallId id, Opnd* tauNullChecked, Opnd* tauTypesChecked, uint32 numArgs, Opnd** args); Inst* makeVMHelperCall(Opnd* dst, CompilationInterface::RuntimeHelperId id, uint32 numArgs, Opnd** args); @@ -1347,7 +1304,6 @@ MethodInst* makeClone(MethodInst*, OpndManager&, OpndRenameTable&); MethodCallInst* makeClone(MethodCallInst*, OpndManager&, OpndRenameTable&); CallInst* makeClone(CallInst*, OpndManager&, OpndRenameTable&); - IntrinsicCallInst* makeClone(IntrinsicCallInst*, OpndManager&, OpndRenameTable&); JitHelperCallInst* makeClone(JitHelperCallInst*, OpndManager&, OpndRenameTable&); VMHelperCallInst* makeClone(VMHelperCallInst*, OpndManager&, OpndRenameTable&); PhiInst* makeClone(PhiInst*, OpndManager&, OpndRenameTable&); @@ -1499,13 +1455,6 @@ Opnd* ptr, uint32 nArgs, Opnd** args); - IntrinsicCallInst* makeIntrinsicCallInst(Opcode op, - Modifier mod, - Type::Tag, - Opnd* dst, - uint32 nArgs, - Opnd** args_, - IntrinsicCallId id); JitHelperCallInst* makeJitHelperCallInst(Opcode op, Modifier mod, Type::Tag, @@ -1654,9 +1603,6 @@ caseIndirectMemoryCall(CallInst* inst)=0;//{return caseDefault(inst);} virtual Inst* - caseIntrinsicCall(IntrinsicCallInst* inst)=0;//{return caseDefault(inst);} - - virtual Inst* caseJitHelperCall(JitHelperCallInst* inst)=0;//{return caseDefault(inst);} virtual Inst* @@ -2040,3 +1986,8 @@ #endif // _INST_H_ + + + + + Index: jitrino/src/optimizer/IRBuilder.cpp =================================================================== --- jitrino/src/optimizer/IRBuilder.cpp (revision 582116) +++ jitrino/src/optimizer/IRBuilder.cpp (working copy) @@ -931,7 +931,7 @@ } if (!dst) { dst = createOpnd(dstType); - Inst* inst = instFactory->makeConv(ovfMod, toType, dst, src); + Inst* inst = instFactory->makeConv(ovfMod, toType, dst, src); appendInst(inst); } insertHash(hashcode, src->getId(), dst->getInst()); @@ -983,7 +983,7 @@ } if (!dst) { dst = createOpnd(dstType); - Inst* inst = instFactory->makeConvZE(ovfMod, toType, dst, src); + Inst* inst = instFactory->makeConvZE(ovfMod, toType, dst, src); appendInst(inst); } insertHash(hashcode, src->getId(), dst->getInst()); @@ -1398,44 +1398,30 @@ } Opnd* -IRBuilder::genIntrinsicCall(IntrinsicCallId intrinsicId, +IRBuilder::genJitHelperCall(JitHelperCallId helperId, Type* returnType, - Opnd* tauNullCheckedRefArgs, - Opnd* tauTypesChecked, uint32 numArgs, Opnd* args[]) { for (uint32 i=0; igetType()); - Opnd *tauTypesCheckedDst = genTauHasType(args[2], args[2]->getType()); - tauTypesChecked = genTauAnd(tauTypesCheckedSrc, - tauTypesCheckedDst); - } else { - tauTypesChecked = propagateCopy(tauTypesChecked); - } - - appendInst(instFactory->makeIntrinsicCall(dst, intrinsicId, - tauNullCheckedRefArgs, - tauTypesChecked, - numArgs, args)); + appendInst(instFactory->makeJitHelperCall(dst, helperId, NULL, NULL, numArgs, args)); return dst; } Opnd* IRBuilder::genJitHelperCall(JitHelperCallId helperId, Type* returnType, + Opnd* tauNullCheckedRefArgs, + Opnd* tauTypesChecked, uint32 numArgs, Opnd* args[]) { for (uint32 i=0; imakeJitHelperCall(dst, helperId, numArgs, args)); + appendInst(instFactory->makeJitHelperCall(dst, helperId, tauNullCheckedRefArgs, tauTypesChecked, numArgs, args)); return dst; } Index: jitrino/src/optimizer/IRBuilder.h =================================================================== --- jitrino/src/optimizer/IRBuilder.h (revision 582116) +++ jitrino/src/optimizer/IRBuilder.h (working copy) @@ -156,18 +156,16 @@ uint32 numArgs, Opnd* args[]); - Opnd* genIntrinsicCall( - IntrinsicCallId intrinsicId, //TR - Type* returnType, - Opnd* tauNullCheckedRefArgs, // 0 for unsafe - Opnd* tauTypesChecked, // 0 to let IRBuilder find it - uint32 numArgs, - Opnd* args[]); - Opnd* genJitHelperCall(JitHelperCallId helperId, Type* returnType, uint32 numArgs, Opnd* args[]); + Opnd* genJitHelperCall(JitHelperCallId helperId, + Type* returnType, + Opnd* tauNullCheckedRefArgs, + Opnd* tauTypesChecked, + uint32 numArgs, + Opnd* args[]); Opnd* genVMHelperCall(CompilationInterface::RuntimeHelperId helperId, Type* returnType, Index: jitrino/src/optimizer/lazyexceptionopt.cpp =================================================================== --- jitrino/src/optimizer/lazyexceptionopt.cpp (revision 582116) +++ jitrino/src/optimizer/lazyexceptionopt.cpp (working copy) @@ -892,7 +892,6 @@ } #endif return methodCallHasSideEffect(inst); - case Op_IntrinsicCall: case Op_JitHelperCall: case Op_VMHelperCall: return true; @@ -1106,3 +1105,8 @@ } //namespace Jitrino + + + + + Index: jitrino/src/optimizer/memoryopt.cpp =================================================================== --- jitrino/src/optimizer/memoryopt.cpp (revision 582116) +++ jitrino/src/optimizer/memoryopt.cpp (working copy) @@ -618,42 +618,6 @@ thePass->effectAnyGlobal(n, i); } break; - case Op_IntrinsicCall: - { - IntrinsicCallInst *calli = i->asIntrinsicCallInst(); - IntrinsicCallId callId = calli->getIntrinsicId(); - switch (callId) { - case CharArrayCopy: - case ArrayCopyDirect: - case ArrayCopyReverse: - { - assert(calli->getNumSrcOperands() == 7); -#ifndef NDEBUG - Opnd *tauNullChecked = calli->getSrc(0); - assert(tauNullChecked->getType()->tag == Type::Tau); - Opnd *tauTypesChecked = calli->getSrc(1); - assert(tauTypesChecked->getType()->tag == Type::Tau); -#endif - Opnd *srcarray = calli->getSrc(2); - Opnd *srcoffset = calli->getSrc(3); - Opnd *dstarray = calli->getSrc(4); - Opnd *dstoffset = calli->getSrc(5); - Opnd *length = calli->getSrc(6); - - // effectXXXArrayElements actually does not depends on offset parameter - // so we do not need any special managing for the case of reverse copying - thePass->effectReadArrayLength(n, i, srcarray); - thePass->effectReadArrayElements(n, i, srcarray, srcoffset, length); - thePass->effectReadArrayLength(n, i, dstarray); - thePass->effectWriteArrayElements(n, i, dstarray, dstoffset, length); - } - break; - default: - assert(0); - break; - } - } - break; case Op_VMHelperCall: break; case Op_JitHelperCall: @@ -666,6 +630,9 @@ case ReadThisState: case LockedCompareAndExchange: case AddValueProfileValue: + case ArrayCopyDirect: + case ArrayCopyReverse: + case ArrayCopyFast: break; default: assert(0); Index: jitrino/src/optimizer/Opcode.cpp =================================================================== --- jitrino/src/optimizer/Opcode.cpp (revision 582116) +++ jitrino/src/optimizer/Opcode.cpp (working copy) @@ -106,7 +106,6 @@ { Op_TauVirtualCall, true, MB::Call, MK::Exception, "callvirt", "callvrt [%2.%d](%a) ((%0,%1)) -) %l %b", }, { Op_IndirectCall, true, MB::Call, MK::Exception, "calli", "calli [%0](%a) ((%1,%2)) -) %l", }, { Op_IndirectMemoryCall, true, MB::Call, MK::Exception, "callimem", "callimem [%0](%a) ((%1,%2)) -) %l", }, - { Op_IntrinsicCall, true, MB::Call, MK::Exception, "callintr", "callintr %d(%p) ((%0,%1)) -) %l", }, { Op_JitHelperCall, true, MB::Call, MK::Exception, "callhelper", "callhelper %d(%s) -) %l", }, { Op_VMHelperCall, true, MB::Call, MK::Exception, "callvmhelper", "callvmhelper %d(%s) -) %l %b", }, { Op_Return, true, MB::ControlFlow, MK::None, "return", "return %s", }, Index: jitrino/src/optimizer/Opcode.h =================================================================== --- jitrino/src/optimizer/Opcode.h (revision 582116) +++ jitrino/src/optimizer/Opcode.h (working copy) @@ -263,19 +263,16 @@ NewModifier2_BitsConsumed = 2 }; -enum IntrinsicCallId { - CharArrayCopy, - ArrayCopyDirect, - ArrayCopyReverse -}; - enum JitHelperCallId { InitializeArray, FillArrayWithConst, SaveThisState, //todo: replace with GetTLS + offset sequence ReadThisState, //todo: replace with GetTLS + offset sequence LockedCompareAndExchange, - AddValueProfileValue + AddValueProfileValue, + ArrayCopyDirect, + ArrayCopyReverse, + ArrayCopyFast }; enum Opcode { @@ -311,7 +308,6 @@ Op_TauVirtualCall, Op_IndirectCall, Op_IndirectMemoryCall, - Op_IntrinsicCall, Op_JitHelperCall, // call to a jit helper routine Op_VMHelperCall, // call to a vm (runtime) helper routine Op_Return, Index: jitrino/src/optimizer/simplifier.h =================================================================== --- jitrino/src/optimizer/simplifier.h (revision 582116) +++ jitrino/src/optimizer/simplifier.h (working copy) @@ -488,8 +488,6 @@ Inst* caseIndirectMemoryCall(CallInst* inst); - Inst* caseIntrinsicCall(IntrinsicCallInst* inst) {return caseDefault(inst);} - Inst* caseJitHelperCall(JitHelperCallInst* inst) {return caseDefault(inst);} Inst* caseVMHelperCall(VMHelperCallInst* inst) {return caseDefault(inst);} @@ -536,9 +534,9 @@ inst->getDst()->getType(), inst->getToken(), inst->getEnclosingMethod()); - if (opnd != NULL) - return opnd->getInst(); - return inst; + if (opnd != NULL) + return opnd->getInst(); + return inst; } Inst* caseLdVar(Inst* inst) {return caseDefault(inst);} Index: jitrino/src/optimizer/simplifytaus.cpp =================================================================== --- jitrino/src/optimizer/simplifytaus.cpp (revision 582116) +++ jitrino/src/optimizer/simplifytaus.cpp (working copy) @@ -477,7 +477,6 @@ case Op_TauVirtualCall: case Op_IndirectCall: case Op_IndirectMemoryCall: - case Op_IntrinsicCall: if (handleCalls) { return genTauSafe(); } else { Index: jitrino/src/translator/java/JavaByteCodeTranslator.cpp =================================================================== --- jitrino/src/translator/java/JavaByteCodeTranslator.cpp (revision 582116) +++ jitrino/src/translator/java/JavaByteCodeTranslator.cpp (working copy) @@ -1641,24 +1641,13 @@ returnType = typeManager.getNullObjectType(); } // - // Try some optimizations for System::arraycopy(...), Min, Max, Abs... + // Try some optimizations for Min, Max, Abs... // - if (!compilationInterface.needWriteBarriers() //genArrayCopyRepMove is not ready to work in WB mode - && translationFlags.genArrayCopyRepMove == true - && genArrayCopyRepMove(methodDesc,numArgs,srcOpnds)) { - return; - } else if (translationFlags.genArrayCopy == true && - genArrayCopy(methodDesc,numArgs,srcOpnds)) { - return; - } else if (translationFlags.genCharArrayCopy == true && - genCharArrayCopy(methodDesc,numArgs,srcOpnds,returnType)) { - return; - } else if (translationFlags.genMinMaxAbs == true && + if (translationFlags.genMinMaxAbs == true && genMinMax(methodDesc,numArgs,srcOpnds,returnType)) { return; - } else { + } else genInvokeStatic(methodDesc,numArgs,srcOpnds,returnType); - } } void @@ -2346,579 +2335,6 @@ } bool -JavaByteCodeTranslator::methodIsArraycopy(MethodDesc * methodDesc) { - - return (strcmp(methodDesc->getName(),"arraycopy") == 0 && - strcmp(methodDesc->getParentType()->getName(),"java/lang/System") ==0); -} - -bool -JavaByteCodeTranslator::arraycopyOptimizable(MethodDesc * methodDesc, - uint32 numArgs, - Opnd ** srcOpnds) { - - // - // an ArrayStoreException is thrown and the destination is not modified: - // - // - The src argument refers to an object that is not an array. - // - The dest argument refers to an object that is not an array. - // - The src argument and dest argument refer to arrays whose component types are different primitive types. - // - The src argument refers to an array with a primitive component type and the dest argument - // refers to an array with a reference component type. - // - The src argument refers to an array with a reference component type and the dest argument - // refers to an array with a primitive component type. - // - assert(numArgs == 5); - Opnd * src = srcOpnds[0]; - Type * srcType = src->getType(); - Opnd * dst = srcOpnds[2]; - Type * dstType = dst->getType(); - assert(srcType->isObject() && - srcOpnds[1]->getType()->isInt4() && // 1 - srcPos - dstType->isObject() && - srcOpnds[3]->getType()->isInt4() && // 3 - dstPos - srcOpnds[4]->getType()->isInt4()); // 4 - length - - bool throwsASE = false; - bool srcIsArray = srcType->isArray() && !srcType->isUnresolvedType(); - bool dstIsArray = dstType->isArray() && !dstType->isUnresolvedType(); - ArrayType* srcAsArrayType = srcType->asArrayType(); - ArrayType* dstAsArrayType = dstType->asArrayType(); - bool srcIsArrOfPrimitive = srcIsArray && VMInterface::isArrayOfPrimitiveElements(srcAsArrayType->getVMTypeHandle()); - bool dstIsArrOfPrimitive = dstIsArray && VMInterface::isArrayOfPrimitiveElements(dstAsArrayType->getVMTypeHandle()); - if ( !(srcIsArray && dstIsArray) ) { - throwsASE = true; - } else if ( srcIsArrOfPrimitive ) { - if( !dstIsArrOfPrimitive || srcType != dstType ) - throwsASE = true; - } else if( dstIsArrOfPrimitive ) { - throwsASE = true; - } else { // the both are of objects - // Here is some inaccuracy. If src is a subclass of dst there is no ASE for sure. - // If it is not, we should check the assignability of each element being copied. - // To avoid this we just reject the inlining of System::arraycopy call in this case. - NamedType* srcElemType = srcAsArrayType->getElementType(); - NamedType* dstElemType = dstAsArrayType->getElementType(); - throwsASE = srcElemType->getVMTypeHandle() != dstElemType->getVMTypeHandle(); - } - if ( throwsASE ) - return false; - else - return true; -} - -bool -JavaByteCodeTranslator::genArrayCopyRepMove(MethodDesc * methodDesc, - uint32 numArgs, - Opnd ** srcOpnds) { - - if( !methodIsArraycopy(methodDesc) || - !arraycopyOptimizable(methodDesc,numArgs,srcOpnds) ) - { - // reject the inlining of System::arraycopy call - return false; - } - - if (Log::isEnabled()) { - Log::out() << "XXX array copy into 'rep move': "; - methodDesc->printFullName(Log::out()); - Log::out() << ::std::endl; - } - - assert(numArgs == 5); - Opnd * src = srcOpnds[0]; - Opnd * srcPos = srcOpnds[1]; - Type * srcPosType = srcPos->getType(); - Opnd * dst = srcOpnds[2]; - Opnd * dstPos = srcOpnds[3]; - Type * dstPosType = dstPos->getType(); - Opnd * len = srcOpnds[4]; - - // - // Generate exception condition checks: - // chknull src - // chknull dst - // cmpbr srcPos < 0, boundsException - // cmpbr dstPos < 0, boundsException - // cmpbr len < 0, boundsException - // srcEnd = add srcPos, len - // srcLen = src.length - // cmpbr srcEnd > srcLen, boundsException - // dstEnd = add dstPos, len - // dstLen = dst.length - // cmpbr dstEnd > dstLen, boundsException - // Skip trivial: - // cmpbr (src == dst) && (dstPos == srcPos), Exit - // Choose a direction: - // cmpbr (dstPos > srcPos) && (src == dst), Reverse - // - // Intrinsic calls will be codeselected into rep move instruction. - // Direct: - // IntrinsicCall id=ArrayCopyDirect - // goto Exit - // Reverse: - // srcPos = srcPos + len - 1 - // dstPos = dstPos + len - 1 - // IntrinsicCall id=ArrayCopyReverse - // goto Exit - // - // boundsException: - // chkbounds -1, src - // Exit: - // - Opnd *tauSrcNullChecked = irBuilder.genTauCheckNull(src); - Opnd *tauDstNullChecked = irBuilder.genTauCheckNull(dst); - Opnd *tauNullCheckedRefArgs = irBuilder.genTauAnd(tauSrcNullChecked,tauDstNullChecked); - - LabelInst * reverseCopying = irBuilder.createLabel(); - LabelInst * boundsException = irBuilder.createLabel(); - LabelInst * Exit = irBuilder.createLabel(); - - Type * intType = typeManager.getInt32Type(); - Type::Tag intTag = intType->tag; - Type * voidType = typeManager.getVoidType(); - - newFallthroughBlock(); - Opnd * zero = irBuilder.genLdConstant((int32)0); - Opnd * minusone = irBuilder.genLdConstant((int32)-1); - - irBuilder.genBranch(intTag,Cmp_GT,boundsException,zero,srcPos); - - newFallthroughBlock(); - irBuilder.genBranch(intTag,Cmp_GT,boundsException,zero,dstPos); - - newFallthroughBlock(); - irBuilder.genBranch(intTag,Cmp_GT,boundsException,zero,len); - - Modifier mod = Modifier(Overflow_None)|Modifier(Exception_Never)|Modifier(Strict_No); - - newFallthroughBlock(); - Opnd * srcLen = irBuilder.genArrayLen(intType,intTag,src); - Opnd * srcEnd = irBuilder.genAdd(intType,mod,srcPos,len); - irBuilder.genBranch(intTag,Cmp_GT,boundsException,srcEnd,srcLen); - - newFallthroughBlock(); - Opnd * dstEnd = irBuilder.genAdd(intType,mod,dstPos,len); - Opnd * dstLen = irBuilder.genArrayLen(intType,intTag,dst); - irBuilder.genBranch(intTag,Cmp_GT,boundsException,dstEnd,dstLen); - - newFallthroughBlock(); - - // The case of same arrays and same positions - Opnd * diff = irBuilder.genCmp3(intType,intTag,Cmp_GT,dstPos,srcPos); - Opnd * sameArrays = irBuilder.genCmp(intType,Type::IntPtr,Cmp_EQ,src,dst); - Opnd * zeroDiff = irBuilder.genCmp(intType,intTag,Cmp_EQ,diff,zero); - Opnd * nothingToCopy = irBuilder.genAnd(intType,sameArrays,zeroDiff); - irBuilder.genBranch(intTag,Cmp_GT,Exit,nothingToCopy,zero); - - newFallthroughBlock(); - - Opnd* tauTypesChecked = irBuilder.genTauSafe(); - - // Choosing direction - Opnd * dstIsGreater = irBuilder.genCmp(intType,intTag,Cmp_GT,diff,zero); - Opnd * reverseCopy = irBuilder.genAnd(intType,sameArrays,dstIsGreater); - irBuilder.genBranch(intTag,Cmp_GT,reverseCopying,reverseCopy,zero); - - newFallthroughBlock(); - - { // Direct Copying - irBuilder.genIntrinsicCall(ArrayCopyDirect,voidType, - tauNullCheckedRefArgs, - tauTypesChecked, - numArgs,srcOpnds); - irBuilder.genJump(Exit); - } // End of Direct Copying - - irBuilder.genLabel(reverseCopying); - cfgBuilder.genBlockAfterCurrent(reverseCopying); - { // Reverse Copying - - Opnd* lastSrcIdx = irBuilder.genAdd(srcPosType,mod,srcEnd,minusone); - Opnd* lastDstIdx = irBuilder.genAdd(dstPosType,mod,dstEnd,minusone); - - Opnd** reverseArgs = new (memManager) Opnd*[numArgs]; - reverseArgs[0] = srcOpnds[0]; // src - reverseArgs[1] = lastSrcIdx; // srcPos+len-1 - reverseArgs[2] = srcOpnds[2]; // dst - reverseArgs[3] = lastDstIdx; // dstPos+len-1 - reverseArgs[4] = srcOpnds[4]; // len - // copy - irBuilder.genIntrinsicCall(ArrayCopyReverse,voidType, - tauNullCheckedRefArgs, - tauTypesChecked, - numArgs,reverseArgs); - irBuilder.genJump(Exit); - } // End of Reverse Copying - - irBuilder.genLabel(boundsException); - cfgBuilder.genBlockAfterCurrent(boundsException); - irBuilder.genTauCheckBounds(src,minusone,tauSrcNullChecked); - - irBuilder.genLabel(Exit); - cfgBuilder.genBlockAfterCurrent(Exit); - - return true; -} - -bool -JavaByteCodeTranslator::genArrayCopy(MethodDesc * methodDesc, - uint32 numArgs, - Opnd ** srcOpnds) { - - if( !methodIsArraycopy(methodDesc) || - !arraycopyOptimizable(methodDesc,numArgs,srcOpnds) ) - { - // reject the inlining of System::arraycopy call - return false; - } - - if (Log::isEnabled()) { - Log::out() << "XXX array copy: "; methodDesc->printFullName(Log::out()); Log::out() << ::std::endl; - } - - assert(numArgs == 5); - Opnd * src = srcOpnds[0]; - Type * srcType = src->getType(); - Opnd * srcPos = srcOpnds[1]; - Type * srcPosType = srcPos->getType(); - Opnd * dst = srcOpnds[2]; - Type * dstType = dst->getType(); - Opnd * dstPos = srcOpnds[3]; - Type * dstPosType = dstPos->getType(); - Opnd * len = srcOpnds[4]; - - // - // Generate exception condition checks: - // chknull src - // chknull dst - // cmpbr srcPos < 0, boundsException - // cmpbr dstPos < 0, boundsException - // cmpbr len < 0, boundsException - // srcEnd = add srcPos, len - // srcLen = src.length - // cmpbr srcEnd > srcLen, boundsException - // dstEnd = add dstPos, len - // dstLen = dst.length - // cmpbr dstEnd > dstLen, boundsException - // - // diff = Cmp3(dstPos,srcPos) - // // 1 if dstPos > srcPos - // // 0 if dstPos == srcPos - // // -1 if dstPos < srcPos - // if (src == dst && diff == 0) // nothing to do - // goto L1: - // - // if (diff > 0) - // goto reverseCopying: - // - // indexSrc = srcPos - // indexDst = dstPos - // increment = 1 - // copyDirectLoopHeader: - // if (indexSrc == srcEnd) - // goto L1: - // dst[indexDst] = src[indexSrc] - // indexSrc += increment - // indexDst += increment - // goto copyDirectLoopHeader: - // - // reverseCopying: - // indexSrc = srcPos + len - 1 - // indexDst = dstPos + len - 1 - // decrement = 1 - // copyReverseLoopHeader: - // if (indexSrc < srcPos) - // goto L1: - // dst[indexDst] = src[indexSrc] - // indexSrc -= decrement - // indexDst -= decrement - // goto copyReverseLoopHeader: - // - // boundsException: - // chkbounds -1, src - // L1: - // - Opnd *tauSrcNullChecked = irBuilder.genTauCheckNull(src); - Opnd *tauDstNullChecked = irBuilder.genTauCheckNull(dst); - - LabelInst * reverseCopying = irBuilder.createLabel(); - LabelInst * boundsException = irBuilder.createLabel(); - LabelInst * L1 = irBuilder.createLabel(); - Type * intType = typeManager.getInt32Type(); - - newFallthroughBlock(); - Opnd * zero = irBuilder.genLdConstant((int32)0); - Opnd * one = irBuilder.genLdConstant((int32)1); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,zero,srcPos); - - newFallthroughBlock(); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,zero,dstPos); - - newFallthroughBlock(); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,zero,len); - - Modifier mod = Modifier(Overflow_None)|Modifier(Exception_Never)|Modifier(Strict_No); - - newFallthroughBlock(); - Opnd * srcLen = irBuilder.genArrayLen(intType,Type::Int32,src); - Opnd * srcEnd = irBuilder.genAdd(intType,mod,srcPos,len); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,srcEnd,srcLen); - - newFallthroughBlock(); - Opnd * dstEnd = irBuilder.genAdd(intType,mod,dstPos,len); - Opnd * dstLen = irBuilder.genArrayLen(intType,Type::Int32,dst); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,dstEnd,dstLen); - - newFallthroughBlock(); - - // The case of same arrays and same positions - Opnd * diff = irBuilder.genCmp3(intType,Type::Int32,Cmp_GT,dstPos,srcPos); - Opnd * sameArrays = irBuilder.genCmp(intType,Type::IntPtr,Cmp_EQ,src,dst); - Opnd * zeroDiff = irBuilder.genCmp(intType,Type::Int32,Cmp_EQ,diff,zero); - Opnd * nothingToCopy = irBuilder.genAnd(intType,sameArrays,zeroDiff); - irBuilder.genBranch(Type::Int32,Cmp_GT,L1,nothingToCopy,zero); - - newFallthroughBlock(); - - // Choosing direction - - irBuilder.genBranch(Type::Int32,Cmp_GT,reverseCopying,diff,zero); - - newFallthroughBlock(); - - { //Direct Copying - - // indexes for using inside the loop - VarOpnd* srcPosVar = irBuilder.genVarDef(srcPosType, false); - VarOpnd* dstPosVar = irBuilder.genVarDef(dstPosType, false); - - irBuilder.genStVar(srcPosVar, srcPos); - irBuilder.genStVar(dstPosVar, dstPos); - - Opnd* srcPosOpnd = NULL; - Opnd* dstPosOpnd = NULL; - - // Loop Header - LabelInst * loopHead = irBuilder.createLabel(); - irBuilder.genLabel(loopHead); - cfgBuilder.genBlockAfterCurrent(loopHead); - - // loop exit condition (srcIndex = srcStartIndex + len) - srcPosOpnd = irBuilder.genLdVar(srcPosType,srcPosVar); - irBuilder.genBranch(Type::Int32,Cmp_EQ,L1,srcPosOpnd,srcEnd); - - newFallthroughBlock(); - // array bounds have been checked directly - // the types have been checked above so tauAddressInRange is tauSafe - Opnd *tauSrcAddressInRange = irBuilder.genTauSafe(); - Opnd *tauDstAddressInRange = irBuilder.genTauSafe(); - Opnd *tauDstBaseTypeChecked = irBuilder.genTauSafe(); - - // load indexes - srcPosOpnd = irBuilder.genLdVar(srcPosType,srcPosVar); - dstPosOpnd = irBuilder.genLdVar(dstPosType,dstPosVar); - - Type* srcElemType = srcType->asArrayType()->getElementType(); - Type* dstElemType = dstType->asArrayType()->getElementType(); - - // copy element - // (Checks are performed before the loop) - Opnd* elem = irBuilder.genLdElem(srcElemType,src,srcPosOpnd, - tauSrcNullChecked, tauSrcAddressInRange); - irBuilder.genStElem(dstElemType,dst,dstPosOpnd,elem, - tauDstNullChecked, tauDstBaseTypeChecked, tauDstAddressInRange); - - // increment indexes - srcPosOpnd = irBuilder.genAdd(srcPosType,mod,srcPosOpnd,one); - dstPosOpnd = irBuilder.genAdd(dstPosType,mod,dstPosOpnd,one); - - // store indexes - irBuilder.genStVar(srcPosVar, srcPosOpnd); - irBuilder.genStVar(dstPosVar, dstPosOpnd); - - // back edge - irBuilder.genPseudoThrow(); - irBuilder.genJump(loopHead); - - } // End of Direct Copying - - { //Reverse Copying - irBuilder.genLabel(reverseCopying); - cfgBuilder.genBlockAfterCurrent(reverseCopying); - - // indexes for using inside the loop - VarOpnd* srcPosVar = irBuilder.genVarDef(srcPosType, false); - VarOpnd* dstPosVar = irBuilder.genVarDef(dstPosType, false); - - Opnd* lastSrcIdx = irBuilder.genSub(srcPosType,mod,srcEnd,one); - Opnd* lastDstIdx = irBuilder.genSub(dstPosType,mod,dstEnd,one); - - irBuilder.genStVar(srcPosVar, lastSrcIdx); - irBuilder.genStVar(dstPosVar, lastDstIdx); - - Opnd* srcPosOpnd = NULL; - Opnd* dstPosOpnd = NULL; - - // Loop Header - LabelInst * loopHead = irBuilder.createLabel(); - irBuilder.genLabel(loopHead); - cfgBuilder.genBlockAfterCurrent(loopHead); - - // loop exit condition (srcIndex < srcPos) - srcPosOpnd = irBuilder.genLdVar(srcPosType,srcPosVar); - irBuilder.genBranch(Type::Int32,Cmp_GT,L1,srcPos,srcPosOpnd); - - newFallthroughBlock(); - // array bounds have been checked directly - // the types have been checked above so tauAddressInRange is tauSafe - Opnd *tauSrcAddressInRange = irBuilder.genTauSafe(); - Opnd *tauDstAddressInRange = irBuilder.genTauSafe(); - Opnd *tauDstBaseTypeChecked = irBuilder.genTauSafe(); - - // load indexes - srcPosOpnd = irBuilder.genLdVar(srcPosType,srcPosVar); - dstPosOpnd = irBuilder.genLdVar(dstPosType,dstPosVar); - - Type* srcElemType = srcType->asArrayType()->getElementType(); - Type* dstElemType = dstType->asArrayType()->getElementType(); - - // copy element - // (Checks are performed before the loop) - Opnd* elem = irBuilder.genLdElem(srcElemType,src,srcPosOpnd, - tauSrcNullChecked, tauSrcAddressInRange); - irBuilder.genStElem(dstElemType,dst,dstPosOpnd,elem, - tauDstNullChecked, tauDstBaseTypeChecked, tauDstAddressInRange); - - // decrement indexes - srcPosOpnd = irBuilder.genSub(srcPosType,mod,srcPosOpnd,one); - dstPosOpnd = irBuilder.genSub(dstPosType,mod,dstPosOpnd,one); - - // store indexes - irBuilder.genStVar(srcPosVar, srcPosOpnd); - irBuilder.genStVar(dstPosVar, dstPosOpnd); - - // back edge - irBuilder.genPseudoThrow(); - irBuilder.genJump(loopHead); - - } // End of Reverse Copying - - - irBuilder.genLabel(boundsException); - cfgBuilder.genBlockAfterCurrent(boundsException); - Opnd * minusOne = irBuilder.genLdConstant((int32)-1); - irBuilder.genTauCheckBounds(src,minusOne,tauSrcNullChecked); - - irBuilder.genLabel(L1); - cfgBuilder.genBlockAfterCurrent(L1); - - return true; -} - -bool -JavaByteCodeTranslator::genCharArrayCopy(MethodDesc * methodDesc, - uint32 numArgs, - Opnd ** srcOpnds, - Type * returnType) { - // - // Check if method is java/lang/System.arraycopy - // (Object src, int srcPos, Object dst, int dstPos, int len) - // - if (strcmp(methodDesc->getName(),"arraycopy") != 0 || - strcmp(methodDesc->getParentType()->getName(),"java/lang/System") !=0) - return false; - // - // Check if arguments are arrays of characters - // - assert(numArgs == 5); - Opnd * src = srcOpnds[0]; - Opnd * srcPos = srcOpnds[1]; - Opnd * dst = srcOpnds[2]; - Opnd * dstPos = srcOpnds[3]; - Opnd * len = srcOpnds[4]; - assert(src->getType()->isObject() && - srcPos->getType()->isInt4() && - dst->getType()->isObject() && - dstPos->getType()->isInt4() && - len->getType()->isInt4()); - Type * srcType = src->getType(); - Type * dstType = dst->getType(); - if (!(srcType->isArray() && dstType->isArray() && - ((ArrayType *)srcType)->getElementType()->isChar() && - ((ArrayType *)dstType)->getElementType()->isChar())) - return false; - - if (Log::isEnabled()) { - Log::out() << "XXX char array copy: "; methodDesc->printFullName(Log::out()); Log::out() << ::std::endl; - } - // - // Generate exception condition checks: - // chknull src - // chknull dst - // cmpbr srcPos < 0, boundsException - // cmpbr dstPos < 0, boundsException - // cmpbr len < 0, boundsException - // srcEnd = add srcPos, len - // srcLen = src.length - // cmpbr srcEnd > srcLen, boundsException - // dstEnd = add dstPos, len - // dstLen = dst.length - // cmpbr dstEnd > dstLen, boundsException - // callintr charArrayCopy(src,srcPos,dst,dstPos,len) - // goto L1: - // boundsException: - // chkbounds -1, src - // L1: - // - Opnd *tauSrcNullChecked = irBuilder.genTauCheckNull(src); - Opnd *tauDstNullChecked = irBuilder.genTauCheckNull(dst); - - LabelInst * boundsException = irBuilder.createLabel(); - LabelInst * L1 = irBuilder.createLabel(); - Type * intType = typeManager.getInt32Type(); - - newFallthroughBlock(); - Opnd * zero = irBuilder.genLdConstant((int32)0); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,zero,srcPos); - - newFallthroughBlock(); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,zero,dstPos); - - newFallthroughBlock(); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,zero,len); - - newFallthroughBlock(); - Opnd * srcLen = irBuilder.genArrayLen(intType,Type::Int32,src); - Opnd * srcEnd = irBuilder.genAdd(intType,Modifier(Overflow_None)|Modifier(Exception_Never)|Modifier(Strict_No),srcPos,len); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,srcEnd,srcLen); - - newFallthroughBlock(); - Opnd * dstEnd = irBuilder.genAdd(intType,Modifier(Overflow_None)|Modifier(Exception_Never)|Modifier(Strict_No),dstPos,len); - Opnd * dstLen = irBuilder.genArrayLen(intType,Type::Int32,dst); - irBuilder.genBranch(Type::Int32,Cmp_GT,boundsException,dstEnd,dstLen); - - newFallthroughBlock(); - Opnd *tauNullCheckedRefArgs = - irBuilder.genTauAnd(tauSrcNullChecked, tauDstNullChecked); - Opnd *tauTypesChecked = 0; - irBuilder.genIntrinsicCall(CharArrayCopy,returnType, - tauNullCheckedRefArgs, - tauTypesChecked, - numArgs,srcOpnds); - irBuilder.genJump(L1); - - irBuilder.genLabel(boundsException); - cfgBuilder.genBlockAfterCurrent(boundsException); - Opnd * minusOne = irBuilder.genLdConstant((int32)-1); - irBuilder.genTauCheckBounds(src,minusOne,tauSrcNullChecked); - - irBuilder.genLabel(L1); - cfgBuilder.genBlockAfterCurrent(L1); - return true; -} - -bool JavaByteCodeTranslator::genMinMax(MethodDesc * methodDesc, uint32 numArgs, Opnd ** srcOpnds, @@ -3321,12 +2737,12 @@ if (loadConst) { ConstInst::ConstValue v; #ifdef _EM64T_ - v.i8 = theConst; + v.i8 = theConst; #else - v.i4 = theConst; + v.i4 = theConst; #endif Opnd* res = irBuilder.genLdConstant(typeManager.getUIntPtrType(), v); - + if (resType->isPtr()) { res = irBuilder.genConv(resType, resType->tag, mod, res); } Index: jitrino/src/translator/java/JavaByteCodeTranslator.h =================================================================== --- jitrino/src/translator/java/JavaByteCodeTranslator.h (revision 582116) +++ jitrino/src/translator/java/JavaByteCodeTranslator.h (working copy) @@ -271,12 +271,14 @@ bool genVMMagic(const char* mname, uint32 numArgs,Opnd ** srcOpnds,Type * returnType); bool genVMHelper(const char* mname, uint32 numArgs,Opnd ** srcOpnds,Type * returnType); - bool methodIsArraycopy(MethodDesc * methodDesc); - bool arraycopyOptimizable(MethodDesc * methodDesc, uint32 numArgs, Opnd ** srcOpnds); - bool genCharArrayCopy(MethodDesc * methodDesc,uint32 numArgs,Opnd ** srcOpnds, Type * returnType); - bool genArrayCopyRepMove(MethodDesc * methodDesc,uint32 numArgs,Opnd ** srcOpnds); - bool genArrayCopy(MethodDesc * methodDesc,uint32 numArgs,Opnd ** srcOpnds); bool genMinMax(MethodDesc * methodDesc,uint32 numArgs,Opnd ** srcOpnds, Type * returnType); void newFallthroughBlock();