[TEZ-3784] Submitting very large DAG throws com.google.protobuf.CodedInputStream exception

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 0.9.0
• Hadoop Flags: Reviewed
• Assignee: Rajesh Balamohan
• Reporter: Rajesh Balamohan
• Created: 03/Jul/17 00:19
• Updated: 22/Aug/17 00:02
• Resolved: 08/Jul/17 03:11

Description

Dag submit failed due to Protocol message was too large.  May be malicious.  Use CodedInputStream.setSizeLimit() to increase the size limit.
        at com.google.protobuf.InvalidProtocolBufferException.sizeLimitExceeded(InvalidProtocolBufferException.java:110)
        at com.google.protobuf.CodedInputStream.refillBuffer(CodedInputStream.java:755)
        at com.google.protobuf.CodedInputStream.isAtEnd(CodedInputStream.java:701)
        at com.google.protobuf.CodedInputStream.readTag(CodedInputStream.java:99)
        at org.apache.tez.dag.api.records.DAGProtos$TezUserPayloadProto.<init>(DAGProtos.java:8282)
        at org.apache.tez.dag.api.records.DAGProtos$TezUserPayloadProto.<init>(DAGProtos.java:8246)
        at org.apache.tez.dag.api.records.DAGProtos$TezUserPayloadProto$1.parsePartialFrom(DAGProtos.java:8334)
        at org.apache.tez.dag.api.records.DAGProtos$TezUserPayloadProto$1.parsePartialFrom(DAGProtos.java:8329)
        at com.google.protobuf.CodedInputStream.readMessage(CodedInputStream.java:309)
        at org.apache.tez.dag.api.records.DAGProtos$TezEntityDescriptorProto.<init>(DAGProtos.java:7520)
        at org.apache.tez.dag.api.records.DAGProtos$TezEntityDescriptorProto.<init>(DAGProtos.java:7462)
        at org.apache.tez.dag.api.records.DAGProtos$TezEntityDescriptorProto$1.parsePartialFrom(DAGProtos.java:7563)
        at org.apache.tez.dag.api.records.DAGProtos$TezEntityDescriptorProto$1.parsePartialFrom(DAGProtos.java:7558)
        at com.google.protobuf.CodedInputStream.readMessage(CodedInputStream.java:309)
        at org.apache.tez.dag.api.records.DAGProtos$RootInputLeafOutputProto.<init>(DAGProtos.java:8854)
        at org.apache.tez.dag.api.records.DAGProtos$RootInputLeafOutputProto.<init>(DAGProtos.java:8796)
        at org.apache.tez.dag.api.records.DAGProtos$RootInputLeafOutputProto$1.parsePartialFrom(DAGProtos.java:8905)
        at org.apache.tez.dag.api.records.DAGProtos$RootInputLeafOutputProto$1.parsePartialFrom(DAGProtos.java:8900)
        at com.google.protobuf.CodedInputStream.readMessage(CodedInputStream.java:309)
        at org.apache.tez.dag.api.records.DAGProtos$VertexPlan.<init>(DAGProtos.java:11012)
        at org.apache.tez.dag.api.records.DAGProtos$VertexPlan.<init>(DAGProtos.java:10893)
        at org.apache.tez.dag.api.records.DAGProtos$VertexPlan$1.parsePartialFrom(DAGProtos.java:11107)
        at org.apache.tez.dag.api.records.DAGProtos$VertexPlan$1.parsePartialFrom(DAGProtos.java:11102)
        at com.google.protobuf.CodedInputStream.readMessage(CodedInputStream.java:309)
        at org.apache.tez.dag.api.records.DAGProtos$DAGPlan.<init>(DAGProtos.java:22507)
        at org.apache.tez.dag.api.records.DAGProtos$DAGPlan.<init>(DAGProtos.java:22449)
        at org.apache.tez.dag.api.records.DAGProtos$DAGPlan$1.parsePartialFrom(DAGProtos.java:22638)
        at org.apache.tez.dag.api.records.DAGProtos$DAGPlan$1.parsePartialFrom(DAGProtos.java:22633)
        at com.google.protobuf.CodedInputStream.readMessage(CodedInputStream.java:309)
        at org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC$SubmitDAGRequestProto.<init>(DAGClientAMProtocolRPC.java:4673)
        at org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC$SubmitDAGRequestProto.<init>(DAGClientAMProtocolRPC.java:4620)
        at org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC$SubmitDAGRequestProto$1.parsePartialFrom(DAGClientAMProtocolRPC.java:4729)
        at org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC$SubmitDAGRequestProto$1.parsePartialFrom(DAGClientAMProtocolRPC.java:4724)
        at com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:200)
        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:217)
        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:223)
        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:49)
        at org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC$SubmitDAGRequestProto.parseFrom(DAGClientAMProtocolRPC.java:4976)
        at org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPBServerImpl.submitDAG(DAGClientAMProtocolBlockingPBServerImpl.java:169)
        at org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC$DAGClientAMProtocol$2.callBlockingMethod(DAGClientAMProtocolRPC.java:7636)
        at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:640)
        at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:982)
        at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2313)
        at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2309)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:422)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724)

Appears that protobuf used in tez has a default size of 64 MB.