Details
-
Improvement
-
Status: Resolved
-
Major
-
Resolution: Fixed
-
1.18.0
-
None
-
None
Description
Exception when attempting to parse strings to timestamp using logRegex
$ tree data
data
└── 2021
└── 01
└── 20
└── sample.log.gz
{
"type": "file",
"connection": "file:///",
"config": {},
"workspaces": {
"tmp": {
"location": "/tmp",
"writable": true,
"defaultInputFormat": null,
"allowAccessOutsideWorkspace": false
},
"root": {
"location": "/data",
"writable": false,
"defaultInputFormat": null,
"allowAccessOutsideWorkspace": false
}
},
"formats": {
"log": {
"type": "logRegex",
"regex": "([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) ([^ ]*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\"($| \"[^ ]*\")(.*)",
"extension": "gz",
"maxErrors": 10,
"schema": [
{
"fieldName": "type"
},
{
"fieldName": "time",
"fieldType": "TIMESTAMP",
"format": "yyyy-MM-dd''T''HH:mm:ssZ"
},
{
"fieldName": "elb"
},
{
"fieldName": "client_ip"
},
{
"fieldName": "client_port",
"fieldType": "INT"
},
{
"fieldName": "target_ip"
},
{
"fieldName": "target_port",
"fieldType": "INT"
},
{
"fieldName": "request_processing_time",
"fieldType": "FLOAT8"
},
{
"fieldName": "target_processing_time",
"fieldType": "FLOAT8"
},
{
"fieldName": "response_processing_time",
"fieldType": "FLOAT8"
},
{
"fieldName": "elb_status_code"
},
{
"fieldName": "target_status_code"
},
{
"fieldName": "received_bytes",
"fieldType": "BIGINT"
},
{
"fieldName": "sent_bytes",
"fieldType": "BIGINT"
},
{
"fieldName": "request_verb"
},
{
"fieldName": "request_url"
},
{
"fieldName": "request_proto"
},
{
"fieldName": "user_agent"
},
{
"fieldName": "ssl_cipher"
},
{
"fieldName": "ssl_protocol"
},
{
"fieldName": "target_group_arn"
},
{
"fieldName": "trace_id"
},
{
"fieldName": "domain_name"
},
{
"fieldName": "chosen_cert_arn"
},
{
"fieldName": "matched_rule_priority"
},
{
"fieldName": "request_creation_time"
},
{
"fieldName": "actions_executed"
},
{
"fieldName": "redirect_url"
},
{
"fieldName": "new_field"
}
]
}
},
"enabled": true
}
apache drill> select * from dfs.root.`2021` limit 5;
org.apache.drill.common.exceptions.UserRemoteException: EXECUTION_ERROR ERROR: java.lang.NoSuchMethodException: org.apache.drill.exec.physical.impl.scan.convert.ConvertStringToTimeStamp.<init>(org.apache.drill.exec.vector.accessor.ScalarWriter)File: file:/data/2021/01/20/sample.log.gz Fragment: 0:0[Error Id: a633bc5a-c13b-4ef0-a5fc-a7833fdec754 on 500c5539c3b7:31010] at org.apache.drill.exec.rpc.user.QueryResultHandler.resultArrived(QueryResultHandler.java:125) at org.apache.drill.exec.rpc.user.UserClient.handle(UserClient.java:422) at org.apache.drill.exec.rpc.user.UserClient.handle(UserClient.java:96) at org.apache.drill.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:273) at org.apache.drill.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:243) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:88) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:287) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:312) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:286) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:335) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1294) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:356) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:342) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:911) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.IllegalStateException: java.lang.NoSuchMethodException: org.apache.drill.exec.physical.impl.scan.convert.ConvertStringToTimeStamp.<init>(org.apache.drill.exec.vector.accessor.ScalarWriter) at org.apache.drill.exec.physical.impl.scan.convert.StandardConversions.newInstance(StandardConversions.java:197) at org.apache.drill.exec.physical.impl.scan.convert.StandardConversions.newInstance(StandardConversions.java:188) at org.apache.drill.exec.physical.impl.scan.convert.StandardConversions.converterFor(StandardConversions.java:429) at org.apache.drill.exec.physical.impl.scan.convert.StandardConversions.converterFor(StandardConversions.java:446) at org.apache.drill.exec.store.log.LogBatchReader$ScalarGroupWriter.<init>(LogBatchReader.java:93) at org.apache.drill.exec.store.log.LogBatchReader.bindColumns(LogBatchReader.java:184) at org.apache.drill.exec.store.log.LogBatchReader.open(LogBatchReader.java:148) at org.apache.drill.exec.store.log.LogBatchReader.open(LogBatchReader.java:44) at org.apache.drill.exec.physical.impl.scan.framework.ManagedScanFramework.open(ManagedScanFramework.java:208) at org.apache.drill.exec.physical.impl.scan.file.FileScanFramework.open(FileScanFramework.java:268) at org.apache.drill.exec.physical.impl.scan.framework.ShimBatchReader.open(ShimBatchReader.java:75) at org.apache.drill.exec.physical.impl.scan.ReaderState.open(ReaderState.java:229) at org.apache.drill.exec.physical.impl.scan.ScanOperatorExec.nextAction(ScanOperatorExec.java:273) at org.apache.drill.exec.physical.impl.scan.ScanOperatorExec.next(ScanOperatorExec.java:229) at org.apache.drill.exec.physical.impl.protocol.OperatorDriver.doNext(OperatorDriver.java:201) at org.apache.drill.exec.physical.impl.protocol.OperatorDriver.start(OperatorDriver.java:179) at org.apache.drill.exec.physical.impl.protocol.OperatorDriver.next(OperatorDriver.java:129) at org.apache.drill.exec.physical.impl.protocol.OperatorRecordBatch.next(OperatorRecordBatch.java:149) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:111) at org.apache.drill.exec.record.AbstractUnaryRecordBatch.innerNext(AbstractUnaryRecordBatch.java:59) at org.apache.drill.exec.physical.impl.limit.LimitRecordBatch.innerNext(LimitRecordBatch.java:93) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:170) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:111) at org.apache.drill.exec.record.AbstractUnaryRecordBatch.innerNext(AbstractUnaryRecordBatch.java:59) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:170) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:119) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:111) at org.apache.drill.exec.record.AbstractUnaryRecordBatch.innerNext(AbstractUnaryRecordBatch.java:59) at org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.innerNext(ProjectRecordBatch.java:85) at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:170) at org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:103) at org.apache.drill.exec.physical.impl.ScreenCreator$ScreenRoot.innerNext(ScreenCreator.java:81) at org.apache.drill.exec.physical.impl.BaseRootExec.next(BaseRootExec.java:93) at org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:323) at org.apache.drill.exec.work.fragment.FragmentExecutor$1.run(FragmentExecutor.java:310) at .......(:0) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730) at org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:310) at org.apache.drill.common.SelfCleaningRunnable.run(SelfCleaningRunnable.java:38) at .......(:0) Caused by: java.lang.NoSuchMethodException: org.apache.drill.exec.physical.impl.scan.convert.ConvertStringToTimeStamp.<init>(org.apache.drill.exec.vector.accessor.ScalarWriter) at .......(:0) at org.apache.drill.exec.physical.impl.scan.convert.StandardConversions.newInstance(StandardConversions.java:194) ... 41 common frames omitted
Also see https://apache-drill.slack.com/archives/CG5CH5571/p1612158646012100