-- Hive CLI log: hive> > > set hive.auto.convert.join=false; hive> set hive.enforce.bucketing=true; hive> set hive.exec.dynamic.partition.mode = nonstrict; hive> set hive.support.concurrency=true; hive> set hive.txn.manager = org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; hive> > > drop table if exists mergedelta_txt_1; OK Time taken: 0.194 seconds hive> drop table if exists mergedelta_txt_2; OK Time taken: 0.188 seconds hive> > CREATE TABLE mergedelta_txt_1 ( > id_str varchar(12), time_key int, value bigint) > PARTITIONED BY (date_key int) > ROW FORMAT DELIMITED > STORED AS TEXTFILE; OK Time taken: 0.093 seconds hive> > CREATE TABLE mergedelta_txt_2 ( > id_str varchar(12), time_key int, value bigint) > PARTITIONED BY (date_key int) > ROW FORMAT DELIMITED > STORED AS TEXTFILE; OK Time taken: 0.073 seconds hive> > INSERT INTO TABLE mergedelta_txt_1 > partition(date_key=20170103) > VALUES > ("AB94LIENR0",46700,12345676836978), > ("AB94LIENR1",46825,12345676836978), > ("AB94LIENS0",46709,12345676836978), > ("AB94LIENS1",46834,12345676836978), > ("AB94LIENT0",46709,12345676836978), > ("AB94LIENT1",46834,12345676836978), > ("AB94LIENU0",46718,12345676836978), > ("AB94LIENU1",46844,12345676836978), > ("AB94LIENV0",46719,12345676836978), > ("AB94LIENV1",46844,12345676836978), > ("AB94LIENW0",46728,12345676836978), > ("AB94LIENW1",46854,12345676836978), > ("AB94LIENX0",46728,12345676836978), > ("AB94LIENX1",46854,12345676836978), > ("AB94LIENY0",46737,12345676836978), > ("AB94LIENY1",46863,12345676836978), > ("AB94LIENZ0",46738,12345676836978), > ("AB94LIENZ1",46863,12345676836978), > ("AB94LIERA0",47176,12345676836982), > ("AB94LIERA1",47302,12345676836982); Query ID = hadoop_20181027050914_5021f377-f803-4b10-8c16-9aebc0ba31ea Total jobs = 3 Launching Job 1 out of 3 Status: Running (Executing on YARN cluster with App id application_1539912995488_0197) ---------------------------------------------------------------------------------------------- VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED ---------------------------------------------------------------------------------------------- Map 1 .......... container SUCCEEDED 1 1 0 0 0 0 ---------------------------------------------------------------------------------------------- VERTICES: 01/01 [==========================>>] 100% ELAPSED TIME: 6.51 s ---------------------------------------------------------------------------------------------- Stage-5 is selected by condition resolver. Stage-4 is filtered out by condition resolver. Stage-6 is filtered out by condition resolver. Moving data to directory hdfs://ip-10-0-5-209.ec2.internal:8020/user/hive/warehouse/mergedelta_txt_1/date_key=20170103/.hive-staging_hive_2018-10-27_05-09-14_899_2860325757268068377-1/-ext-10000 Loading data to table default.mergedelta_txt_1 partition (date_key=20170103) OK Time taken: 7.63 seconds hive> > INSERT INTO TABLE mergedelta_txt_2 > partition(date_key=20170103) > VALUES > ("AB94LIENT1",46834,12345676836978), > ("AB94LIENU0",46718,12345676836978), > ("AB94LIENU1",46844,12345676836978), > ("AB94LIENV0",46719,12345676836978), > ("AB94LIENV1",46844,12345676836978), > ("AB94LIENW0",46728,12345676836978), > ("AB94LIENW1",46854,12345676836978), > ("AB94LIENX0",46728,12345676836978), > ("AB94LIENX1",46854,12345676836978), > ("AB94LIENY0",46737,12345676836978), > ("AB94LIENY1",46863,12345676836978), > ("AB94LIENZ0",46738,12345676836978), > ("AB94LIENZ1",46863,12345676836978), > ("AB94LIERA0",47176,12345676836982), > ("AB94LIERA1",47302,12345676836982), > ("AB94LIERA2",47418,12345676836982), > ("AB94LIERB0",47176,12345676836982), > ("AB94LIERB1",47302,12345676836982), > ("AB94LIERB2",47418,12345676836982), > ("AB94LIERC0",47185,12345676836982); Query ID = hadoop_20181027050922_1fe2e9bc-7362-41fc-8b22-06d8b2e6014c Total jobs = 3 Launching Job 1 out of 3 Status: Running (Executing on YARN cluster with App id application_1539912995488_0197) ---------------------------------------------------------------------------------------------- VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED ---------------------------------------------------------------------------------------------- Map 1 .......... container SUCCEEDED 1 1 0 0 0 0 ---------------------------------------------------------------------------------------------- VERTICES: 01/01 [==========================>>] 100% ELAPSED TIME: 0.55 s ---------------------------------------------------------------------------------------------- Stage-5 is selected by condition resolver. Stage-4 is filtered out by condition resolver. Stage-6 is filtered out by condition resolver. Moving data to directory hdfs://ip-10-0-5-209.ec2.internal:8020/user/hive/warehouse/mergedelta_txt_2/date_key=20170103/.hive-staging_hive_2018-10-27_05-09-22_543_617955932933865389-1/-ext-10000 Loading data to table default.mergedelta_txt_2 partition (date_key=20170103) OK Time taken: 1.675 seconds hive> > DROP TABLE IF EXISTS mergebase_1; OK Time taken: 0.193 seconds hive> CREATE TABLE mergebase_1 ( > id_str varchar(12) , time_key int , value bigint) > PARTITIONED BY (date_key int) > CLUSTERED BY (id_str,time_key) INTO 32 BUCKETS > STORED AS ORC > TBLPROPERTIES ( > 'orc.compress'='SNAPPY', > 'pk_columns'='id_str,date_key,time_key', > 'NO_AUTO_COMPACTION'='true', > 'transactional'='true'); OK Time taken: 0.104 seconds hive> > MERGE INTO mergebase_1 AS base > USING (SELECT * > FROM ( > SELECT id_str ,time_key ,value, date_key, rank() OVER (PARTITION BY id_str,date_key,time_key ORDER BY id_str,date_key,time_key) AS rk > FROM mergedelta_txt_1 > DISTRIBUTE BY date_key > ) rankedtbl > WHERE rankedtbl.rk=1 > ) AS delta > ON delta.id_str=base.id_str AND delta.date_key=base.date_key AND delta.time_key=base.time_key > WHEN MATCHED THEN UPDATE SET value=delta.value > WHEN NOT MATCHED THEN INSERT VALUES ( delta.id_str , delta.time_key , delta.value, delta.date_key); Query ID = hadoop_20181027050924_ddd80537-99fc-4502-9180-26885297e3da Total jobs = 3 Launching Job 1 out of 3 Status: Running (Executing on YARN cluster with App id application_1539912995488_0197) ---------------------------------------------------------------------------------------------- VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED ---------------------------------------------------------------------------------------------- Map 1 .......... container SUCCEEDED 1 1 0 0 0 0 Map 8 container SUCCEEDED 0 0 0 0 0 0 Reducer 2 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 3 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 4 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 5 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 6 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 7 ...... container SUCCEEDED 2 2 0 0 0 0 ---------------------------------------------------------------------------------------------- VERTICES: 07/08 [==========================>>] 100% ELAPSED TIME: 5.28 s ---------------------------------------------------------------------------------------------- Stage-8 is selected by condition resolver. Stage-7 is filtered out by condition resolver. Stage-9 is filtered out by condition resolver. Moving data to directory hdfs://ip-10-0-5-209.ec2.internal:8020/tmp/hive/hadoop/7f2a0304-1349-4767-beb8-e285efc930ff/_tmp_space.db/merge_tmp_table/.hive-staging_hive_2018-10-27_05-09-24_548_9024797063892304386-1/-ext-10004 Loading data to table default.mergebase_1 partition (date_key=null) Time taken to load dynamic partitions: 0.026 seconds Time taken for adding to write entity : 0.0 seconds Loading data to table default.merge_tmp_table Loading data to table default.mergebase_1 partition (date_key=null) Loaded : 1/1 partitions. Time taken to load dynamic partitions: 0.125 seconds Time taken for adding to write entity : 0.0 seconds OK Time taken: 6.836 seconds hive> > MERGE INTO mergebase_1 AS base > USING (SELECT * > FROM ( > SELECT id_str ,time_key ,value, date_key, rank() OVER (PARTITION BY id_str,date_key,time_key ORDER BY id_str,date_key,time_key) AS rk > FROM mergedelta_txt_2 > DISTRIBUTE BY date_key > ) rankedtbl > WHERE rankedtbl.rk=1 > ) AS delta > ON delta.id_str=base.id_str AND delta.date_key=base.date_key AND delta.time_key=base.time_key > WHEN MATCHED THEN UPDATE SET value=delta.value > WHEN NOT MATCHED THEN INSERT VALUES ( delta.id_str , delta.time_key , delta.value, delta.date_key); Query ID = hadoop_20181027050931_992c381e-e8ea-465b-bd75-514f4d1921d0 Total jobs = 3 Launching Job 1 out of 3 Status: Running (Executing on YARN cluster with App id application_1539912995488_0197) ---------------------------------------------------------------------------------------------- VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED ---------------------------------------------------------------------------------------------- Map 1 .......... container SUCCEEDED 1 1 0 0 0 0 Map 8 .......... container SUCCEEDED 32 32 0 0 0 0 Reducer 2 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 3 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 4 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 5 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 6 ...... container SUCCEEDED 2 2 0 0 0 0 Reducer 7 ...... container SUCCEEDED 2 2 0 0 0 0 ---------------------------------------------------------------------------------------------- VERTICES: 08/08 [==========================>>] 100% ELAPSED TIME: 2.17 s ---------------------------------------------------------------------------------------------- Stage-8 is selected by condition resolver. Stage-7 is filtered out by condition resolver. Stage-9 is filtered out by condition resolver. Moving data to directory hdfs://ip-10-0-5-209.ec2.internal:8020/tmp/hive/hadoop/7f2a0304-1349-4767-beb8-e285efc930ff/_tmp_space.db/merge_tmp_table/.hive-staging_hive_2018-10-27_05-09-31_386_5629585231872337681-1/-ext-10004 Loading data to table default.mergebase_1 partition (date_key=null) Loaded : 1/1 partitions. Time taken to load dynamic partitions: 0.154 seconds Time taken for adding to write entity : 0.0 seconds Loading data to table default.merge_tmp_table Loading data to table default.mergebase_1 partition (date_key=null) Loaded : 1/1 partitions. Time taken to load dynamic partitions: 0.137 seconds Time taken for adding to write entity : 0.0 seconds OK Time taken: 3.891 seconds hive> > select count(*) from mergebase_1; Query ID = hadoop_20181027050935_c685be9a-f21b-43ea-977f-6c2a66613a6c Total jobs = 1 Launching Job 1 out of 1 Status: Running (Executing on YARN cluster with App id application_1539912995488_0197) ---------------------------------------------------------------------------------------------- VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED ---------------------------------------------------------------------------------------------- Map 1 ......... container RUNNING 32 31 0 1 4 0 Reducer 2 container INITED 1 0 0 1 0 0 ---------------------------------------------------------------------------------------------- VERTICES: 00/02 [========================>>--] 93% ELAPSED TIME: 0.63 s ---------------------------------------------------------------------------------------------- Status: Failed Vertex failed, vertexName=Map 1, vertexId=vertex_1539912995488_0197_66_00, diagnostics=[Task failed, taskId=task_1539912995488_0197_66_00_000020, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_1539912995488_0197_66_00_000020_0:java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111) at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157) at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83) at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694) at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653) at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145) at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184) ... 14 more Caused by: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203) ... 25 more Caused by: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:511) at org.apache.orc.impl.ReaderImpl.(ReaderImpl.java:378) at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.(ReaderImpl.java:63) at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:90) at org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.(OrcRawRecordMerger.java:491) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getReader(OrcInputFormat.java:1970) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getRecordReader(OrcInputFormat.java:1867) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ... 26 more ], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_1539912995488_0197_66_00_000020_1:java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111) at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157) at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83) at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694) at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653) at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145) at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184) ... 14 more Caused by: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203) ... 25 more Caused by: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:511) at org.apache.orc.impl.ReaderImpl.(ReaderImpl.java:378) at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.(ReaderImpl.java:63) at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:90) at org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.(OrcRawRecordMerger.java:491) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getReader(OrcInputFormat.java:1970) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getRecordReader(OrcInputFormat.java:1867) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ... 26 more ], TaskAttempt 2 failed, info=[Error: Error while running task ( failure ) : attempt_1539912995488_0197_66_00_000020_2:java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111) at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157) at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83) at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694) at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653) at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145) at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184) ... 14 more Caused by: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203) ... 25 more Caused by: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:511) at org.apache.orc.impl.ReaderImpl.(ReaderImpl.java:378) at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.(ReaderImpl.java:63) at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:90) at org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.(OrcRawRecordMerger.java:491) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getReader(OrcInputFormat.java:1970) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getRecordReader(OrcInputFormat.java:1867) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ... 26 more ], TaskAttempt 3 failed, info=[Error: Error while running task ( failure ) : attempt_1539912995488_0197_66_00_000020_3:java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111) at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157) at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83) at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694) at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653) at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145) at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184) ... 14 more Caused by: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203) ... 25 more Caused by: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:511) at org.apache.orc.impl.ReaderImpl.(ReaderImpl.java:378) at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.(ReaderImpl.java:63) at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:90) at org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.(OrcRawRecordMerger.java:491) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getReader(OrcInputFormat.java:1970) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getRecordReader(OrcInputFormat.java:1867) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ... 26 more ]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_1539912995488_0197_66_00 [Map 1] killed/failed due to:OWN_TASK_FAILURE] Vertex killed, vertexName=Reducer 2, vertexId=vertex_1539912995488_0197_66_01, diagnostics=[Vertex received Kill while in RUNNING state., Vertex did not succeed due to OTHER_VERTEX_FAILURE, failedTasks:0 killedTasks:1, Vertex vertex_1539912995488_0197_66_01 [Reducer 2] killed/failed due to:OTHER_VERTEX_FAILURE] DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:1 FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Map 1, vertexId=vertex_1539912995488_0197_66_00, diagnostics=[Task failed, taskId=task_1539912995488_0197_66_00_000020, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_1539912995488_0197_66_00_000020_0:java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111) at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157) at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83) at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694) at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653) at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145) at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184) ... 14 more Caused by: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203) ... 25 more Caused by: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:511) at org.apache.orc.impl.ReaderImpl.(ReaderImpl.java:378) at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.(ReaderImpl.java:63) at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:90) at org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.(OrcRawRecordMerger.java:491) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getReader(OrcInputFormat.java:1970) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getRecordReader(OrcInputFormat.java:1867) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ... 26 more ], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_1539912995488_0197_66_00_000020_1:java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111) at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157) at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83) at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694) at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653) at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145) at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184) ... 14 more Caused by: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203) ... 25 more Caused by: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:511) at org.apache.orc.impl.ReaderImpl.(ReaderImpl.java:378) at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.(ReaderImpl.java:63) at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:90) at org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.(OrcRawRecordMerger.java:491) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getReader(OrcInputFormat.java:1970) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getRecordReader(OrcInputFormat.java:1867) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ... 26 more ], TaskAttempt 2 failed, info=[Error: Error while running task ( failure ) : attempt_1539912995488_0197_66_00_000020_2:java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111) at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157) at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83) at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694) at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653) at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145) at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184) ... 14 more Caused by: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203) ... 25 more Caused by: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:511) at org.apache.orc.impl.ReaderImpl.(ReaderImpl.java:378) at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.(ReaderImpl.java:63) at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:90) at org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.(OrcRawRecordMerger.java:491) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getReader(OrcInputFormat.java:1970) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getRecordReader(OrcInputFormat.java:1867) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ... 26 more ], TaskAttempt 3 failed, info=[Error: Error while running task ( failure ) : attempt_1539912995488_0197_66_00_000020_3:java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73) at org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1840) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61) at org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111) at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157) at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83) at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694) at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653) at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145) at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184) ... 14 more Caused by: java.io.IOException: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203) ... 25 more Caused by: org.apache.orc.FileFormatException: Not a valid ORC file at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:511) at org.apache.orc.impl.ReaderImpl.(ReaderImpl.java:378) at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.(ReaderImpl.java:63) at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:90) at org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger.(OrcRawRecordMerger.java:491) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getReader(OrcInputFormat.java:1970) at org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.getRecordReader(OrcInputFormat.java:1867) at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ... 26 more ]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_1539912995488_0197_66_00 [Map 1] killed/failed due to:OWN_TASK_FAILURE]Vertex killed, vertexName=Reducer 2, vertexId=vertex_1539912995488_0197_66_01, diagnostics=[Vertex received Kill while in RUNNING state., Vertex did not succeed due to OTHER_VERTEX_FAILURE, failedTasks:0 killedTasks:1, Vertex vertex_1539912995488_0197_66_01 [Reducer 2] killed/failed due to:OTHER_VERTEX_FAILURE]DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:1 hive> >