Major Description
Exception
2014-09-04 19:28:24,032 INFO [TezChild] org.apache.hadoop.hive.ql.exec.JoinOperator: 41 Close done 2014-09-04 19:28:24,033 ERROR [TezChild] org.apache.hadoop.hive.ql.exec.tez.TezProcessor: java.lang.RuntimeException: Reduce operator initialization failed at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:254) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:164) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:309) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:180) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:172) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1557) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.call(TezTaskRunner.java:172) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.call(TezTaskRunner.java:167) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.RuntimeException: cannot find field _col69 from [0:_col18, 1:_col23, 2:_col45, 3:_col47, 4:_col49, 5:_col51, 6:_col108, 7:_col109] at org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:410) at org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:147) at org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:55) at org.apache.hadoop.hive.ql.exec.Operator.initEvaluators(Operator.java:938) at org.apache.hadoop.hive.ql.exec.Operator.initEvaluatorsAndReturnStruct(Operator.java:964) at org.apache.hadoop.hive.ql.exec.SelectOperator.initializeOp(SelectOperator.java:63) at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:380) at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:464) at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:420) at org.apache.hadoop.hive.ql.exec.JoinOperator.initializeOp(JoinOperator.java:60) at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:380) at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:223) ... 13 more 2014-09-04 19:28:24,034 INFO [TezChild] org.apache.tez.runtime.task.TezTaskRunner: Encounted an error while executing task: attempt_1406566393272_3882_3_24_000028_0 java.lang.RuntimeException: java.lang.RuntimeException: Reduce operator initialization failed at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:188) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:309) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:180) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:172) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1557) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.call(TezTaskRunner.java:172) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.call(TezTaskRunner.java:167) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.RuntimeException: Reduce operator initialization failed at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:254) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:164) ... 12 more Caused by: java.lang.RuntimeException: cannot find field _col69 from [0:_col18, 1:_col23, 2:_col45, 3:_col47, 4:_col49, 5:_col51, 6:_col108, 7:_col109] at org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:410) at org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.getStructFieldRef(StandardStructObjectInspector.java:147) at org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:55) at org.apache.hadoop.hive.ql.exec.Operator.initEvaluators(Operator.java:938) at org.apache.hadoop.hive.ql.exec.Operator.initEvaluatorsAndReturnStruct(Operator.java:964) at org.apache.hadoop.hive.ql.exec.SelectOperator.initializeOp(SelectOperator.java:63) at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:380) at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:464) at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:420) at org.apache.hadoop.hive.ql.exec.JoinOperator.initializeOp(JoinOperator.java:60) at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:380) at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.init(ReduceRecordProcessor.java:223) ... 13 more
Query
WITH all_sales AS (
SELECT d_year
,i_brand_id
,i_class_id
,i_category_id
,i_manufact_id
,SUM(sales_cnt) AS sales_cnt
,SUM(sales_amt) AS sales_amt
FROM (SELECT d_year
,i_brand_id
,i_class_id
,i_category_id
,i_manufact_id
,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt
,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt
FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk
JOIN date_dim ON d_date_sk=cs_sold_date_sk
LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number
AND cs_item_sk=cr_item_sk)
WHERE i_category='Sports'
UNION ALL
SELECT d_year
,i_brand_id
,i_class_id
,i_category_id
,i_manufact_id
,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt
,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt
FROM store_sales JOIN item ON i_item_sk=ss_item_sk
JOIN date_dim ON d_date_sk=ss_sold_date_sk
LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number
AND ss_item_sk=sr_item_sk)
WHERE i_category='Sports'
UNION ALL
SELECT d_year
,i_brand_id
,i_class_id
,i_category_id
,i_manufact_id
,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt
,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt
FROM web_sales JOIN item ON i_item_sk=ws_item_sk
JOIN date_dim ON d_date_sk=ws_sold_date_sk
LEFT JOIN web_returns ON (ws_order_number=wr_order_number
AND ws_item_sk=wr_item_sk)
WHERE i_category='Sports') sales_detail
GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id)
SELECT prev_yr.d_year AS prev_year
,curr_yr.d_year AS year
,curr_yr.i_brand_id
,curr_yr.i_class_id
,curr_yr.i_category_id
,curr_yr.i_manufact_id
,prev_yr.sales_cnt AS prev_yr_cnt
,curr_yr.sales_cnt AS curr_yr_cnt
,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff
,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff
FROM all_sales curr_yr, all_sales prev_yr
WHERE curr_yr.i_brand_id=prev_yr.i_brand_id
AND curr_yr.i_class_id=prev_yr.i_class_id
AND curr_yr.i_category_id=prev_yr.i_category_id
AND curr_yr.i_manufact_id=prev_yr.i_manufact_id
AND curr_yr.d_year=2002
AND prev_yr.d_year=2002-1
AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9
ORDER BY sales_cnt_diff
limit 100