Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (revision 1214129) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (working copy) @@ -190,6 +190,8 @@ PARTSPEC_DIFFER_FROM_SCHEMA("Partition columns in partition specification are not the same as " + "that defined in the table schema. The names and orders have to be exactly the same."), PARTITION_COLUMN_NON_PRIMITIVE("Partition column must be of primitive type."), + INSERT_INTO_DYNAMICPARTITION_IFNOTEXISTS( + "Dynamic partitions do not support IF NOT EXISTS. Specified partitions with value :"), ; private String mesg; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 1214129) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -1427,7 +1427,7 @@ @init { msgs.push("insert clause"); } @after { msgs.pop(); } : - KW_INSERT KW_OVERWRITE destination -> ^(TOK_DESTINATION destination) + KW_INSERT KW_OVERWRITE destination ifNotExists? -> ^(TOK_DESTINATION destination ifNotExists?) | KW_INSERT KW_INTO KW_TABLE tableOrPartition -> ^(TOK_INSERT_INTO ^(tableOrPartition)) ; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1214129) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -170,6 +170,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.mapred.InputFormat; +import org.antlr.runtime.tree.Tree; /** * Implementation of the semantic analyzer. @@ -696,9 +697,10 @@ * @throws SemanticException */ @SuppressWarnings({"fallthrough", "nls"}) - public void doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) + public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) throws SemanticException { + boolean phase1Result = true; QBParseInfo qbp = qb.getParseInfo(); boolean skipRecursion = false; @@ -864,6 +866,48 @@ ErrorMsg.UNION_NOTIN_SUBQ.getMsg())); } + case HiveParser.TOK_INSERT: + ASTNode destination = (ASTNode) ast.getChild(0); + Tree tab = destination.getChild(0); + // Proceed if AST contains partition & If Not Exists + if (destination.getChildCount() == 2 && + tab.getChildCount() == 2 && + destination.getChild(1).getType() == HiveParser.TOK_IFNOTEXISTS) { + String tableName = tab.getChild(0).getChild(0).getText(); + Tree partitions = tab.getChild(1); + int childCount = partitions.getChildCount(); + HashMap partition = new HashMap(); + for (int i = 0; i < childCount; i++) { + String partitionName = partitions.getChild(i).getChild(0).getText(); + Tree pvalue = partitions.getChild(i).getChild(1); + if (pvalue == null) { + break; + } + String partitionVal = stripQuotes(pvalue.getText()); + partition.put(partitionName, partitionVal); + } + // if it is a dynamic partition throw the exception + if (childCount == partition.size()) { + try { + Table table = db.getTable(tableName); + Partition parMetaData = db.getPartition(table, partition, false); + // Check partition exists if it exists skip the overwrite + if (parMetaData != null) { + phase1Result = false; + skipRecursion = true; + LOG.info("Partition already exists so insert into overwrite " + + "skipped for partition : " + parMetaData.toString()); + break; + } + } catch (HiveException e) { + LOG.info("Error while getting metadata : ", e); + } + } else { + throw new SemanticException(ErrorMsg.INSERT_INTO_DYNAMICPARTITION_IFNOTEXISTS + .getMsg(partition.toString())); + } + } + default: skipRecursion = false; break; @@ -873,12 +917,12 @@ if (!skipRecursion) { // Iterate over the rest of the children int child_count = ast.getChildCount(); - for (int child_pos = 0; child_pos < child_count; ++child_pos) { - + for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) { // Recurse - doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1); + phase1Result = phase1Result && doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1); } } + return phase1Result; } private void getMetaData(QBExpr qbexpr) throws SemanticException { @@ -7268,7 +7312,10 @@ } // continue analyzing from the child ASTNode. - doPhase1(child, qb, initPhase1Ctx()); + if (!doPhase1(child, qb, initPhase1Ctx())) { + // if phase1Result false return + return; + } LOG.info("Completed phase 1 of Semantic Analysis"); Index: ql/src/test/queries/clientnegative/insertover_dynapart_ifnotexists.q =================================================================== --- ql/src/test/queries/clientnegative/insertover_dynapart_ifnotexists.q (revision 0) +++ ql/src/test/queries/clientnegative/insertover_dynapart_ifnotexists.q (revision 0) @@ -0,0 +1,9 @@ +set hive.exec.dynamic.partition=true; + +create table srcpart_dp like srcpart; + +create table destpart_dp like srcpart; + +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_dp partition(ds='2008-04-08', hr=11); + +insert overwrite table destpart_dp partition (ds='2008-04-08', hr) if not exists select key, value, hr from srcpart_dp where ds='2008-04-08'; \ No newline at end of file Index: ql/src/test/queries/clientpositive/insert1_overwrite_partitions.q =================================================================== --- ql/src/test/queries/clientpositive/insert1_overwrite_partitions.q (revision 0) +++ ql/src/test/queries/clientpositive/insert1_overwrite_partitions.q (revision 0) @@ -0,0 +1,35 @@ +CREATE TABLE sourceTable (one string,two string) PARTITIONED BY (ds string,hr string); + +load data local inpath '../data/files/kv1.txt' INTO TABLE sourceTable partition(ds='2011-11-11', hr='11'); + +load data local inpath '../data/files/kv3.txt' INTO TABLE sourceTable partition(ds='2011-11-11', hr='12'); + +CREATE TABLE destinTable (one string,two string) PARTITIONED BY (ds string,hr string); + +EXPLAIN INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5; + +INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5; + +select one,two from destinTable; + +EXPLAIN INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one limit 5; + +INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one limit 5; + +select one,two from destinTable; + +drop table destinTable; + +CREATE TABLE destinTable (one string,two string); + +EXPLAIN INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5; + +INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5; + +drop table destinTable; + +drop table sourceTable; Index: ql/src/test/results/clientnegative/insertover_dynapart_ifnotexists.q.out =================================================================== --- ql/src/test/results/clientnegative/insertover_dynapart_ifnotexists.q.out (revision 0) +++ ql/src/test/results/clientnegative/insertover_dynapart_ifnotexists.q.out (revision 0) @@ -0,0 +1,18 @@ +PREHOOK: query: create table srcpart_dp like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table srcpart_dp like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_dp +PREHOOK: query: create table destpart_dp like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table destpart_dp like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@destpart_dp +PREHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_dp +POSTHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_dp +POSTHOOK: Output: default@srcpart_dp@ds=2008-04-08/hr=11 +FAILED: Error in semantic analysis: Dynamic partitions do not support IF NOT EXISTS. Specified partitions with value : {ds=2008-04-08} Index: ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out =================================================================== --- ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out (revision 0) +++ ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out (revision 0) @@ -0,0 +1,327 @@ +PREHOOK: query: CREATE TABLE sourceTable (one string,two string) PARTITIONED BY (ds string,hr string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE sourceTable (one string,two string) PARTITIONED BY (ds string,hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@sourceTable +PREHOOK: query: load data local inpath '../data/files/kv1.txt' INTO TABLE sourceTable partition(ds='2011-11-11', hr='11') +PREHOOK: type: LOAD +PREHOOK: Output: default@sourcetable +POSTHOOK: query: load data local inpath '../data/files/kv1.txt' INTO TABLE sourceTable partition(ds='2011-11-11', hr='11') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@sourcetable +POSTHOOK: Output: default@sourcetable@ds=2011-11-11/hr=11 +PREHOOK: query: load data local inpath '../data/files/kv3.txt' INTO TABLE sourceTable partition(ds='2011-11-11', hr='12') +PREHOOK: type: LOAD +PREHOOK: Output: default@sourcetable +POSTHOOK: query: load data local inpath '../data/files/kv3.txt' INTO TABLE sourceTable partition(ds='2011-11-11', hr='12') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@sourcetable +POSTHOOK: Output: default@sourcetable@ds=2011-11-11/hr=12 +PREHOOK: query: CREATE TABLE destinTable (one string,two string) PARTITIONED BY (ds string,hr string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE destinTable (one string,two string) PARTITIONED BY (ds string,hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@destinTable +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME sourceTable))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME destinTable) (TOK_PARTSPEC (TOK_PARTVAL ds '2011-11-11') (TOK_PARTVAL hr '11'))) TOK_IFNOTEXISTS) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL one)) (TOK_SELEXPR (TOK_TABLE_OR_COL two))) (TOK_WHERE (AND (= (TOK_TABLE_OR_COL ds) '2011-11-11') (= (TOK_TABLE_OR_COL hr) '11'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL one))) (TOK_LIMIT 5))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + sourcetable + TableScan + alias: sourcetable + Select Operator + expressions: + expr: one + type: string + expr: two + type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.destintable + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 2011-11-11 + hr 11 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.destintable + + Stage: Stage-3 + Stats-Aggr Operator + + +PREHOOK: query: INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@sourcetable@ds=2011-11-11/hr=11 +PREHOOK: Output: default@destintable@ds=2011-11-11/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sourcetable@ds=2011-11-11/hr=11 +POSTHOOK: Output: default@destintable@ds=2011-11-11/hr=11 +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +PREHOOK: query: select one,two from destinTable +PREHOOK: type: QUERY +PREHOOK: Input: default@destintable@ds=2011-11-11/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select one,two from destinTable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@destintable@ds=2011-11-11/hr=11 +#### A masked pattern was here #### +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one limit 5 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME sourceTable))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME destinTable) (TOK_PARTSPEC (TOK_PARTVAL ds '2011-11-11') (TOK_PARTVAL hr '11'))) TOK_IFNOTEXISTS) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL one)) (TOK_SELEXPR (TOK_TABLE_OR_COL two))) (TOK_WHERE (AND (= (TOK_TABLE_OR_COL ds) '2011-11-11') (= (TOK_TABLE_OR_COL hr) '12'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL one))) (TOK_LIMIT 5))) + +STAGE DEPENDENCIES: + +STAGE PLANS: +STAGE PLANS: +PREHOOK: query: INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one limit 5 +PREHOOK: type: QUERY +POSTHOOK: query: INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists +SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='12' order by one limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +PREHOOK: query: select one,two from destinTable +PREHOOK: type: QUERY +PREHOOK: Input: default@destintable@ds=2011-11-11/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select one,two from destinTable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@destintable@ds=2011-11-11/hr=11 +#### A masked pattern was here #### +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +PREHOOK: query: drop table destinTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@destintable +PREHOOK: Output: default@destintable +POSTHOOK: query: drop table destinTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@destintable +POSTHOOK: Output: default@destintable +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +PREHOOK: query: CREATE TABLE destinTable (one string,two string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE destinTable (one string,two string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@destinTable +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME sourceTable))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME destinTable))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL one)) (TOK_SELEXPR (TOK_TABLE_OR_COL two))) (TOK_WHERE (AND (= (TOK_TABLE_OR_COL ds) '2011-11-11') (= (TOK_TABLE_OR_COL hr) '11'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL one))) (TOK_LIMIT 5))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + sourcetable + TableScan + alias: sourcetable + Select Operator + expressions: + expr: one + type: string + expr: two + type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: +#### A masked pattern was here #### + Reduce Output Operator + key expressions: + expr: _col0 + type: string + sort order: + + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.destintable + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.destintable + + Stage: Stage-3 + Stats-Aggr Operator + + +PREHOOK: query: INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@sourcetable@ds=2011-11-11/hr=11 +PREHOOK: Output: default@destintable +POSTHOOK: query: INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sourcetable@ds=2011-11-11/hr=11 +POSTHOOK: Output: default@destintable +POSTHOOK: Lineage: destintable.one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable.two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +PREHOOK: query: drop table destinTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@destintable +PREHOOK: Output: default@destintable +POSTHOOK: query: drop table destinTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@destintable +POSTHOOK: Output: default@destintable +POSTHOOK: Lineage: destintable.one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable.two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +PREHOOK: query: drop table sourceTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@sourcetable +PREHOOK: Output: default@sourcetable +POSTHOOK: query: drop table sourceTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@sourcetable +POSTHOOK: Output: default@sourcetable +POSTHOOK: Lineage: destintable.one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable.two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).one SIMPLE [(sourcetable)sourcetable.FieldSchema(name:one, type:string, comment:null), ] +POSTHOOK: Lineage: destintable PARTITION(ds=2011-11-11,hr=11).two SIMPLE [(sourcetable)sourcetable.FieldSchema(name:two, type:string, comment:null), ]