diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index aba2946f7a..15e0f8f549 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -540,6 +540,7 @@ minillaplocal.query.files=\ dynpart_sort_opt_vectorization.q,\ dynpart_sort_optimization.q,\ dynpart_sort_optimization_acid.q,\ + dynpart_sort_opt_bucketing.q,\ enforce_constraint_notnull.q,\ escape1.q,\ escape2.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index c6798e9933..0df645804a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -59,7 +59,7 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; +import org.apache.hadoop.hive.ql.parse.type.*; import org.apache.hadoop.hive.ql.plan.ColStatistics; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -76,6 +76,7 @@ import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.orc.OrcConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -288,16 +289,29 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, List descs = new ArrayList(allRSCols.size()); List colNames = new ArrayList(); String colName; + final List fileSinkSchema = fsOp.getSchema().getSignature(); for (int i = 0; i < allRSCols.size(); i++) { ExprNodeDesc col = allRSCols.get(i); + ExprNodeDesc newColumnExpr = null; colName = col.getExprString(); colNames.add(colName); if (partitionPositions.contains(i) || sortPositions.contains(i)) { - descs.add(new ExprNodeColumnDesc(col.getTypeInfo(), ReduceField.KEY.toString()+"."+colName, null, false)); + newColumnExpr = (new ExprNodeColumnDesc(col.getTypeInfo(), ReduceField.KEY.toString()+"."+colName, null, false)); } else { - descs.add(new ExprNodeColumnDesc(col.getTypeInfo(), ReduceField.VALUE.toString()+"."+colName, null, false)); + newColumnExpr = (new ExprNodeColumnDesc(col.getTypeInfo(), ReduceField.VALUE.toString()+"."+colName, null, false)); } + + // make sure column type matches with expected types in FS op + if(i < fileSinkSchema.size()) { + final ColumnInfo fsColInfo = fileSinkSchema.get(i); + if (!newColumnExpr.getTypeInfo().equals(fsColInfo.getType())) { + newColumnExpr = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor() + .createConversionCast(newColumnExpr, (PrimitiveTypeInfo) fsColInfo.getType()); + } + } + descs.add(newColumnExpr); } + RowSchema selRS = new RowSchema(fsParent.getSchema()); if (bucketColumns!= null && !bucketColumns.isEmpty()) { descs.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, @@ -407,14 +421,7 @@ private boolean removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) { rsChild.getSchema().getSignature().size()) { return false; } - // if child is select and contains expression which isn't column it shouldn't - // be removed because otherwise we will end up with different types/schema later - // while introducing select for RS - for(ExprNodeDesc expr: rsChild.getColumnExprMap().values()){ - if(!(expr instanceof ExprNodeColumnDesc)){ - return false; - } - } + rsParent.getChildOperators().remove(rsToRemove); rsParent.getChildOperators().add(rsGrandChild); rsGrandChild.getParentOperators().clear(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5fcc367cc9..9b6914008a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6824,7 +6824,7 @@ private Operator genBucketingSortingDest(String dest, Operator input, QB qb, if (updating(dest) || deleting(dest)) { partnCols = getPartitionColsFromBucketColsForUpdateDelete(input, true); } else { - partnCols = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, true); + partnCols = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, false); } } else { if(updating(dest) || deleting(dest)) { @@ -7385,6 +7385,10 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // Add NOT NULL constraint check input = genConstraintsPlan(dest, qb, input); + if (!qb.getIsQuery()) { + input = genConversionSelectOperator(dest, qb, input, tableDescriptor, dpCtx); + } + if (destinationTable.isMaterializedView() && mvRebuildMode == MaterializationRebuildMode.INSERT_OVERWRITE_REBUILD) { // Data organization (DISTRIBUTED, SORTED, CLUSTERED) for materialized view @@ -7510,6 +7514,10 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // Add NOT NULL constraint check input = genConstraintsPlan(dest, qb, input); + if (!qb.getIsQuery()) { + input = genConversionSelectOperator(dest, qb, input, tableDescriptor, dpCtx); + } + if (destinationTable.isMaterializedView() && mvRebuildMode == MaterializationRebuildMode.INSERT_OVERWRITE_REBUILD) { // Data organization (DISTRIBUTED, SORTED, CLUSTERED) for materialized view @@ -7832,9 +7840,6 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) throw new SemanticException("Unknown destination type: " + destType); } - if (!(destType == QBMetaData.DEST_DFS_FILE && qb.getIsQuery())) { - input = genConversionSelectOperator(dest, qb, input, tableDescriptor, dpCtx); - } inputRR = opParseCtx.get(input).getRowResolver(); @@ -8737,7 +8742,7 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, } } - return genConvertCol(dest, qb, tableDesc, input, posns, true); + return genConvertCol(dest, qb, tableDesc, input, posns, false); } private List getSortOrders(Table tab) { diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q index 7cb90d3f2d..5430ebdb48 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q @@ -74,6 +74,7 @@ dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1_n147/e=epart/000008_0; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.stats.autogather=false; +set hive.optimize.sort.dynamic.partition.threshold=1; CREATE TABLE dynpart_sort_opt_bucketing_test (ca_address_sk int, ca_address_id string, ca_street_number string, ca_street_name string, ca_street_type string, ca_suite_number string, ca_city string, ca_county string, ca_state string, @@ -88,6 +89,58 @@ explain INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', '-5', 'single family'); +select * from dynpart_sort_opt_bucketing_test; + +-- with auto stats +set hive.stats.autogather=true; +explain INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family'); +INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family'); +select * from dynpart_sort_opt_bucketing_test; + DROP TABLE dynpart_sort_opt_bucketing_test; +-- test case to test that CAST on bucketing column doesn't prevent sort dynamic partition + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +drop table if exists t1_staging; +create table t1_staging( + a string, + b int, + c int, + d string) + partitioned by (e decimal(18,0)) + clustered by(a) + into 256 buckets STORED AS TEXTFILE; +load data local inpath '../../data/files/sortdp/000000_0' overwrite into table t1_staging partition (e=100); + +drop table t1_n147; +create table t1_n147( + a string, + b decimal(6,0), + c int, + d string) + partitioned by (e decimal(3,0)) + clustered by(a,b) + into 10 buckets STORED AS ORC TBLPROPERTIES ('transactional'='true'); + +set hive.stats.autogather=false; +set hive.optimize.bucketingsorting = true; +explain insert overwrite table t1_n147 partition(e) select a,b,c,d,e from t1_staging; +insert overwrite table t1_n147 partition(e) select a,b,c,d,e from t1_staging; + +with q1 as (select count(*) as cnt from t1_staging), + q2 as (select count(*) as cnt from t1_n147) +select q1.cnt = q2.cnt from q1 join q2; + +drop table t1_staging; +drop table t1_n147; + + + diff --git a/ql/src/test/results/clientpositive/bucket1.q.out b/ql/src/test/results/clientpositive/bucket1.q.out index f5158ad921..6e8965ff79 100644 --- a/ql/src/test/results/clientpositive/bucket1.q.out +++ b/ql/src/test/results/clientpositive/bucket1.q.out @@ -35,17 +35,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: vectorized Path -> Alias: @@ -104,7 +104,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/bucket2.q.out b/ql/src/test/results/clientpositive/bucket2.q.out index ef6d630c97..e21515e03b 100644 --- a/ql/src/test/results/clientpositive/bucket2.q.out +++ b/ql/src/test/results/clientpositive/bucket2.q.out @@ -34,17 +34,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: vectorized Path -> Alias: @@ -103,7 +103,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/bucket3.q.out b/ql/src/test/results/clientpositive/bucket3.q.out index d02c957774..a494c21ee3 100644 --- a/ql/src/test/results/clientpositive/bucket3.q.out +++ b/ql/src/test/results/clientpositive/bucket3.q.out @@ -35,17 +35,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: vectorized Path -> Alias: @@ -104,7 +104,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out index 618b698700..0c7826a7a1 100644 --- a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out +++ b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out @@ -180,19 +180,19 @@ STAGE PLANS: filterExpr: (ds = '1') (type: boolean) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: int), value (type: string) + expressions: CAST( key AS STRING) (type: string), UDFToInteger(value) (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col1) (type: int) + key expressions: _col1 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col1) (type: int) - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string) + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 500 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator - expressions: CAST( VALUE._col0 AS STRING) (type: string), UDFToInteger(VALUE._col1) (type: int) + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/dynpart_sort_opt_bucketing.q.out b/ql/src/test/results/clientpositive/dynpart_sort_opt_bucketing.q.out deleted file mode 100644 index 390230e855..0000000000 --- a/ql/src/test/results/clientpositive/dynpart_sort_opt_bucketing.q.out +++ /dev/null @@ -1,356 +0,0 @@ -PREHOOK: query: drop table if exists t1_staging -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table if exists t1_staging -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table t1_staging( -a string, -b int, -c int, -d string) -partitioned by (e string) -clustered by(a) -sorted by(a desc) -into 256 buckets stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t1_staging -POSTHOOK: query: create table t1_staging( -a string, -b int, -c int, -d string) -partitioned by (e string) -clustered by(a) -sorted by(a desc) -into 256 buckets stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1_staging -PREHOOK: query: load data local inpath '../../data/files/sortdp/000000_0' overwrite into table t1_staging partition (e='epart') -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@t1_staging -POSTHOOK: query: load data local inpath '../../data/files/sortdp/000000_0' overwrite into table t1_staging partition (e='epart') -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@t1_staging -POSTHOOK: Output: default@t1_staging@e=epart -PREHOOK: query: drop table t1_n147 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table t1_n147 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table t1_n147( -a string, -b int, -c int, -d string) -partitioned by (e string) -clustered by(a) -sorted by(a desc) into 10 buckets stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t1_n147 -POSTHOOK: query: create table t1_n147( -a string, -b int, -c int, -d string) -partitioned by (e string) -clustered by(a) -sorted by(a desc) into 10 buckets stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1_n147 -PREHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,'epart' from t1_staging -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_staging -PREHOOK: Input: default@t1_staging@e=epart -PREHOOK: Output: default@t1_n147 -POSTHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,'epart' from t1_staging -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_staging -POSTHOOK: Input: default@t1_staging@e=epart -POSTHOOK: Output: default@t1_n147@e=epart -POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).a SIMPLE [(t1_staging)t1_staging.FieldSchema(name:a, type:string, comment:null), ] -POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).b SIMPLE [(t1_staging)t1_staging.FieldSchema(name:b, type:int, comment:null), ] -POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).c SIMPLE [(t1_staging)t1_staging.FieldSchema(name:c, type:int, comment:null), ] -POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).d SIMPLE [(t1_staging)t1_staging.FieldSchema(name:d, type:string, comment:null), ] -PREHOOK: query: select 'bucket_0' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_0' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_0 -fff06c6e0fd675ebeff09350e6b7a3900115f72341fd353e5e185e8983d10534002015-01-21 -PREHOOK: query: select 'bucket_2' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_2' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_2 -ffff67aef705abda0b89d899e408c28ef230fd0bb2cb0bb23b057e946ba9ca91\N\N2015-01-21 -fff3474e56ee23c0df629b538268a438d74da36208bdb114bda2da4253f0b4c9\N\N2015-01-21 -PREHOOK: query: select 'bucket_4' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_4' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_4 -fffcb494aa56beb88fddb83cc3b0296d417d3ab7782be76c8c12d33e3f3d6a3c\N\N2015-01-21 -fffad1074d813e6db5c23302a9170fe472c2968844499c90445cbc8559d64fe1082015-01-21 -fff03007f38c32085bb4c9389270b965d371168032845555b663b7d4653ec8ee\N\N2015-01-21 -ffefa017a261a0b3e94c2386e0c47a015e2095e5d56b0f4fca2033d9755c9e45012015-01-21 -PREHOOK: query: select 'bucket_6' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_6' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_6 -fffee943d640a7714d09f9bd50dba08a9d0ebdd146655e4642c293a4396cb385\N\N2015-01-21 -fffc0450ec9b28bae495dffc87a37cc3eea6d2d067ccea8b333185d28847ae2a\N\N2015-01-21 -PREHOOK: query: select 'bucket_8' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_8' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_8 -fffcf74695669d1f444936243869f3586418c6d61d3abb5cc9acb67a3ad7bd2f002015-01-21 -fffbe3c110c390ec20218e5ad4a026ff515668ed55488b717319b556daa962a1002015-01-21 -fffb1b226efc3cfaac8d73647ce4fa4e82413d67265fb55366ac3a4996518738012015-01-21 -fff56191e39b15f0e2f04984c70152fb1bde2ecba52ff5a73b4c28bf4d58c017002015-01-21 -fff4166378aa9d94cd4f8a9cd543375890a61b4f09a57dbfb31a66b33b3e3fd9\N\N2015-01-21 -PREHOOK: query: drop table t1_n147 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t1_n147 -PREHOOK: Output: default@t1_n147 -POSTHOOK: query: drop table t1_n147 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t1_n147 -POSTHOOK: Output: default@t1_n147 -PREHOOK: query: create table t1_n147( -a string, -b int, -c int, -d string) -partitioned by (e string) -clustered by(a) -sorted by(a desc) into 10 buckets stored as textfile -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t1_n147 -POSTHOOK: query: create table t1_n147( -a string, -b int, -c int, -d string) -partitioned by (e string) -clustered by(a) -sorted by(a desc) into 10 buckets stored as textfile -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1_n147 -PREHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,'epart' from t1_staging -PREHOOK: type: QUERY -PREHOOK: Input: default@t1_staging -PREHOOK: Input: default@t1_staging@e=epart -PREHOOK: Output: default@t1_n147 -POSTHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,'epart' from t1_staging -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1_staging -POSTHOOK: Input: default@t1_staging@e=epart -POSTHOOK: Output: default@t1_n147@e=epart -POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).a SIMPLE [(t1_staging)t1_staging.FieldSchema(name:a, type:string, comment:null), ] -POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).b SIMPLE [(t1_staging)t1_staging.FieldSchema(name:b, type:int, comment:null), ] -POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).c SIMPLE [(t1_staging)t1_staging.FieldSchema(name:c, type:int, comment:null), ] -POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).d SIMPLE [(t1_staging)t1_staging.FieldSchema(name:d, type:string, comment:null), ] -PREHOOK: query: select 'bucket_0' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_0' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_0 -fff06c6e0fd675ebeff09350e6b7a3900115f72341fd353e5e185e8983d10534002015-01-21 -PREHOOK: query: select 'bucket_2' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_2' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_2 -ffff67aef705abda0b89d899e408c28ef230fd0bb2cb0bb23b057e946ba9ca91\N\N2015-01-21 -fff3474e56ee23c0df629b538268a438d74da36208bdb114bda2da4253f0b4c9\N\N2015-01-21 -PREHOOK: query: select 'bucket_4' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_4' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_4 -fffcb494aa56beb88fddb83cc3b0296d417d3ab7782be76c8c12d33e3f3d6a3c\N\N2015-01-21 -fffad1074d813e6db5c23302a9170fe472c2968844499c90445cbc8559d64fe1082015-01-21 -fff03007f38c32085bb4c9389270b965d371168032845555b663b7d4653ec8ee\N\N2015-01-21 -ffefa017a261a0b3e94c2386e0c47a015e2095e5d56b0f4fca2033d9755c9e45012015-01-21 -PREHOOK: query: select 'bucket_6' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_6' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_6 -fffee943d640a7714d09f9bd50dba08a9d0ebdd146655e4642c293a4396cb385\N\N2015-01-21 -fffc0450ec9b28bae495dffc87a37cc3eea6d2d067ccea8b333185d28847ae2a\N\N2015-01-21 -PREHOOK: query: select 'bucket_8' -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -POSTHOOK: query: select 'bucket_8' -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -#### A masked pattern was here #### -bucket_8 -fffcf74695669d1f444936243869f3586418c6d61d3abb5cc9acb67a3ad7bd2f002015-01-21 -fffbe3c110c390ec20218e5ad4a026ff515668ed55488b717319b556daa962a1002015-01-21 -fffb1b226efc3cfaac8d73647ce4fa4e82413d67265fb55366ac3a4996518738012015-01-21 -fff56191e39b15f0e2f04984c70152fb1bde2ecba52ff5a73b4c28bf4d58c017002015-01-21 -fff4166378aa9d94cd4f8a9cd543375890a61b4f09a57dbfb31a66b33b3e3fd9\N\N2015-01-21 -PREHOOK: query: CREATE TABLE dynpart_sort_opt_bucketing_test (ca_address_sk int, ca_address_id string, ca_street_number string, ca_street_name string, - ca_street_type string, ca_suite_number string, ca_city string, ca_county string, ca_state string, - ca_zip string, ca_country string, ca_gmt_offset decimal(5,2)) - PARTITIONED BY (ca_location_type string) - CLUSTERED BY (ca_state) INTO 50 BUCKETS STORED AS ORC TBLPROPERTIES('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dynpart_sort_opt_bucketing_test -POSTHOOK: query: CREATE TABLE dynpart_sort_opt_bucketing_test (ca_address_sk int, ca_address_id string, ca_street_number string, ca_street_name string, - ca_street_type string, ca_suite_number string, ca_city string, ca_county string, ca_state string, - ca_zip string, ca_country string, ca_gmt_offset decimal(5,2)) - PARTITIONED BY (ca_location_type string) - CLUSTERED BY (ca_state) INTO 50 BUCKETS STORED AS ORC TBLPROPERTIES('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dynpart_sort_opt_bucketing_test -PREHOOK: query: explain INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', - 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', - '-5', 'single family') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@dynpart_sort_opt_bucketing_test -POSTHOOK: query: explain INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', - 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', - '-5', 'single family') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: _dummy_table - Row Limit Per Split: 1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: array(const struct(5555,'AAAAAAAADLFBAAAA','126','Highland Park','Court','Suite E','San Jose','King George County','VA','28003','United States','-5','single family')) (type: array>) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - UDTF Operator - Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - function name: inline - Select Operator - expressions: col1 (type: int), col2 (type: string), col3 (type: string), col4 (type: string), col5 (type: string), col6 (type: string), col7 (type: string), col8 (type: string), col9 (type: string), col10 (type: string), col11 (type: string), col12 (type: string), col13 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col8 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col8 (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: string), VALUE._col7 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col8 (type: string), VALUE._col9 (type: string), CAST( VALUE._col10 AS decimal(5,2)) (type: decimal(5,2)), VALUE._col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.dynpart_sort_opt_bucketing_test - Write Type: INSERT - - Stage: Stage-0 - Move Operator - tables: - partition: - ca_location_type - replace: false - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.dynpart_sort_opt_bucketing_test - Write Type: INSERT - -PREHOOK: query: INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', - 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', - '-5', 'single family') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@dynpart_sort_opt_bucketing_test -POSTHOOK: query: INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', - 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', - '-5', 'single family') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@dynpart_sort_opt_bucketing_test@ca_location_type=single family -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_address_id SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_address_sk SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_city SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_country SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_county SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_gmt_offset SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_state SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_name SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_number SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_type SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_suite_number SCRIPT [] -POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_zip SCRIPT [] -PREHOOK: query: DROP TABLE dynpart_sort_opt_bucketing_test -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@dynpart_sort_opt_bucketing_test -PREHOOK: Output: default@dynpart_sort_opt_bucketing_test -POSTHOOK: query: DROP TABLE dynpart_sort_opt_bucketing_test -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@dynpart_sort_opt_bucketing_test -POSTHOOK: Output: default@dynpart_sort_opt_bucketing_test diff --git a/ql/src/test/results/clientpositive/llap/bucket2.q.out b/ql/src/test/results/clientpositive/llap/bucket2.q.out index 9a7df0bff8..9b82a96fc1 100644 --- a/ql/src/test/results/clientpositive/llap/bucket2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket2.q.out @@ -41,17 +41,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: vectorized, llap LLAP IO: no inputs @@ -113,7 +113,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket3.q.out b/ql/src/test/results/clientpositive/llap/bucket3.q.out index 9a5a1dfa4a..e8cd3b9ef2 100644 --- a/ql/src/test/results/clientpositive/llap/bucket3.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket3.q.out @@ -42,17 +42,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: vectorized, llap LLAP IO: no inputs @@ -114,7 +114,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket4.q.out b/ql/src/test/results/clientpositive/llap/bucket4.q.out index ec91b3ec61..ea9dc76a3a 100644 --- a/ql/src/test/results/clientpositive/llap/bucket4.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket4.q.out @@ -41,17 +41,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: vectorized, llap LLAP IO: no inputs @@ -113,7 +113,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket5.q.out b/ql/src/test/results/clientpositive/llap/bucket5.q.out index 443e480912..814c2e718b 100644 --- a/ql/src/test/results/clientpositive/llap/bucket5.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket5.q.out @@ -61,17 +61,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Select Operator expressions: key (type: string), value (type: string) @@ -148,7 +148,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket_many.q.out b/ql/src/test/results/clientpositive/llap/bucket_many.q.out index 02f8feb253..92482dbece 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_many.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_many.q.out @@ -42,17 +42,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: vectorized, llap LLAP IO: no inputs @@ -114,7 +114,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/bucket_num_reducers.q.out b/ql/src/test/results/clientpositive/llap/bucket_num_reducers.q.out index a5bbc253d6..7405578c2a 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_num_reducers.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_num_reducers.q.out @@ -39,17 +39,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: llap LLAP IO: no inputs @@ -111,7 +111,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -187,13 +187,11 @@ STAGE PLANS: Basic Stats Work: #### A masked pattern was here #### -PREHOOK: query: insert overwrite table bucket_nr -select * from src +PREHOOK: query: insert overwrite table bucket_nr select * from src PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@bucket_nr -POSTHOOK: query: insert overwrite table bucket_nr -select * from src +POSTHOOK: query: insert overwrite table bucket_nr select * from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@bucket_nr diff --git a/ql/src/test/results/clientpositive/llap/bucket_num_reducers2.q.out b/ql/src/test/results/clientpositive/llap/bucket_num_reducers2.q.out index bd96436cf2..17f30f9e58 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_num_reducers2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_num_reducers2.q.out @@ -39,17 +39,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: llap LLAP IO: no inputs @@ -111,7 +111,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/check_constraint.q.out b/ql/src/test/results/clientpositive/llap/check_constraint.q.out index 955a07112d..b4acc55f15 100644 --- a/ql/src/test/results/clientpositive/llap/check_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/check_constraint.q.out @@ -118,20 +118,24 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint((((((((- _col0) > (- 10)) is not false and (_col1 > 10) is not false) and _col2 is not null is not false) and _col3 BETWEEN _col0 AND _col1 is not false) and ((_col4 = 23.4) or (_col4 = 56) or (_col4 = 4)) is not false) and ((_col5 > round(567.6)) and (_col5 < round(1000.4))) is not false)) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: boolean), _col3 (type: int), _col4 (type: decimal(3,1)), _col5 (type: decimal(4,1)) + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: boolean), _col3 (type: int), UDFToFloat(_col4) (type: float), UDFToLong(_col5) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: boolean), _col3 (type: int), _col4 (type: float), _col5 (type: bigint) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: boolean), VALUE._col2 (type: int), UDFToFloat(VALUE._col3) (type: float), UDFToLong(VALUE._col4) (type: bigint) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: boolean), VALUE._col2 (type: int), VALUE._col3 (type: float), VALUE._col4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1375,20 +1379,24 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint((_col1 is not null and (_col1 >= CAST( _col0 AS decimal(5,2))) is not false)) (type: boolean) Statistics: Num rows: 250 Data size: 51750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 250 Data size: 51750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), CAST( _col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 82000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 250 Data size: 82000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), CAST( VALUE._col1 AS varchar(128)) (type: varchar(128)) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 82000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1487,18 +1495,22 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint((_col1 is not null and (_col1 >= CAST( _col0 AS decimal(5,2))) is not false)) (type: boolean) Statistics: Num rows: 5 Data size: 1035 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1035 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), CAST( _col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 1640 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 1640 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), CAST( VALUE._col1 AS varchar(128)) (type: varchar(128)) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 5 Data size: 1640 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1644,18 +1656,22 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint((_col1 is not null and (_col1 >= CAST( _col0 AS decimal(5,2))) is not false)) (type: boolean) Statistics: Num rows: 5 Data size: 923 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 923 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), CAST( _col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) Reducer 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), CAST( VALUE._col1 AS varchar(128)) (type: varchar(128)) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 5 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1819,18 +1835,22 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint((_col1 is not null and (_col1 >= CAST( _col0 AS decimal(5,2))) is not false)) (type: boolean) Statistics: Num rows: 5 Data size: 1470 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1470 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(5,2)), CAST( _col2 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 1640 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 1640 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) Reducer 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), CAST( VALUE._col1 AS varchar(128)) (type: varchar(128)) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 5 Data size: 1640 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1941,16 +1961,16 @@ STAGE PLANS: predicate: ((key < 10) and enforce_constraint((CAST( key AS decimal(5,2)) is not null and (CAST( key AS decimal(5,2)) >= CAST( UDFToInteger(key) AS decimal(5,2))) is not false))) (type: boolean) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) + expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), CAST( value AS varchar(128)) (type: varchar(128)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 27224 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 83 Data size: 17181 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) + Statistics: Num rows: 83 Data size: 27224 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(5,2)), _col2 (type: varchar(128)) Filter Operator predicate: ((key > 10) and (key < 20) and enforce_constraint(value is not null)) (type: boolean) Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE @@ -1972,7 +1992,7 @@ STAGE PLANS: Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), CAST( VALUE._col1 AS varchar(128)) (type: varchar(128)) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: decimal(5,2)), VALUE._col1 (type: varchar(128)) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 83 Data size: 27224 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -3130,23 +3150,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: 127Y (type: tinyint), col1 (type: int) + expressions: 127Y (type: tinyint), UDFToLong(col1) (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToLong(_col1) (type: bigint) + key expressions: _col1 (type: bigint) null sort order: a sort order: + - Map-reduce partition columns: UDFToLong(_col1) (type: bigint) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: int) + Map-reduce partition columns: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: tinyint), UDFToLong(VALUE._col1) (type: bigint) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -3291,20 +3311,24 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint(((null = 4) or (null = 5)) is not false) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: bigint) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) + Select Operator + expressions: UDFToByte(_col0) (type: tinyint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: bigint) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: UDFToByte(VALUE._col0) (type: tinyint), KEY.reducesinkkey0 (type: bigint) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -3372,20 +3396,24 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint(((_col1 = 4) or (_col1 = 5)) is not false) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToLong(_col1) (type: bigint) - null sort order: a - sort order: + - Map-reduce partition columns: UDFToLong(_col1) (type: bigint) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: int) + Select Operator + expressions: _col0 (type: tinyint), UDFToLong(_col1) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: bigint) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: tinyint), UDFToLong(VALUE._col1) (type: bigint) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/default_constraint.q.out b/ql/src/test/results/clientpositive/llap/default_constraint.q.out index 1270a426e8..f6e05f2ded 100644 --- a/ql/src/test/results/clientpositive/llap/default_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/default_constraint.q.out @@ -205,7 +205,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: 127Y (type: tinyint), 32767S (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), col1 (type: decimal(2,1)), col2 (type: decimal(4,1)) + expressions: 127Y (type: tinyint), 32767S (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), UDFToDouble(col1) (type: double), CAST( col2 AS decimal(9,2)) (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -214,14 +214,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: tinyint) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: decimal(2,1)), _col5 (type: decimal(4,1)) + value expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), UDFToDouble(VALUE._col3) (type: double), CAST( VALUE._col4 AS decimal(9,2)) (type: decimal(9,2)) + expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), VALUE._col4 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -722,7 +722,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: 'current_database()' (type: string), CURRENT_USER() (type: string), 'Current_User()' (type: string), col1 (type: string), col2 (type: string) + expressions: 'current_database()' (type: string), CURRENT_USER() (type: string), 'Current_User()' (type: string), CAST( col1 AS varchar(350)) (type: varchar(350)), CAST( col2 AS CHAR(20)) (type: char(20)) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -731,14 +731,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: varchar(350)), _col4 (type: char(20)) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), CAST( VALUE._col2 AS varchar(350)) (type: varchar(350)), CAST( VALUE._col3 AS CHAR(20)) (type: char(20)) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: varchar(350)), VALUE._col3 (type: char(20)) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -981,7 +981,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: true (type: boolean), col1 (type: string) + expressions: true (type: boolean), CAST( col1 AS BINARY) (type: binary) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -990,14 +990,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: boolean) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + value expressions: _col1 (type: binary) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: boolean), CAST( VALUE._col0 AS BINARY) (type: binary) + expressions: KEY.reducesinkkey0 (type: boolean), VALUE._col0 (type: binary) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1463,23 +1463,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: null (type: tinyint), col1 (type: int), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2)) + expressions: null (type: tinyint), UDFToShort(col1) (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToShort(_col1) (type: smallint) + key expressions: _col1 (type: smallint) null sort order: a sort order: + - Map-reduce partition columns: UDFToShort(_col1) (type: smallint) + Map-reduce partition columns: _col1 (type: smallint) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) + value expressions: _col0 (type: tinyint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: tinyint), UDFToShort(VALUE._col1) (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2)) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), VALUE._col4 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1633,23 +1633,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: null (type: tinyint), col1 (type: int), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2)) + expressions: null (type: tinyint), UDFToShort(col1) (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToShort(_col1) (type: smallint) + key expressions: _col1 (type: smallint) null sort order: a sort order: + - Map-reduce partition columns: UDFToShort(_col1) (type: smallint) + Map-reduce partition columns: _col1 (type: smallint) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) + value expressions: _col0 (type: tinyint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: tinyint), UDFToShort(VALUE._col1) (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2)) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), VALUE._col4 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1904,20 +1904,24 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint(127Y is not null) (type: boolean) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: smallint) + Select Operator + expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), CAST( _col5 AS decimal(9,2)) (type: decimal(9,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(5,2)) + Reduce Output Operator + key expressions: _col1 (type: smallint) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: smallint) + Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), CAST( VALUE._col4 AS decimal(9,2)) (type: decimal(9,2)) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), VALUE._col4 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -2246,20 +2250,24 @@ STAGE PLANS: Filter Operator predicate: enforce_constraint(108Y is not null) (type: boolean) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: smallint) + Select Operator + expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), CAST( _col5 AS decimal(9,2)) (type: decimal(9,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(5,2)) + Reduce Output Operator + key expressions: _col1 (type: smallint) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: smallint) + Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), CAST( VALUE._col4 AS decimal(9,2)) (type: decimal(9,2)) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), VALUE._col4 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -2657,23 +2665,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: null (type: tinyint), col1 (type: int), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2)) + expressions: null (type: tinyint), UDFToShort(col1) (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToShort(_col1) (type: smallint) + key expressions: _col1 (type: smallint) null sort order: a sort order: + - Map-reduce partition columns: UDFToShort(_col1) (type: smallint) + Map-reduce partition columns: _col1 (type: smallint) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) + value expressions: _col0 (type: tinyint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: tinyint), UDFToShort(VALUE._col1) (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2)) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), VALUE._col4 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -2827,23 +2835,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: null (type: tinyint), col1 (type: int), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2)) + expressions: null (type: tinyint), UDFToShort(col1) (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToShort(_col1) (type: smallint) + key expressions: _col1 (type: smallint) null sort order: a sort order: + - Map-reduce partition columns: UDFToShort(_col1) (type: smallint) + Map-reduce partition columns: _col1 (type: smallint) Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) + value expressions: _col0 (type: tinyint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2)) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: tinyint), UDFToShort(VALUE._col1) (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2)) + expressions: VALUE._col0 (type: tinyint), KEY.reducesinkkey0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: double), VALUE._col4 (type: decimal(9,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out index 72b19e1c44..389a5f2769 100644 --- a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out +++ b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out @@ -41,17 +41,17 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) auto parallelism: false Execution mode: vectorized, llap LLAP IO: no inputs @@ -113,7 +113,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_bucketing.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_bucketing.q.out new file mode 100644 index 0000000000..453d2451df --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_bucketing.q.out @@ -0,0 +1,701 @@ +PREHOOK: query: drop table if exists t1_staging +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists t1_staging +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1_staging( +a string, +b int, +c int, +d string) +partitioned by (e string) +clustered by(a) +sorted by(a desc) +into 256 buckets stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_staging +POSTHOOK: query: create table t1_staging( +a string, +b int, +c int, +d string) +partitioned by (e string) +clustered by(a) +sorted by(a desc) +into 256 buckets stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_staging +PREHOOK: query: load data local inpath '../../data/files/sortdp/000000_0' overwrite into table t1_staging partition (e='epart') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1_staging +POSTHOOK: query: load data local inpath '../../data/files/sortdp/000000_0' overwrite into table t1_staging partition (e='epart') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1_staging +POSTHOOK: Output: default@t1_staging@e=epart +PREHOOK: query: drop table t1_n147 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table t1_n147 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1_n147( +a string, +b int, +c int, +d string) +partitioned by (e string) +clustered by(a) +sorted by(a desc) into 10 buckets stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: create table t1_n147( +a string, +b int, +c int, +d string) +partitioned by (e string) +clustered by(a) +sorted by(a desc) into 10 buckets stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_n147 +PREHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,'epart' from t1_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_staging +PREHOOK: Input: default@t1_staging@e=epart +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,'epart' from t1_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_staging +POSTHOOK: Input: default@t1_staging@e=epart +POSTHOOK: Output: default@t1_n147@e=epart +POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).a SIMPLE [(t1_staging)t1_staging.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).b SIMPLE [(t1_staging)t1_staging.FieldSchema(name:b, type:int, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).c SIMPLE [(t1_staging)t1_staging.FieldSchema(name:c, type:int, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).d SIMPLE [(t1_staging)t1_staging.FieldSchema(name:d, type:string, comment:null), ] +PREHOOK: query: select 'bucket_0' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_0' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_0 +fff06c6e0fd675ebeff09350e6b7a3900115f72341fd353e5e185e8983d10534002015-01-21 +PREHOOK: query: select 'bucket_2' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_2' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_2 +ffff67aef705abda0b89d899e408c28ef230fd0bb2cb0bb23b057e946ba9ca91\N\N2015-01-21 +fff3474e56ee23c0df629b538268a438d74da36208bdb114bda2da4253f0b4c9\N\N2015-01-21 +PREHOOK: query: select 'bucket_4' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_4' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_4 +fffcb494aa56beb88fddb83cc3b0296d417d3ab7782be76c8c12d33e3f3d6a3c\N\N2015-01-21 +fffad1074d813e6db5c23302a9170fe472c2968844499c90445cbc8559d64fe1082015-01-21 +fff03007f38c32085bb4c9389270b965d371168032845555b663b7d4653ec8ee\N\N2015-01-21 +ffefa017a261a0b3e94c2386e0c47a015e2095e5d56b0f4fca2033d9755c9e45012015-01-21 +PREHOOK: query: select 'bucket_6' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_6' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_6 +fffee943d640a7714d09f9bd50dba08a9d0ebdd146655e4642c293a4396cb385\N\N2015-01-21 +fffc0450ec9b28bae495dffc87a37cc3eea6d2d067ccea8b333185d28847ae2a\N\N2015-01-21 +PREHOOK: query: select 'bucket_8' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_8' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_8 +fffcf74695669d1f444936243869f3586418c6d61d3abb5cc9acb67a3ad7bd2f002015-01-21 +fffbe3c110c390ec20218e5ad4a026ff515668ed55488b717319b556daa962a1002015-01-21 +fffb1b226efc3cfaac8d73647ce4fa4e82413d67265fb55366ac3a4996518738012015-01-21 +fff56191e39b15f0e2f04984c70152fb1bde2ecba52ff5a73b4c28bf4d58c017002015-01-21 +fff4166378aa9d94cd4f8a9cd543375890a61b4f09a57dbfb31a66b33b3e3fd9\N\N2015-01-21 +PREHOOK: query: drop table t1_n147 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1_n147 +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: drop table t1_n147 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1_n147 +POSTHOOK: Output: default@t1_n147 +PREHOOK: query: create table t1_n147( +a string, +b int, +c int, +d string) +partitioned by (e string) +clustered by(a) +sorted by(a desc) into 10 buckets stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: create table t1_n147( +a string, +b int, +c int, +d string) +partitioned by (e string) +clustered by(a) +sorted by(a desc) into 10 buckets stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_n147 +PREHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,'epart' from t1_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_staging +PREHOOK: Input: default@t1_staging@e=epart +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,'epart' from t1_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_staging +POSTHOOK: Input: default@t1_staging@e=epart +POSTHOOK: Output: default@t1_n147@e=epart +POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).a SIMPLE [(t1_staging)t1_staging.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).b SIMPLE [(t1_staging)t1_staging.FieldSchema(name:b, type:int, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).c SIMPLE [(t1_staging)t1_staging.FieldSchema(name:c, type:int, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=epart).d SIMPLE [(t1_staging)t1_staging.FieldSchema(name:d, type:string, comment:null), ] +PREHOOK: query: select 'bucket_0' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_0' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_0 +fff06c6e0fd675ebeff09350e6b7a3900115f72341fd353e5e185e8983d10534002015-01-21 +PREHOOK: query: select 'bucket_2' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_2' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_2 +ffff67aef705abda0b89d899e408c28ef230fd0bb2cb0bb23b057e946ba9ca91\N\N2015-01-21 +fff3474e56ee23c0df629b538268a438d74da36208bdb114bda2da4253f0b4c9\N\N2015-01-21 +PREHOOK: query: select 'bucket_4' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_4' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_4 +fffcb494aa56beb88fddb83cc3b0296d417d3ab7782be76c8c12d33e3f3d6a3c\N\N2015-01-21 +fffad1074d813e6db5c23302a9170fe472c2968844499c90445cbc8559d64fe1082015-01-21 +fff03007f38c32085bb4c9389270b965d371168032845555b663b7d4653ec8ee\N\N2015-01-21 +ffefa017a261a0b3e94c2386e0c47a015e2095e5d56b0f4fca2033d9755c9e45012015-01-21 +PREHOOK: query: select 'bucket_6' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_6' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_6 +fffee943d640a7714d09f9bd50dba08a9d0ebdd146655e4642c293a4396cb385\N\N2015-01-21 +fffc0450ec9b28bae495dffc87a37cc3eea6d2d067ccea8b333185d28847ae2a\N\N2015-01-21 +PREHOOK: query: select 'bucket_8' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 'bucket_8' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +bucket_8 +fffcf74695669d1f444936243869f3586418c6d61d3abb5cc9acb67a3ad7bd2f002015-01-21 +fffbe3c110c390ec20218e5ad4a026ff515668ed55488b717319b556daa962a1002015-01-21 +fffb1b226efc3cfaac8d73647ce4fa4e82413d67265fb55366ac3a4996518738012015-01-21 +fff56191e39b15f0e2f04984c70152fb1bde2ecba52ff5a73b4c28bf4d58c017002015-01-21 +fff4166378aa9d94cd4f8a9cd543375890a61b4f09a57dbfb31a66b33b3e3fd9\N\N2015-01-21 +PREHOOK: query: CREATE TABLE dynpart_sort_opt_bucketing_test (ca_address_sk int, ca_address_id string, ca_street_number string, ca_street_name string, + ca_street_type string, ca_suite_number string, ca_city string, ca_county string, ca_state string, + ca_zip string, ca_country string, ca_gmt_offset decimal(5,2)) + PARTITIONED BY (ca_location_type string) + CLUSTERED BY (ca_state) INTO 50 BUCKETS STORED AS ORC TBLPROPERTIES('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dynpart_sort_opt_bucketing_test +POSTHOOK: query: CREATE TABLE dynpart_sort_opt_bucketing_test (ca_address_sk int, ca_address_id string, ca_street_number string, ca_street_name string, + ca_street_type string, ca_suite_number string, ca_city string, ca_county string, ca_state string, + ca_zip string, ca_country string, ca_gmt_offset decimal(5,2)) + PARTITIONED BY (ca_location_type string) + CLUSTERED BY (ca_state) INTO 50 BUCKETS STORED AS ORC TBLPROPERTIES('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dynpart_sort_opt_bucketing_test +PREHOOK: query: explain INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dynpart_sort_opt_bucketing_test +POSTHOOK: query: explain INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct(5555,'AAAAAAAADLFBAAAA','126','Highland Park','Court','Suite E','San Jose','King George County','VA','28003','United States','-5','single family')) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: int), col2 (type: string), col3 (type: string), col4 (type: string), col5 (type: string), col6 (type: string), col7 (type: string), col8 (type: string), col9 (type: string), col10 (type: string), col11 (type: string), CAST( col12 AS decimal(5,2)) (type: decimal(5,2)), col13 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col12 (type: string), _bucket_number (type: string), _col8 (type: string) + null sort order: aaa + sort order: +++ + Map-reduce partition columns: _col12 (type: string) + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: decimal(5,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: string), VALUE._col7 (type: string), KEY._col8 (type: string), VALUE._col9 (type: string), VALUE._col10 (type: string), VALUE._col11 (type: decimal(5,2)), KEY._col12 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.dynpart_sort_opt_bucketing_test + Write Type: INSERT + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ca_location_type + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.dynpart_sort_opt_bucketing_test + Write Type: INSERT + +PREHOOK: query: INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dynpart_sort_opt_bucketing_test +POSTHOOK: query: INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dynpart_sort_opt_bucketing_test@ca_location_type=single family +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_address_id SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_address_sk SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_city SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_country SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_county SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_gmt_offset SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_state SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_name SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_number SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_type SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_suite_number SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_zip SCRIPT [] +PREHOOK: query: select * from dynpart_sort_opt_bucketing_test +PREHOOK: type: QUERY +PREHOOK: Input: default@dynpart_sort_opt_bucketing_test +PREHOOK: Input: default@dynpart_sort_opt_bucketing_test@ca_location_type=single family +#### A masked pattern was here #### +POSTHOOK: query: select * from dynpart_sort_opt_bucketing_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dynpart_sort_opt_bucketing_test +POSTHOOK: Input: default@dynpart_sort_opt_bucketing_test@ca_location_type=single family +#### A masked pattern was here #### +5555 AAAAAAAADLFBAAAA 126 Highland Park Court Suite E San Jose King George County VA 28003 United States -5.00 single family +PREHOOK: query: explain INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dynpart_sort_opt_bucketing_test +POSTHOOK: query: explain INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct(5555,'AAAAAAAADLFBAAAA','126','Highland Park','Court','Suite E','San Jose','King George County','VA','28003','United States','-5','single family')) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: int), col2 (type: string), col3 (type: string), col4 (type: string), col5 (type: string), col6 (type: string), col7 (type: string), col8 (type: string), col9 (type: string), col10 (type: string), col11 (type: string), CAST( col12 AS decimal(5,2)) (type: decimal(5,2)), col13 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col12 (type: string), _bucket_number (type: string), _col8 (type: string) + null sort order: aaa + sort order: +++ + Map-reduce partition columns: _col12 (type: string) + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: decimal(5,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: string), VALUE._col7 (type: string), KEY._col8 (type: string), VALUE._col9 (type: string), VALUE._col10 (type: string), VALUE._col11 (type: decimal(5,2)), KEY._col12 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.dynpart_sort_opt_bucketing_test + Write Type: INSERT + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ca_location_type + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.dynpart_sort_opt_bucketing_test + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: ca_address_sk, ca_address_id, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset + Column Types: int, string, string, string, string, string, string, string, string, string, string, decimal(5,2) + Table: default.dynpart_sort_opt_bucketing_test + +PREHOOK: query: INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dynpart_sort_opt_bucketing_test +POSTHOOK: query: INSERT INTO TABLE dynpart_sort_opt_bucketing_test PARTITION (ca_location_type) VALUES (5555, 'AAAAAAAADLFBAAAA', '126', + 'Highland Park', 'Court', 'Suite E', 'San Jose', 'King George County', 'VA', '28003', 'United States', + '-5', 'single family') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dynpart_sort_opt_bucketing_test@ca_location_type=single family +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_address_id SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_address_sk SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_city SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_country SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_county SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_gmt_offset SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_state SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_name SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_number SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_street_type SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_suite_number SCRIPT [] +POSTHOOK: Lineage: dynpart_sort_opt_bucketing_test PARTITION(ca_location_type=single family).ca_zip SCRIPT [] +PREHOOK: query: select * from dynpart_sort_opt_bucketing_test +PREHOOK: type: QUERY +PREHOOK: Input: default@dynpart_sort_opt_bucketing_test +PREHOOK: Input: default@dynpart_sort_opt_bucketing_test@ca_location_type=single family +#### A masked pattern was here #### +POSTHOOK: query: select * from dynpart_sort_opt_bucketing_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dynpart_sort_opt_bucketing_test +POSTHOOK: Input: default@dynpart_sort_opt_bucketing_test@ca_location_type=single family +#### A masked pattern was here #### +5555 AAAAAAAADLFBAAAA 126 Highland Park Court Suite E San Jose King George County VA 28003 United States -5.00 single family +5555 AAAAAAAADLFBAAAA 126 Highland Park Court Suite E San Jose King George County VA 28003 United States -5.00 single family +PREHOOK: query: DROP TABLE dynpart_sort_opt_bucketing_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@dynpart_sort_opt_bucketing_test +PREHOOK: Output: default@dynpart_sort_opt_bucketing_test +POSTHOOK: query: DROP TABLE dynpart_sort_opt_bucketing_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@dynpart_sort_opt_bucketing_test +POSTHOOK: Output: default@dynpart_sort_opt_bucketing_test +PREHOOK: query: drop table if exists t1_staging +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1_staging +PREHOOK: Output: default@t1_staging +POSTHOOK: query: drop table if exists t1_staging +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1_staging +POSTHOOK: Output: default@t1_staging +PREHOOK: query: create table t1_staging( + a string, + b int, + c int, + d string) + partitioned by (e decimal(18,0)) + clustered by(a) + into 256 buckets STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_staging +POSTHOOK: query: create table t1_staging( + a string, + b int, + c int, + d string) + partitioned by (e decimal(18,0)) + clustered by(a) + into 256 buckets STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_staging +PREHOOK: query: load data local inpath '../../data/files/sortdp/000000_0' overwrite into table t1_staging partition (e=100) +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1_staging +POSTHOOK: query: load data local inpath '../../data/files/sortdp/000000_0' overwrite into table t1_staging partition (e=100) +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1_staging +POSTHOOK: Output: default@t1_staging@e=100 +PREHOOK: query: drop table t1_n147 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1_n147 +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: drop table t1_n147 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1_n147 +POSTHOOK: Output: default@t1_n147 +PREHOOK: query: create table t1_n147( + a string, + b decimal(6,0), + c int, + d string) + partitioned by (e decimal(3,0)) + clustered by(a,b) + into 10 buckets STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: create table t1_n147( + a string, + b decimal(6,0), + c int, + d string) + partitioned by (e decimal(3,0)) + clustered by(a,b) + into 10 buckets STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_n147 +PREHOOK: query: explain insert overwrite table t1_n147 partition(e) select a,b,c,d,e from t1_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_staging +PREHOOK: Input: default@t1_staging@e=100 +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: explain insert overwrite table t1_n147 partition(e) select a,b,c,d,e from t1_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_staging +POSTHOOK: Input: default@t1_staging@e=100 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1_staging + Statistics: Num rows: 51 Data size: 23008 Basic stats: PARTIAL Column stats: PARTIAL + Select Operator + expressions: a (type: string), CAST( b AS decimal(6,0)) (type: decimal(6,0)), c (type: int), d (type: string), e (type: decimal(18,0)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 51 Data size: 23008 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col4 (type: decimal(18,0)), _bucket_number (type: string), _col0 (type: string), _col1 (type: decimal(6,0)) + null sort order: aaaa + sort order: ++++ + Map-reduce partition columns: _col4 (type: decimal(18,0)) + Statistics: Num rows: 51 Data size: 23008 Basic stats: PARTIAL Column stats: PARTIAL + value expressions: _col2 (type: int), _col3 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY._col0 (type: string), KEY._col1 (type: decimal(6,0)), VALUE._col2 (type: int), VALUE._col3 (type: string), KEY._col4 (type: decimal(18,0)), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 51 Data size: 23008 Basic stats: PARTIAL Column stats: PARTIAL + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.t1_n147 + Write Type: INSERT + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + e + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.t1_n147 + Write Type: INSERT + +PREHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,e from t1_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_staging +PREHOOK: Input: default@t1_staging@e=100 +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: insert overwrite table t1_n147 partition(e) select a,b,c,d,e from t1_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_staging +POSTHOOK: Input: default@t1_staging@e=100 +POSTHOOK: Output: default@t1_n147@e=100 +POSTHOOK: Lineage: t1_n147 PARTITION(e=100).a SIMPLE [(t1_staging)t1_staging.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=100).b EXPRESSION [(t1_staging)t1_staging.FieldSchema(name:b, type:int, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=100).c SIMPLE [(t1_staging)t1_staging.FieldSchema(name:c, type:int, comment:null), ] +POSTHOOK: Lineage: t1_n147 PARTITION(e=100).d SIMPLE [(t1_staging)t1_staging.FieldSchema(name:d, type:string, comment:null), ] +Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +PREHOOK: query: with q1 as (select count(*) as cnt from t1_staging), + q2 as (select count(*) as cnt from t1_n147) +select q1.cnt = q2.cnt from q1 join q2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n147 +PREHOOK: Input: default@t1_n147@e=100 +PREHOOK: Input: default@t1_staging +PREHOOK: Input: default@t1_staging@e=100 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select count(*) as cnt from t1_staging), + q2 as (select count(*) as cnt from t1_n147) +select q1.cnt = q2.cnt from q1 join q2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n147 +POSTHOOK: Input: default@t1_n147@e=100 +POSTHOOK: Input: default@t1_staging +POSTHOOK: Input: default@t1_staging@e=100 +#### A masked pattern was here #### +true +PREHOOK: query: drop table t1_staging +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1_staging +PREHOOK: Output: default@t1_staging +POSTHOOK: query: drop table t1_staging +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1_staging +POSTHOOK: Output: default@t1_staging +PREHOOK: query: drop table t1_n147 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1_n147 +PREHOOK: Output: default@t1_n147 +POSTHOOK: query: drop table t1_n147 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1_n147 +POSTHOOK: Output: default@t1_n147 diff --git a/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out b/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out index 9b423dc38f..bdf1a65cb1 100644 --- a/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_into_default_keyword.q.out @@ -49,28 +49,28 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: null (type: void) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + expressions: null (type: int), null (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: void) + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -80,7 +80,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: key, value - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.0 @@ -192,28 +192,28 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col1 (type: int), null (type: void) + expressions: col1 (type: int), null (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: void) + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -223,7 +223,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: key, value - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.0 @@ -335,23 +335,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col1 (type: int), col2 (type: int) + expressions: col1 (type: int), CAST( col2 AS STRING) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -479,23 +479,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: null (type: void), null (type: string) + expressions: null (type: int), null (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: void), _col1 (type: string) + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -622,28 +622,28 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col1 (type: int), null (type: void) + expressions: col1 (type: int), null (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: void) + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -653,7 +653,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: key, value - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.0 @@ -783,28 +783,28 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col1 (type: int), null (type: void) + expressions: col1 (type: int), null (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: void) + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -814,7 +814,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: key, value - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.0 @@ -936,23 +936,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: null (type: void), col2 (type: string) + expressions: null (type: int), col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: void), _col1 (type: string) + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1079,28 +1079,28 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col1 (type: int), null (type: void) + expressions: col1 (type: int), null (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: void) + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1110,7 +1110,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: key, value - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.0 @@ -1222,23 +1222,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col1 (type: int), col2 (type: int) + expressions: col1 (type: int), CAST( col2 AS STRING) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1509,28 +1509,28 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col1 (type: int), null (type: void) + expressions: col1 (type: int), null (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: void) + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1540,7 +1540,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: key, value - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.0 @@ -1652,23 +1652,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col2 (type: int), col1 (type: int) + expressions: col2 (type: int), CAST( col1 AS STRING) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1795,28 +1795,28 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE function name: inline Select Operator - expressions: col1 (type: int), null (type: void) + expressions: col1 (type: int), null (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: void) + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), CAST( VALUE._col0 AS STRING) (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1826,7 +1826,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: key, value - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') minReductionHashAggr: 0.0 @@ -2700,13 +2700,13 @@ STAGE PLANS: Select Operator expressions: _col6 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: null (type: string) null sort order: a sort order: + Map-reduce partition columns: null (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) Reducer 3 Execution mode: llap @@ -2959,13 +2959,13 @@ STAGE PLANS: Select Operator expressions: _col6 (type: int), _col7 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: null (type: string) null sort order: a sort order: + Map-reduce partition columns: null (type: string) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string) Reducer 3 Execution mode: vectorized, llap diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_20.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_20.q.out index f69b74d5e5..0a8355d03f 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_20.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_20.q.out @@ -59,20 +59,20 @@ STAGE PLANS: filterExpr: (ds = '1') (type: boolean) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + expressions: CAST( key AS STRING) (type: string), value (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 183000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: CAST( _col0 AS STRING) (type: string) + key expressions: _col0 (type: string) null sort order: a sort order: + - Map-reduce partition columns: CAST( _col0 AS STRING) (type: string) - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 183000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: CAST( VALUE._col0 AS STRING) (type: string), VALUE._col1 (type: string), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 183000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1462,20 +1462,20 @@ STAGE PLANS: filterExpr: (ds = '1') (type: boolean) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: (key + key) (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + expressions: CAST( (key + key) AS STRING) (type: string), value (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 183000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: CAST( _col0 AS STRING) (type: string) + key expressions: _col0 (type: string) null sort order: a sort order: + - Map-reduce partition columns: CAST( _col0 AS STRING) (type: string) - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string) + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 183000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: CAST( VALUE._col0 AS STRING) (type: string), VALUE._col1 (type: string), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 183000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/stats10.q.out b/ql/src/test/results/clientpositive/stats10.q.out index cc642e22cd..8a38013477 100644 --- a/ql/src/test/results/clientpositive/stats10.q.out +++ b/ql/src/test/results/clientpositive/stats10.q.out @@ -32,20 +32,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string), value (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator