diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index b48889ef9d..a49958dca6 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -444,7 +444,66 @@ minillap.query.files=acid_bucket_pruning.q,\ cttl.q,\ vector_offset_limit.q,\ temp_table_add_part_with_loc.q,\ - temp_table_drop_partitions_filter4.q + temp_table_drop_partitions_filter4.q,\ + autoColumnStats_6.q,\ + autogen_colalias.q,\ + binary_output_format.q,\ + create_genericudaf.q,\ + create_udaf.q,\ + create_view.q,\ + gen_udf_example_add10.q,\ + groupby_bigdata.q,\ + input14.q,\ + input14_limit.q,\ + input17.q,\ + input18.q,\ + input20.q,\ + input33.q,\ + input34.q,\ + input35.q,\ + input36.q,\ + input38.q,\ + input5.q,\ + insert_into3.q,\ + insert_into4.q,\ + insert_into5.q,\ + insert_into6.q,\ + load_binary_data.q,\ + macro_1.q,\ + macro_duplicate.q,\ + mapreduce3.q,\ + mapreduce4.q,\ + mapreduce7.q,\ + mapreduce8.q,\ + merge_test_dummy_operator.q,\ + newline.q,\ + nonreserved_keywords_insert_into1.q,\ + nullscript.q,\ + orc_createas1.q,\ + partcols1.q,\ + ppd_transform.q,\ + query_with_semi.q,\ + rcfile_bigdata.q,\ + regexp_extract.q,\ + script_env_var1.q,\ + script_env_var2.q,\ + script_pipe.q,\ + scriptfile1.q,\ + select_transform_hint.q,\ + str_to_map.q,\ + temp_table_partcols1.q,\ + transform1.q,\ + transform2.q,\ + transform3.q,\ + transform_acid.q,\ + transform_ppr1.q,\ + transform_ppr2.q,\ + udaf_sum_list.q,\ + udf_printf.q,\ + union23.q,\ + union_script.q,\ + vector_custom_udf_configure.q,\ + vector_udf3.q minillaplocal.query.files=\ empty_files_external_table.q,\ diff --git ql/src/test/queries/clientpositive/input5.q ql/src/test/queries/clientpositive/input5.q index bf598de0c8..72a1e74aa4 100644 --- ql/src/test/queries/clientpositive/input5.q +++ ql/src/test/queries/clientpositive/input5.q @@ -1,4 +1,6 @@ --! qt:dataset:src_thrift +-- SORT_QUERY_RESULTS + CREATE TABLE dest1_n94(key STRING, value STRING) STORED AS TEXTFILE; EXPLAIN diff --git ql/src/test/queries/clientpositive/regexp_extract.q ql/src/test/queries/clientpositive/regexp_extract.q index 678b8faf6e..e32094bfef 100644 --- ql/src/test/queries/clientpositive/regexp_extract.q +++ ql/src/test/queries/clientpositive/regexp_extract.q @@ -1,4 +1,6 @@ --! qt:dataset:src +-- SORT_QUERY_RESULTS + EXPLAIN EXTENDED FROM ( FROM src diff --git ql/src/test/queries/clientpositive/select_transform_hint.q ql/src/test/queries/clientpositive/select_transform_hint.q index 39577d1376..9ad7a92a21 100644 --- ql/src/test/queries/clientpositive/select_transform_hint.q +++ ql/src/test/queries/clientpositive/select_transform_hint.q @@ -1,4 +1,6 @@ --! qt:dataset:src +-- SORT_QUERY_RESULTS + set hive.entity.capture.transform=true; EXPLAIN diff --git ql/src/test/results/clientpositive/binary_output_format.q.out ql/src/test/results/clientpositive/binary_output_format.q.out deleted file mode 100644 index b414360855..0000000000 --- ql/src/test/results/clientpositive/binary_output_format.q.out +++ /dev/null @@ -1,1010 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n109(mydata STRING) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -WITH SERDEPROPERTIES ( - 'serialization.last.column.takes.rest'='true' -) -STORED AS - INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' - OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat' -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n109 -POSTHOOK: query: CREATE TABLE dest1_n109(mydata STRING) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' -WITH SERDEPROPERTIES ( - 'serialization.last.column.takes.rest'='true' -) -STORED AS - INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' - OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat' -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n109 -PREHOOK: query: EXPLAIN EXTENDED -INSERT OVERWRITE TABLE dest1_n109 -SELECT TRANSFORM(*) - USING 'cat' - AS mydata STRING - ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' - WITH SERDEPROPERTIES ( - 'serialization.last.column.takes.rest'='true' - ) - RECORDREADER 'org.apache.hadoop.hive.ql.exec.BinaryRecordReader' -FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n109 -POSTHOOK: query: EXPLAIN EXTENDED -INSERT OVERWRITE TABLE dest1_n109 -SELECT TRANSFORM(*) - USING 'cat' - AS mydata STRING - ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' - WITH SERDEPROPERTIES ( - 'serialization.last.column.takes.rest'='true' - ) - RECORDREADER 'org.apache.hadoop.hive.ql.exec.BinaryRecordReader' -FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n109 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucketing_version -1 - columns _col0 - columns.types string - field.delim 9 - serialization.format 9 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns mydata - columns.comments - columns.types string -#### A masked pattern was here #### - name default.dest1_n109 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest1_n109 { string mydata} - serialization.format 1 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n109 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Select Operator - expressions: _col0 (type: string) - outputColumnNames: mydata - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(mydata, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - null sort order: - numBuckets: -1 - sort order: - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col0 (type: struct) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [src] - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0 - columns.types struct - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: true -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns mydata - columns.comments - columns.types string -#### A masked pattern was here #### - name default.dest1_n109 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest1_n109 { string mydata} - serialization.format 1 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n109 - - Stage: Stage-2 - Stats Work - Basic Stats Work: -#### A masked pattern was here #### - Column Stats Desc: - Columns: mydata - Column Types: string - Table: default.dest1_n109 - Is Table Level Stats: true - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns mydata - columns.comments - columns.types string -#### A masked pattern was here #### - name default.dest1_n109 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest1_n109 { string mydata} - serialization.format 1 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n109 - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -ext-10002 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns mydata - columns.comments - columns.types string -#### A masked pattern was here #### - name default.dest1_n109 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest1_n109 { string mydata} - serialization.format 1 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns mydata - columns.comments - columns.types string -#### A masked pattern was here #### - name default.dest1_n109 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest1_n109 { string mydata} - serialization.format 1 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n109 - name: default.dest1_n109 - Truncated Path -> Alias: -#### A masked pattern was here #### - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns mydata - columns.comments - columns.types string -#### A masked pattern was here #### - name default.dest1_n109 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest1_n109 { string mydata} - serialization.format 1 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n109 - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -ext-10002 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns mydata - columns.comments - columns.types string -#### A masked pattern was here #### - name default.dest1_n109 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest1_n109 { string mydata} - serialization.format 1 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns mydata - columns.comments - columns.types string -#### A masked pattern was here #### - name default.dest1_n109 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest1_n109 { string mydata} - serialization.format 1 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n109 - name: default.dest1_n109 - Truncated Path -> Alias: -#### A masked pattern was here #### - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT OVERWRITE TABLE dest1_n109 -SELECT TRANSFORM(*) - USING 'cat' - AS mydata STRING - ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' - WITH SERDEPROPERTIES ( - 'serialization.last.column.takes.rest'='true' - ) - RECORDREADER 'org.apache.hadoop.hive.ql.exec.BinaryRecordReader' -FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n109 -POSTHOOK: query: INSERT OVERWRITE TABLE dest1_n109 -SELECT TRANSFORM(*) - USING 'cat' - AS mydata STRING - ROW FORMAT SERDE - 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' - WITH SERDEPROPERTIES ( - 'serialization.last.column.takes.rest'='true' - ) - RECORDREADER 'org.apache.hadoop.hive.ql.exec.BinaryRecordReader' -FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n109 -POSTHOOK: Lineage: dest1_n109.mydata SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT * FROM dest1_n109 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n109 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM dest1_n109 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n109 -#### A masked pattern was here #### -238 val_238 -86 val_86 -311 val_311 -27 val_27 -165 val_165 -409 val_409 -255 val_255 -278 val_278 -98 val_98 -484 val_484 -265 val_265 -193 val_193 -401 val_401 -150 val_150 -273 val_273 -224 val_224 -369 val_369 -66 val_66 -128 val_128 -213 val_213 -146 val_146 -406 val_406 -429 val_429 -374 val_374 -152 val_152 -469 val_469 -145 val_145 -495 val_495 -37 val_37 -327 val_327 -281 val_281 -277 val_277 -209 val_209 -15 val_15 -82 val_82 -403 val_403 -166 val_166 -417 val_417 -430 val_430 -252 val_252 -292 val_292 -219 val_219 -287 val_287 -153 val_153 -193 val_193 -338 val_338 -446 val_446 -459 val_459 -394 val_394 -237 val_237 -482 val_482 -174 val_174 -413 val_413 -494 val_494 -207 val_207 -199 val_199 -466 val_466 -208 val_208 -174 val_174 -399 val_399 -396 val_396 -247 val_247 -417 val_417 -489 val_489 -162 val_162 -377 val_377 -397 val_397 -309 val_309 -365 val_365 -266 val_266 -439 val_439 -342 val_342 -367 val_367 -325 val_325 -167 val_167 -195 val_195 -475 val_475 -17 val_17 -113 val_113 -155 val_155 -203 val_203 -339 val_339 -0 val_0 -455 val_455 -128 val_128 -311 val_311 -316 val_316 -57 val_57 -302 val_302 -205 val_205 -149 val_149 -438 val_438 -345 val_345 -129 val_129 -170 val_170 -20 val_20 -489 val_489 -157 val_157 -378 val_378 -221 val_221 -92 val_92 -111 val_111 -47 val_47 -72 val_72 -4 val_4 -280 val_280 -35 val_35 -427 val_427 -277 val_277 -208 val_208 -356 val_356 -399 val_399 -169 val_169 -382 val_382 -498 val_498 -125 val_125 -386 val_386 -437 val_437 -469 val_469 -192 val_192 -286 val_286 -187 val_187 -176 val_176 -54 val_54 -459 val_459 -51 val_51 -138 val_138 -103 val_103 -239 val_239 -213 val_213 -216 val_216 -430 val_430 -278 val_278 -176 val_176 -289 val_289 -221 val_221 -65 val_65 -318 val_318 -332 val_332 -311 val_311 -275 val_275 -137 val_137 -241 val_241 -83 val_83 -333 val_333 -180 val_180 -284 val_284 -12 val_12 -230 val_230 -181 val_181 -67 val_67 -260 val_260 -404 val_404 -384 val_384 -489 val_489 -353 val_353 -373 val_373 -272 val_272 -138 val_138 -217 val_217 -84 val_84 -348 val_348 -466 val_466 -58 val_58 -8 val_8 -411 val_411 -230 val_230 -208 val_208 -348 val_348 -24 val_24 -463 val_463 -431 val_431 -179 val_179 -172 val_172 -42 val_42 -129 val_129 -158 val_158 -119 val_119 -496 val_496 -0 val_0 -322 val_322 -197 val_197 -468 val_468 -393 val_393 -454 val_454 -100 val_100 -298 val_298 -199 val_199 -191 val_191 -418 val_418 -96 val_96 -26 val_26 -165 val_165 -327 val_327 -230 val_230 -205 val_205 -120 val_120 -131 val_131 -51 val_51 -404 val_404 -43 val_43 -436 val_436 -156 val_156 -469 val_469 -468 val_468 -308 val_308 -95 val_95 -196 val_196 -288 val_288 -481 val_481 -457 val_457 -98 val_98 -282 val_282 -197 val_197 -187 val_187 -318 val_318 -318 val_318 -409 val_409 -470 val_470 -137 val_137 -369 val_369 -316 val_316 -169 val_169 -413 val_413 -85 val_85 -77 val_77 -0 val_0 -490 val_490 -87 val_87 -364 val_364 -179 val_179 -118 val_118 -134 val_134 -395 val_395 -282 val_282 -138 val_138 -238 val_238 -419 val_419 -15 val_15 -118 val_118 -72 val_72 -90 val_90 -307 val_307 -19 val_19 -435 val_435 -10 val_10 -277 val_277 -273 val_273 -306 val_306 -224 val_224 -309 val_309 -389 val_389 -327 val_327 -242 val_242 -369 val_369 -392 val_392 -272 val_272 -331 val_331 -401 val_401 -242 val_242 -452 val_452 -177 val_177 -226 val_226 -5 val_5 -497 val_497 -402 val_402 -396 val_396 -317 val_317 -395 val_395 -58 val_58 -35 val_35 -336 val_336 -95 val_95 -11 val_11 -168 val_168 -34 val_34 -229 val_229 -233 val_233 -143 val_143 -472 val_472 -322 val_322 -498 val_498 -160 val_160 -195 val_195 -42 val_42 -321 val_321 -430 val_430 -119 val_119 -489 val_489 -458 val_458 -78 val_78 -76 val_76 -41 val_41 -223 val_223 -492 val_492 -149 val_149 -449 val_449 -218 val_218 -228 val_228 -138 val_138 -453 val_453 -30 val_30 -209 val_209 -64 val_64 -468 val_468 -76 val_76 -74 val_74 -342 val_342 -69 val_69 -230 val_230 -33 val_33 -368 val_368 -103 val_103 -296 val_296 -113 val_113 -216 val_216 -367 val_367 -344 val_344 -167 val_167 -274 val_274 -219 val_219 -239 val_239 -485 val_485 -116 val_116 -223 val_223 -256 val_256 -263 val_263 -70 val_70 -487 val_487 -480 val_480 -401 val_401 -288 val_288 -191 val_191 -5 val_5 -244 val_244 -438 val_438 -128 val_128 -467 val_467 -432 val_432 -202 val_202 -316 val_316 -229 val_229 -469 val_469 -463 val_463 -280 val_280 -2 val_2 -35 val_35 -283 val_283 -331 val_331 -235 val_235 -80 val_80 -44 val_44 -193 val_193 -321 val_321 -335 val_335 -104 val_104 -466 val_466 -366 val_366 -175 val_175 -403 val_403 -483 val_483 -53 val_53 -105 val_105 -257 val_257 -406 val_406 -409 val_409 -190 val_190 -406 val_406 -401 val_401 -114 val_114 -258 val_258 -90 val_90 -203 val_203 -262 val_262 -348 val_348 -424 val_424 -12 val_12 -396 val_396 -201 val_201 -217 val_217 -164 val_164 -431 val_431 -454 val_454 -478 val_478 -298 val_298 -125 val_125 -431 val_431 -164 val_164 -424 val_424 -187 val_187 -382 val_382 -5 val_5 -70 val_70 -397 val_397 -480 val_480 -291 val_291 -24 val_24 -351 val_351 -255 val_255 -104 val_104 -70 val_70 -163 val_163 -438 val_438 -119 val_119 -414 val_414 -200 val_200 -491 val_491 -237 val_237 -439 val_439 -360 val_360 -248 val_248 -479 val_479 -305 val_305 -417 val_417 -199 val_199 -444 val_444 -120 val_120 -429 val_429 -169 val_169 -443 val_443 -323 val_323 -325 val_325 -277 val_277 -230 val_230 -478 val_478 -178 val_178 -468 val_468 -310 val_310 -317 val_317 -333 val_333 -493 val_493 -460 val_460 -207 val_207 -249 val_249 -265 val_265 -480 val_480 -83 val_83 -136 val_136 -353 val_353 -172 val_172 -214 val_214 -462 val_462 -233 val_233 -406 val_406 -133 val_133 -175 val_175 -189 val_189 -454 val_454 -375 val_375 -401 val_401 -421 val_421 -407 val_407 -384 val_384 -256 val_256 -26 val_26 -134 val_134 -67 val_67 -384 val_384 -379 val_379 -18 val_18 -462 val_462 -492 val_492 -100 val_100 -298 val_298 -9 val_9 -341 val_341 -498 val_498 -146 val_146 -458 val_458 -362 val_362 -186 val_186 -285 val_285 -348 val_348 -167 val_167 -18 val_18 -273 val_273 -183 val_183 -281 val_281 -344 val_344 -97 val_97 -469 val_469 -315 val_315 -84 val_84 -28 val_28 -37 val_37 -448 val_448 -152 val_152 -348 val_348 -307 val_307 -194 val_194 -414 val_414 -477 val_477 -222 val_222 -126 val_126 -90 val_90 -169 val_169 -403 val_403 -400 val_400 -200 val_200 -97 val_97 diff --git ql/src/test/results/clientpositive/create_genericudaf.q.out ql/src/test/results/clientpositive/create_genericudaf.q.out deleted file mode 100644 index 85d7850888..0000000000 --- ql/src/test/results/clientpositive/create_genericudaf.q.out +++ /dev/null @@ -1,107 +0,0 @@ -PREHOOK: query: EXPLAIN -CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage' -PREHOOK: type: CREATEFUNCTION -PREHOOK: Output: test_avg -POSTHOOK: query: EXPLAIN -CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage' -POSTHOOK: type: CREATEFUNCTION -POSTHOOK: Output: test_avg -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Create Function - class: org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage - name: test_avg - temporary: true - -PREHOOK: query: CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage' -PREHOOK: type: CREATEFUNCTION -PREHOOK: Output: test_avg -POSTHOOK: query: CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage' -POSTHOOK: type: CREATEFUNCTION -POSTHOOK: Output: test_avg -PREHOOK: query: EXPLAIN -SELECT - test_avg(1), - test_avg(substr(value,5)) -FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT - test_avg(1), - test_avg(substr(value,5)) -FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: substr(value, 5) (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: test_avg(1), test_avg(_col1) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: test_avg(VALUE._col0), test_avg(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT - test_avg(1), - test_avg(substr(value,5)) -FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT - test_avg(1), - test_avg(substr(value,5)) -FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -1.0 260.182 -PREHOOK: query: DROP TEMPORARY FUNCTIOn test_avg -PREHOOK: type: DROPFUNCTION -PREHOOK: Output: test_avg -POSTHOOK: query: DROP TEMPORARY FUNCTIOn test_avg -POSTHOOK: type: DROPFUNCTION -POSTHOOK: Output: test_avg diff --git ql/src/test/results/clientpositive/input14.q.out ql/src/test/results/clientpositive/input14.q.out deleted file mode 100644 index 0e61434791..0000000000 --- ql/src/test/results/clientpositive/input14.q.out +++ /dev/null @@ -1,250 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n42(key INT, value STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n42 -POSTHOOK: query: CREATE TABLE dest1_n42(key INT, value STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n42 -PREHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n42 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n42 -POSTHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n42 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n42 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 15770 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 166 Data size: 15770 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n42 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 166 Data size: 15770 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n42 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.dest1_n42 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n42 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n42 -POSTHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n42 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n42 -POSTHOOK: Lineage: dest1_n42.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest1_n42.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT dest1_n42.* FROM dest1_n42 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n42 -#### A masked pattern was here #### -POSTHOOK: query: SELECT dest1_n42.* FROM dest1_n42 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n42 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -10 val_10 -11 val_11 -12 val_12 -12 val_12 -15 val_15 -15 val_15 -17 val_17 -18 val_18 -18 val_18 -19 val_19 -2 val_2 -20 val_20 -24 val_24 -24 val_24 -26 val_26 -26 val_26 -27 val_27 -28 val_28 -30 val_30 -33 val_33 -34 val_34 -35 val_35 -35 val_35 -35 val_35 -37 val_37 -37 val_37 -4 val_4 -41 val_41 -42 val_42 -42 val_42 -43 val_43 -44 val_44 -47 val_47 -5 val_5 -5 val_5 -5 val_5 -51 val_51 -51 val_51 -53 val_53 -54 val_54 -57 val_57 -58 val_58 -58 val_58 -64 val_64 -65 val_65 -66 val_66 -67 val_67 -67 val_67 -69 val_69 -70 val_70 -70 val_70 -70 val_70 -72 val_72 -72 val_72 -74 val_74 -76 val_76 -76 val_76 -77 val_77 -78 val_78 -8 val_8 -80 val_80 -82 val_82 -83 val_83 -83 val_83 -84 val_84 -84 val_84 -85 val_85 -86 val_86 -87 val_87 -9 val_9 -90 val_90 -90 val_90 -90 val_90 -92 val_92 -95 val_95 -95 val_95 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 diff --git ql/src/test/results/clientpositive/input14_limit.q.out ql/src/test/results/clientpositive/input14_limit.q.out deleted file mode 100644 index fe9d907663..0000000000 --- ql/src/test/results/clientpositive/input14_limit.q.out +++ /dev/null @@ -1,163 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n13(key INT, value STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n13 -POSTHOOK: query: CREATE TABLE dest1_n13(key INT, value STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n13 -PREHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey LIMIT 20 -) tmap -INSERT OVERWRITE TABLE dest1_n13 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n13 -POSTHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey LIMIT 20 -) tmap -INSERT OVERWRITE TABLE dest1_n13 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n13 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 20 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 6 Data size: 1068 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n13 - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n13 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - -PREHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey LIMIT 20 -) tmap -INSERT OVERWRITE TABLE dest1_n13 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n13 -POSTHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey LIMIT 20 -) tmap -INSERT OVERWRITE TABLE dest1_n13 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n13 -POSTHOOK: Lineage: dest1_n13.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest1_n13.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT dest1_n13.* FROM dest1_n13 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n13 -#### A masked pattern was here #### -POSTHOOK: query: SELECT dest1_n13.* FROM dest1_n13 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n13 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -10 val_10 -11 val_11 diff --git ql/src/test/results/clientpositive/input17.q.out ql/src/test/results/clientpositive/input17.q.out deleted file mode 100644 index 9c03f5b0af..0000000000 --- ql/src/test/results/clientpositive/input17.q.out +++ /dev/null @@ -1,174 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n81(key INT, value STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n81 -POSTHOOK: query: CREATE TABLE dest1_n81(key INT, value STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n81 -PREHOOK: query: EXPLAIN -FROM ( - FROM src_thrift - SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n81 SELECT tmap.tkey, tmap.tvalue -PREHOOK: type: QUERY -PREHOOK: Input: default@src_thrift -PREHOOK: Output: default@dest1_n81 -POSTHOOK: query: EXPLAIN -FROM ( - FROM src_thrift - SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n81 SELECT tmap.tkey, tmap.tvalue -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src_thrift -POSTHOOK: Output: default@dest1_n81 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_thrift - Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (aint + lint[0]) (type: int), lintstring[0] (type: struct) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n81 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3548 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n81 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.dest1_n81 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 3548 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3564 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 3564 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: FROM ( - FROM src_thrift - SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n81 SELECT tmap.tkey, tmap.tvalue -PREHOOK: type: QUERY -PREHOOK: Input: default@src_thrift -PREHOOK: Output: default@dest1_n81 -POSTHOOK: query: FROM ( - FROM src_thrift - SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n81 SELECT tmap.tkey, tmap.tvalue -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src_thrift -POSTHOOK: Output: default@dest1_n81 -POSTHOOK: Lineage: dest1_n81.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] -POSTHOOK: Lineage: dest1_n81.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] -PREHOOK: query: SELECT dest1_n81.* FROM dest1_n81 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n81 -#### A masked pattern was here #### -POSTHOOK: query: SELECT dest1_n81.* FROM dest1_n81 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n81 -#### A masked pattern was here #### --1461153966 {"myint":49,"mystring":"343","underscore_int":7} --1952710705 {"myint":25,"mystring":"125","underscore_int":5} --734328905 {"myint":16,"mystring":"64","underscore_int":4} --751827636 {"myint":4,"mystring":"8","underscore_int":2} -1244525196 {"myint":36,"mystring":"216","underscore_int":6} -1638581586 {"myint":64,"mystring":"512","underscore_int":8} -1712634731 {"myint":0,"mystring":"0","underscore_int":0} -336964422 {"myint":81,"mystring":"729","underscore_int":9} -465985201 {"myint":1,"mystring":"1","underscore_int":1} -477111225 {"myint":9,"mystring":"27","underscore_int":3} -NULL NULL diff --git ql/src/test/results/clientpositive/input18.q.out ql/src/test/results/clientpositive/input18.q.out deleted file mode 100644 index ce731e6b2b..0000000000 --- ql/src/test/results/clientpositive/input18.q.out +++ /dev/null @@ -1,250 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n124(key INT, value STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n124 -POSTHOOK: query: CREATE TABLE dest1_n124(key INT, value STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n124 -PREHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -INSERT OVERWRITE TABLE dest1_n124 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n124 -POSTHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -INSERT OVERWRITE TABLE dest1_n124 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n124 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), 3 (type: int), 7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), regexp_replace(VALUE._col1, ' ', '+') (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 31208 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 166 Data size: 31208 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n124 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 166 Data size: 31208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n124 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.dest1_n124 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -INSERT OVERWRITE TABLE dest1_n124 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1_n124 -POSTHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -INSERT OVERWRITE TABLE dest1_n124 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1_n124 -POSTHOOK: Lineage: dest1_n124.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest1_n124.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT dest1_n124.* FROM dest1_n124 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n124 -#### A masked pattern was here #### -POSTHOOK: query: SELECT dest1_n124.* FROM dest1_n124 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n124 -#### A masked pattern was here #### -0 val_0+3+7 -0 val_0+3+7 -0 val_0+3+7 -10 val_10+3+7 -11 val_11+3+7 -12 val_12+3+7 -12 val_12+3+7 -15 val_15+3+7 -15 val_15+3+7 -17 val_17+3+7 -18 val_18+3+7 -18 val_18+3+7 -19 val_19+3+7 -2 val_2+3+7 -20 val_20+3+7 -24 val_24+3+7 -24 val_24+3+7 -26 val_26+3+7 -26 val_26+3+7 -27 val_27+3+7 -28 val_28+3+7 -30 val_30+3+7 -33 val_33+3+7 -34 val_34+3+7 -35 val_35+3+7 -35 val_35+3+7 -35 val_35+3+7 -37 val_37+3+7 -37 val_37+3+7 -4 val_4+3+7 -41 val_41+3+7 -42 val_42+3+7 -42 val_42+3+7 -43 val_43+3+7 -44 val_44+3+7 -47 val_47+3+7 -5 val_5+3+7 -5 val_5+3+7 -5 val_5+3+7 -51 val_51+3+7 -51 val_51+3+7 -53 val_53+3+7 -54 val_54+3+7 -57 val_57+3+7 -58 val_58+3+7 -58 val_58+3+7 -64 val_64+3+7 -65 val_65+3+7 -66 val_66+3+7 -67 val_67+3+7 -67 val_67+3+7 -69 val_69+3+7 -70 val_70+3+7 -70 val_70+3+7 -70 val_70+3+7 -72 val_72+3+7 -72 val_72+3+7 -74 val_74+3+7 -76 val_76+3+7 -76 val_76+3+7 -77 val_77+3+7 -78 val_78+3+7 -8 val_8+3+7 -80 val_80+3+7 -82 val_82+3+7 -83 val_83+3+7 -83 val_83+3+7 -84 val_84+3+7 -84 val_84+3+7 -85 val_85+3+7 -86 val_86+3+7 -87 val_87+3+7 -9 val_9+3+7 -90 val_90+3+7 -90 val_90+3+7 -90 val_90+3+7 -92 val_92+3+7 -95 val_95+3+7 -95 val_95+3+7 -96 val_96+3+7 -97 val_97+3+7 -97 val_97+3+7 -98 val_98+3+7 -98 val_98+3+7 diff --git ql/src/test/results/clientpositive/input5.q.out ql/src/test/results/clientpositive/input5.q.out deleted file mode 100644 index becfc1876a..0000000000 --- ql/src/test/results/clientpositive/input5.q.out +++ /dev/null @@ -1,174 +0,0 @@ -PREHOOK: query: CREATE TABLE dest1_n94(key STRING, value STRING) STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1_n94 -POSTHOOK: query: CREATE TABLE dest1_n94(key STRING, value STRING) STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1_n94 -PREHOOK: query: EXPLAIN -FROM ( - FROM src_thrift - SELECT TRANSFORM(src_thrift.lint, src_thrift.lintstring) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n94 SELECT tmap.tkey, tmap.tvalue -PREHOOK: type: QUERY -PREHOOK: Input: default@src_thrift -PREHOOK: Output: default@dest1_n94 -POSTHOOK: query: EXPLAIN -FROM ( - FROM src_thrift - SELECT TRANSFORM(src_thrift.lint, src_thrift.lintstring) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n94 SELECT tmap.tkey, tmap.tvalue -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src_thrift -POSTHOOK: Output: default@dest1_n94 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src_thrift - Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: lint (type: array), lintstring (type: array>) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n94 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n94 - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.dest1_n94 - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: FROM ( - FROM src_thrift - SELECT TRANSFORM(src_thrift.lint, src_thrift.lintstring) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n94 SELECT tmap.tkey, tmap.tvalue -PREHOOK: type: QUERY -PREHOOK: Input: default@src_thrift -PREHOOK: Output: default@dest1_n94 -POSTHOOK: query: FROM ( - FROM src_thrift - SELECT TRANSFORM(src_thrift.lint, src_thrift.lintstring) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1_n94 SELECT tmap.tkey, tmap.tvalue -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src_thrift -POSTHOOK: Output: default@dest1_n94 -POSTHOOK: Lineage: dest1_n94.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] -POSTHOOK: Lineage: dest1_n94.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] -PREHOOK: query: SELECT dest1_n94.* FROM dest1_n94 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1_n94 -#### A masked pattern was here #### -POSTHOOK: query: SELECT dest1_n94.* FROM dest1_n94 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1_n94 -#### A masked pattern was here #### -NULL NULL -[0,0,0] [{"myint":0,"mystring":"0","underscore_int":0}] -[1,2,3] [{"myint":1,"mystring":"1","underscore_int":1}] -[2,4,6] [{"myint":4,"mystring":"8","underscore_int":2}] -[3,6,9] [{"myint":9,"mystring":"27","underscore_int":3}] -[4,8,12] [{"myint":16,"mystring":"64","underscore_int":4}] -[5,10,15] [{"myint":25,"mystring":"125","underscore_int":5}] -[6,12,18] [{"myint":36,"mystring":"216","underscore_int":6}] -[7,14,21] [{"myint":49,"mystring":"343","underscore_int":7}] -[8,16,24] [{"myint":64,"mystring":"512","underscore_int":8}] -[9,18,27] [{"myint":81,"mystring":"729","underscore_int":9}] diff --git ql/src/test/results/clientpositive/insert_into3.q.out ql/src/test/results/clientpositive/insert_into3.q.out deleted file mode 100644 index 60fd42d6fe..0000000000 --- ql/src/test/results/clientpositive/insert_into3.q.out +++ /dev/null @@ -1,568 +0,0 @@ -PREHOOK: query: DROP TABLE insert_into3a -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE insert_into3a -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE insert_into3b -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE insert_into3b -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE insert_into3a (key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert_into3a -POSTHOOK: query: CREATE TABLE insert_into3a (key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert_into3a -PREHOOK: query: CREATE TABLE insert_into3b (key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert_into3b -POSTHOOK: query: CREATE TABLE insert_into3b (key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert_into3b -PREHOOK: query: EXPLAIN FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 - INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into3a -PREHOOK: Output: default@insert_into3b -POSTHOOK: query: EXPLAIN FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 - INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into3a -POSTHOOK: Output: default@insert_into3b -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0, Stage-4, Stage-7 - Stage-4 depends on stages: Stage-2 - Stage-6 depends on stages: Stage-1, Stage-4, Stage-7 - Stage-5 depends on stages: Stage-2 - Stage-1 depends on stages: Stage-5 - Stage-7 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 50 - Statistics: Num rows: 50 Data size: 8900 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 50 Data size: 4750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 50 Data size: 4750 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into3a - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 50 Data size: 4750 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into3a - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into3a - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-6 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into3b - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 100 - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into3b - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-1 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into3b - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 - INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into3a -PREHOOK: Output: default@insert_into3b -POSTHOOK: query: FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 - INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into3a -POSTHOOK: Output: default@insert_into3b -POSTHOOK: Lineage: insert_into3a.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into3a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into3b.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into3b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3a -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into3a -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3a -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into3a -#### A masked pattern was here #### --1254133670 -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3b -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into3b -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3b -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into3b -#### A masked pattern was here #### --1142373758 -PREHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 - INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into3a -PREHOOK: Output: default@insert_into3b -POSTHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 - INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into3a -POSTHOOK: Output: default@insert_into3b -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0, Stage-4, Stage-7 - Stage-4 depends on stages: Stage-2 - Stage-6 depends on stages: Stage-1, Stage-4, Stage-7 - Stage-5 depends on stages: Stage-2 - Stage-1 depends on stages: Stage-5 - Stage-7 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into3a - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into3a - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into3a - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-6 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into3b - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into3b - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-1 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into3b - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 - INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into3a -PREHOOK: Output: default@insert_into3b -POSTHOOK: query: FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 - INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into3a -POSTHOOK: Output: default@insert_into3b -POSTHOOK: Lineage: insert_into3a.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into3a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into3b.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into3b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3a -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into3a -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3a -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into3a -#### A masked pattern was here #### --826625916 -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3b -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into3b -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3b -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into3b -#### A masked pattern was here #### --1968999674 -PREHOOK: query: DROP TABLE insert_into3a -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@insert_into3a -PREHOOK: Output: default@insert_into3a -POSTHOOK: query: DROP TABLE insert_into3a -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@insert_into3a -POSTHOOK: Output: default@insert_into3a -PREHOOK: query: DROP TABLE insert_into3b -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@insert_into3b -PREHOOK: Output: default@insert_into3b -POSTHOOK: query: DROP TABLE insert_into3b -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@insert_into3b -POSTHOOK: Output: default@insert_into3b diff --git ql/src/test/results/clientpositive/insert_into4.q.out ql/src/test/results/clientpositive/insert_into4.q.out deleted file mode 100644 index 031d562a43..0000000000 --- ql/src/test/results/clientpositive/insert_into4.q.out +++ /dev/null @@ -1,460 +0,0 @@ -PREHOOK: query: DROP TABLE insert_into4a -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE insert_into4a -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE insert_into4b -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE insert_into4b -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE insert_into4a (key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert_into4a -POSTHOOK: query: CREATE TABLE insert_into4a (key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert_into4a -PREHOOK: query: CREATE TABLE insert_into4b (key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert_into4b -POSTHOOK: query: CREATE TABLE insert_into4b (key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert_into4b -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into4a -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into4a -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into4a - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into4a - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into4a - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into4a -POSTHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into4a -POSTHOOK: Lineage: insert_into4a.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into4a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4a -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into4a -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4a -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into4a -#### A masked pattern was here #### --826625916 -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into4a -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into4a -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into4a - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into4a - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into4a - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into4a -POSTHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into4a -POSTHOOK: Lineage: insert_into4a.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into4a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4a -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into4a -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4a -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into4a -#### A masked pattern was here #### --1653251832 -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into4a -PREHOOK: Output: default@insert_into4b -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into4a -POSTHOOK: Output: default@insert_into4b -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: insert_into4a - Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into4b - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into4b - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into4b - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into4b - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into4b - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into4a -PREHOOK: Output: default@insert_into4b -POSTHOOK: query: INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into4a -POSTHOOK: Output: default@insert_into4b -POSTHOOK: Lineage: insert_into4b.key SIMPLE [(insert_into4a)insert_into4a.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: insert_into4b.value SIMPLE [(insert_into4a)insert_into4a.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4b -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into4b -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4b -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into4b -#### A masked pattern was here #### --1653251832 -PREHOOK: query: DROP TABLE insert_into4a -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@insert_into4a -PREHOOK: Output: default@insert_into4a -POSTHOOK: query: DROP TABLE insert_into4a -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@insert_into4a -POSTHOOK: Output: default@insert_into4a -PREHOOK: query: DROP TABLE insert_into4b -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@insert_into4b -PREHOOK: Output: default@insert_into4b -POSTHOOK: query: DROP TABLE insert_into4b -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@insert_into4b -POSTHOOK: Output: default@insert_into4b diff --git ql/src/test/results/clientpositive/insert_into5.q.out ql/src/test/results/clientpositive/insert_into5.q.out deleted file mode 100644 index 8ca94ee136..0000000000 --- ql/src/test/results/clientpositive/insert_into5.q.out +++ /dev/null @@ -1,637 +0,0 @@ -PREHOOK: query: DROP TABLE insert_into5a -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE insert_into5a -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE insert_into5b -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE insert_into5b -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE insert_into5a (key int, value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert_into5a -POSTHOOK: query: CREATE TABLE insert_into5a (key int, value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert_into5a -PREHOOK: query: CREATE TABLE insert_into5b (key int, value string) PARTITIONED BY (ds string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert_into5b -POSTHOOK: query: CREATE TABLE insert_into5b (key int, value string) PARTITIONED BY (ds string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert_into5b -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into5a -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into5a -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int), 'one' (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: int), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5a - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5a - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into5a - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into5a -POSTHOOK: query: INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into5a -POSTHOOK: Lineage: insert_into5a.key SIMPLE [] -POSTHOOK: Lineage: insert_into5a.value SIMPLE [] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5a -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5a -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5a -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5a -#### A masked pattern was here #### -481928560 -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5a -PREHOOK: Output: default@insert_into5a -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5a -POSTHOOK: Output: default@insert_into5a -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: insert_into5a - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5a - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5a - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into5a - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5a - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5a - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5a -PREHOOK: Output: default@insert_into5a -POSTHOOK: query: INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5a -POSTHOOK: Output: default@insert_into5a -POSTHOOK: Lineage: insert_into5a.key SIMPLE [(insert_into5a)insert_into5a.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: insert_into5a.value SIMPLE [(insert_into5a)insert_into5a.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5a -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5a -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5a -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5a -#### A masked pattern was here #### -963857120 -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5b PARTITION (ds='1') - SELECT * FROM insert_into5a -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5a -PREHOOK: Output: default@insert_into5b@ds=1 -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5b PARTITION (ds='1') - SELECT * FROM insert_into5a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5a -POSTHOOK: Output: default@insert_into5b@ds=1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: insert_into5a - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5b - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string) - outputColumnNames: key, value, ds - Statistics: Num rows: 20 Data size: 3520 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: struct), _col2 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - partition: - ds 1 - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5b - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into5b - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5b - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5b - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') SELECT * FROM insert_into5a -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5a -PREHOOK: Output: default@insert_into5b@ds=1 -POSTHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') SELECT * FROM insert_into5a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5a -POSTHOOK: Output: default@insert_into5b@ds=1 -POSTHOOK: Lineage: insert_into5b PARTITION(ds=1).key SIMPLE [(insert_into5a)insert_into5a.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: insert_into5b PARTITION(ds=1).value SIMPLE [(insert_into5a)insert_into5a.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5b -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5b -PREHOOK: Input: default@insert_into5b@ds=1 -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5b -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5b -POSTHOOK: Input: default@insert_into5b@ds=1 -#### A masked pattern was here #### --18626052920 -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5b PARTITION (ds='1') - SELECT key, value FROM insert_into5b -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5b -PREHOOK: Input: default@insert_into5b@ds=1 -PREHOOK: Output: default@insert_into5b@ds=1 -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5b PARTITION (ds='1') - SELECT key, value FROM insert_into5b -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5b -POSTHOOK: Input: default@insert_into5b@ds=1 -POSTHOOK: Output: default@insert_into5b@ds=1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: insert_into5b - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5b - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string) - outputColumnNames: key, value, ds - Statistics: Num rows: 20 Data size: 3520 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: struct), _col2 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - partition: - ds 1 - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5b - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into5b - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5b - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into5b - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') - SELECT key, value FROM insert_into5b -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5b -PREHOOK: Input: default@insert_into5b@ds=1 -PREHOOK: Output: default@insert_into5b@ds=1 -POSTHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') - SELECT key, value FROM insert_into5b -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5b -POSTHOOK: Input: default@insert_into5b@ds=1 -POSTHOOK: Output: default@insert_into5b@ds=1 -POSTHOOK: Lineage: insert_into5b PARTITION(ds=1).key SIMPLE [(insert_into5b)insert_into5b.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: insert_into5b PARTITION(ds=1).value SIMPLE [(insert_into5b)insert_into5b.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5b -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into5b -PREHOOK: Input: default@insert_into5b@ds=1 -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5b -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into5b -POSTHOOK: Input: default@insert_into5b@ds=1 -#### A masked pattern was here #### --37252105840 -PREHOOK: query: DROP TABLE insert_into5a -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@insert_into5a -PREHOOK: Output: default@insert_into5a -POSTHOOK: query: DROP TABLE insert_into5a -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@insert_into5a -POSTHOOK: Output: default@insert_into5a diff --git ql/src/test/results/clientpositive/insert_into6.q.out ql/src/test/results/clientpositive/insert_into6.q.out deleted file mode 100644 index 2c6cab53e6..0000000000 --- ql/src/test/results/clientpositive/insert_into6.q.out +++ /dev/null @@ -1,383 +0,0 @@ -PREHOOK: query: DROP TABLE insert_into6a -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE insert_into6a -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE insert_into6b -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE insert_into6b -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE insert_into6a (key int, value string) PARTITIONED BY (ds string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert_into6a -POSTHOOK: query: CREATE TABLE insert_into6a (key int, value string) PARTITIONED BY (ds string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert_into6a -PREHOOK: query: CREATE TABLE insert_into6b (key int, value string) PARTITIONED BY (ds string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert_into6b -POSTHOOK: query: CREATE TABLE insert_into6b (key int, value string) PARTITIONED BY (ds string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert_into6b -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into6a PARTITION (ds='1') - SELECT * FROM src LIMIT 150 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into6a@ds=1 -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into6a PARTITION (ds='1') - SELECT * FROM src LIMIT 150 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into6a@ds=1 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 150 - Statistics: Num rows: 150 Data size: 26700 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 150 Data size: 26700 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 150 Data size: 26700 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 150 - Statistics: Num rows: 150 Data size: 26700 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 150 Data size: 14250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 150 Data size: 14250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6a - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string) - outputColumnNames: key, value, ds - Statistics: Num rows: 150 Data size: 27000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - partition: - ds 1 - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6a - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into6a - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: struct), _col2 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='1') SELECT * FROM src LIMIT 150 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into6a@ds=1 -POSTHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='1') SELECT * FROM src LIMIT 150 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into6a@ds=1 -POSTHOOK: Lineage: insert_into6a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into6a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='2') SELECT * FROM src LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert_into6a@ds=2 -POSTHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='2') SELECT * FROM src LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert_into6a@ds=2 -POSTHOOK: Lineage: insert_into6a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: insert_into6a PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into6a -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into6a -PREHOOK: Input: default@insert_into6a@ds=1 -PREHOOK: Input: default@insert_into6a@ds=2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into6a -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into6a -POSTHOOK: Input: default@insert_into6a@ds=1 -POSTHOOK: Input: default@insert_into6a@ds=2 -#### A masked pattern was here #### --35226404960 -PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into6b PARTITION (ds) - SELECT * FROM insert_into6a -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into6a -PREHOOK: Input: default@insert_into6a@ds=1 -PREHOOK: Input: default@insert_into6a@ds=2 -PREHOOK: Output: default@insert_into6b -POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into6b PARTITION (ds) - SELECT * FROM insert_into6a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into6a -POSTHOOK: Input: default@insert_into6a@ds=1 -POSTHOOK: Input: default@insert_into6a@ds=2 -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: insert_into6a - Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6b - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, value, ds - Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 2096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 2096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: struct), _col2 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 2128 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 2128 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 2128 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - partition: - ds - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6b - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, value - Column Types: int, string - Table: default.insert_into6b - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6b - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6b - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - -PREHOOK: query: INSERT INTO TABLE insert_into6b PARTITION (ds) SELECT * FROM insert_into6a -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into6a -PREHOOK: Input: default@insert_into6a@ds=1 -PREHOOK: Input: default@insert_into6a@ds=2 -PREHOOK: Output: default@insert_into6b -POSTHOOK: query: INSERT INTO TABLE insert_into6b PARTITION (ds) SELECT * FROM insert_into6a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into6a -POSTHOOK: Input: default@insert_into6a@ds=1 -POSTHOOK: Input: default@insert_into6a@ds=2 -POSTHOOK: Output: default@insert_into6b@ds=1 -POSTHOOK: Output: default@insert_into6b@ds=2 -POSTHOOK: Lineage: insert_into6b PARTITION(ds=1).key SIMPLE [(insert_into6a)insert_into6a.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: insert_into6b PARTITION(ds=1).value SIMPLE [(insert_into6a)insert_into6a.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: insert_into6b PARTITION(ds=2).key SIMPLE [(insert_into6a)insert_into6a.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: insert_into6b PARTITION(ds=2).value SIMPLE [(insert_into6a)insert_into6a.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into6b -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert_into6b -PREHOOK: Input: default@insert_into6b@ds=1 -PREHOOK: Input: default@insert_into6b@ds=2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into6b -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert_into6b -POSTHOOK: Input: default@insert_into6b@ds=1 -POSTHOOK: Input: default@insert_into6b@ds=2 -#### A masked pattern was here #### --35226404960 -PREHOOK: query: SHOW PARTITIONS insert_into6b -PREHOOK: type: SHOWPARTITIONS -PREHOOK: Input: default@insert_into6b -POSTHOOK: query: SHOW PARTITIONS insert_into6b -POSTHOOK: type: SHOWPARTITIONS -POSTHOOK: Input: default@insert_into6b -ds=1 -ds=2 -PREHOOK: query: DROP TABLE insert_into6a -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@insert_into6a -PREHOOK: Output: default@insert_into6a -POSTHOOK: query: DROP TABLE insert_into6a -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@insert_into6a -POSTHOOK: Output: default@insert_into6a -PREHOOK: query: DROP TABLE insert_into6b -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@insert_into6b -PREHOOK: Output: default@insert_into6b -POSTHOOK: query: DROP TABLE insert_into6b -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@insert_into6b -POSTHOOK: Output: default@insert_into6b diff --git ql/src/test/results/clientpositive/autoColumnStats_6.q.out ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out similarity index 71% rename from ql/src/test/results/clientpositive/autoColumnStats_6.q.out rename to ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out index ff708cb6b0..b674164063 100644 --- ql/src/test/results/clientpositive/autoColumnStats_6.q.out +++ ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out @@ -35,83 +35,87 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 10) AS STRING) (type: string), CAST( (hash(value) pmod 10) AS STRING) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, one, two, three - Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: one (type: string), two (type: string), three (type: string) - minReductionHashAggr: 0.99 - mode: hash + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), CAST( (hash(key) pmod 10) AS STRING) (type: string), CAST( (hash(value) pmod 10) AS STRING) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: key, value, one, two, three + Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: one (type: string), two (type: string), three (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + null sort order: zzz + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: struct), _col4 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: struct), _col4 (type: struct), _col0 (type: string), _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 250 Data size: 329250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - null sort order: zzz - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 250 Data size: 329250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: struct), _col4 (type: struct) - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 250 Data size: 333250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: struct), _col4 (type: struct), _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 250 Data size: 333250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 333250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: string), _col3 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col2 (type: string), _col3 (type: string) - Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 500 Data size: 231500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge2a + Dependency Collection Stage: Stage-0 Move Operator @@ -209,7 +213,7 @@ POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=1).value SIMPLE [ POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items -#### A masked pattern was here #### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 351 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge2a @@ -236,7 +240,7 @@ PREHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=0 PREHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=6 PREHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=1 PREHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge2a @@ -263,7 +267,7 @@ POSTHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=0 POSTHOOK: Input: default@orcfile_merge2a@one=1/two=8/three=6 POSTHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=1 POSTHOOK: Input: default@orcfile_merge2a@one=1/two=9/three=7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### -4209012844 PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value, '1', PMOD(HASH(key), 10), @@ -272,7 +276,7 @@ PREHOOK: query: SELECT SUM(HASH(c)) FROM ( ) t PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value, '1', PMOD(HASH(key), 10), PMOD(HASH(value), 10)) USING 'tr \t _' AS (c) @@ -280,7 +284,7 @@ POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( ) t POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### -4209012844 PREHOOK: query: DROP TABLE orcfile_merge2a PREHOOK: type: DROPTABLE diff --git ql/src/test/results/clientpositive/autogen_colalias.q.out ql/src/test/results/clientpositive/llap/autogen_colalias.q.out similarity index 100% rename from ql/src/test/results/clientpositive/autogen_colalias.q.out rename to ql/src/test/results/clientpositive/llap/autogen_colalias.q.out diff --git ql/src/test/results/clientpositive/llap/binary_output_format.q.out ql/src/test/results/clientpositive/llap/binary_output_format.q.out new file mode 100644 index 0000000000..f99b82315d --- /dev/null +++ ql/src/test/results/clientpositive/llap/binary_output_format.q.out @@ -0,0 +1,818 @@ +PREHOOK: query: CREATE TABLE dest1_n109(mydata STRING) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'serialization.last.column.takes.rest'='true' +) +STORED AS + INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n109 +POSTHOOK: query: CREATE TABLE dest1_n109(mydata STRING) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'serialization.last.column.takes.rest'='true' +) +STORED AS + INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n109 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1_n109 +SELECT TRANSFORM(*) + USING 'cat' + AS mydata STRING + ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' + WITH SERDEPROPERTIES ( + 'serialization.last.column.takes.rest'='true' + ) + RECORDREADER 'org.apache.hadoop.hive.ql.exec.BinaryRecordReader' +FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n109 +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1_n109 +SELECT TRANSFORM(*) + USING 'cat' + AS mydata STRING + ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' + WITH SERDEPROPERTIES ( + 'serialization.last.column.takes.rest'='true' + ) + RECORDREADER 'org.apache.hadoop.hive.ql.exec.BinaryRecordReader' +FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n109 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucketing_version -1 + columns _col0 + columns.types string + field.delim 9 + serialization.format 9 + serialization.last.column.takes.rest true + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 1 + directory: hdfs://### HDFS PATH ### + NumFilesPerFileSink: 1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns mydata + columns.comments + columns.types string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.dest1_n109 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct dest1_n109 { string mydata} + serialization.format 1 + serialization.last.column.takes.rest true + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n109 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Select Operator + expressions: _col0 (type: string) + outputColumnNames: mydata + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(mydata, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + null sort order: + numBuckets: -1 + sort order: + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: struct) + auto parallelism: false + Path -> Alias: + hdfs://### HDFS PATH ### [src] + Path -> Partition: + hdfs://### HDFS PATH ### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 + directory: hdfs://### HDFS PATH ### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0 + columns.types struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + source: hdfs://### HDFS PATH ### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns mydata + columns.comments + columns.types string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.dest1_n109 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct dest1_n109 { string mydata} + serialization.format 1 + serialization.last.column.takes.rest true + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n109 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: hdfs://### HDFS PATH ### + Column Stats Desc: + Columns: mydata + Column Types: string + Table: default.dest1_n109 + Is Table Level Stats: true + +PREHOOK: query: INSERT OVERWRITE TABLE dest1_n109 +SELECT TRANSFORM(*) + USING 'cat' + AS mydata STRING + ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' + WITH SERDEPROPERTIES ( + 'serialization.last.column.takes.rest'='true' + ) + RECORDREADER 'org.apache.hadoop.hive.ql.exec.BinaryRecordReader' +FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n109 +POSTHOOK: query: INSERT OVERWRITE TABLE dest1_n109 +SELECT TRANSFORM(*) + USING 'cat' + AS mydata STRING + ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' + WITH SERDEPROPERTIES ( + 'serialization.last.column.takes.rest'='true' + ) + RECORDREADER 'org.apache.hadoop.hive.ql.exec.BinaryRecordReader' +FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n109 +POSTHOOK: Lineage: dest1_n109.mydata SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT * FROM dest1_n109 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n109 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM dest1_n109 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n109 +POSTHOOK: Output: hdfs://### HDFS PATH ### +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +409 val_409 +255 val_255 +278 val_278 +98 val_98 +484 val_484 +265 val_265 +193 val_193 +401 val_401 +150 val_150 +273 val_273 +224 val_224 +369 val_369 +66 val_66 +128 val_128 +213 val_213 +146 val_146 +406 val_406 +429 val_429 +374 val_374 +152 val_152 +469 val_469 +145 val_145 +495 val_495 +37 val_37 +327 val_327 +281 val_281 +277 val_277 +209 val_209 +15 val_15 +82 val_82 +403 val_403 +166 val_166 +417 val_417 +430 val_430 +252 val_252 +292 val_292 +219 val_219 +287 val_287 +153 val_153 +193 val_193 +338 val_338 +446 val_446 +459 val_459 +394 val_394 +237 val_237 +482 val_482 +174 val_174 +413 val_413 +494 val_494 +207 val_207 +199 val_199 +466 val_466 +208 val_208 +174 val_174 +399 val_399 +396 val_396 +247 val_247 +417 val_417 +489 val_489 +162 val_162 +377 val_377 +397 val_397 +309 val_309 +365 val_365 +266 val_266 +439 val_439 +342 val_342 +367 val_367 +325 val_325 +167 val_167 +195 val_195 +475 val_475 +17 val_17 +113 val_113 +155 val_155 +203 val_203 +339 val_339 +0 val_0 +455 val_455 +128 val_128 +311 val_311 +316 val_316 +57 val_57 +302 val_302 +205 val_205 +149 val_149 +438 val_438 +345 val_345 +129 val_129 +170 val_170 +20 val_20 +489 val_489 +157 val_157 +378 val_378 +221 val_221 +92 val_92 +111 val_111 +47 val_47 +72 val_72 +4 val_4 +280 val_280 +35 val_35 +427 val_427 +277 val_277 +208 val_208 +356 val_356 +399 val_399 +169 val_169 +382 val_382 +498 val_498 +125 val_125 +386 val_386 +437 val_437 +469 val_469 +192 val_192 +286 val_286 +187 val_187 +176 val_176 +54 val_54 +459 val_459 +51 val_51 +138 val_138 +103 val_103 +239 val_239 +213 val_213 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +289 val_289 +221 val_221 +65 val_65 +318 val_318 +332 val_332 +311 val_311 +275 val_275 +137 val_137 +241 val_241 +83 val_83 +333 val_333 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +181 val_181 +67 val_67 +260 val_260 +404 val_404 +384 val_384 +489 val_489 +353 val_353 +373 val_373 +272 val_272 +138 val_138 +217 val_217 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +411 val_411 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +463 val_463 +431 val_431 +179 val_179 +172 val_172 +42 val_42 +129 val_129 +158 val_158 +119 val_119 +496 val_496 +0 val_0 +322 val_322 +197 val_197 +468 val_468 +393 val_393 +454 val_454 +100 val_100 +298 val_298 +199 val_199 +191 val_191 +418 val_418 +96 val_96 +26 val_26 +165 val_165 +327 val_327 +230 val_230 +205 val_205 +120 val_120 +131 val_131 +51 val_51 +404 val_404 +43 val_43 +436 val_436 +156 val_156 +469 val_469 +468 val_468 +308 val_308 +95 val_95 +196 val_196 +288 val_288 +481 val_481 +457 val_457 +98 val_98 +282 val_282 +197 val_197 +187 val_187 +318 val_318 +318 val_318 +409 val_409 +470 val_470 +137 val_137 +369 val_369 +316 val_316 +169 val_169 +413 val_413 +85 val_85 +77 val_77 +0 val_0 +490 val_490 +87 val_87 +364 val_364 +179 val_179 +118 val_118 +134 val_134 +395 val_395 +282 val_282 +138 val_138 +238 val_238 +419 val_419 +15 val_15 +118 val_118 +72 val_72 +90 val_90 +307 val_307 +19 val_19 +435 val_435 +10 val_10 +277 val_277 +273 val_273 +306 val_306 +224 val_224 +309 val_309 +389 val_389 +327 val_327 +242 val_242 +369 val_369 +392 val_392 +272 val_272 +331 val_331 +401 val_401 +242 val_242 +452 val_452 +177 val_177 +226 val_226 +5 val_5 +497 val_497 +402 val_402 +396 val_396 +317 val_317 +395 val_395 +58 val_58 +35 val_35 +336 val_336 +95 val_95 +11 val_11 +168 val_168 +34 val_34 +229 val_229 +233 val_233 +143 val_143 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +195 val_195 +42 val_42 +321 val_321 +430 val_430 +119 val_119 +489 val_489 +458 val_458 +78 val_78 +76 val_76 +41 val_41 +223 val_223 +492 val_492 +149 val_149 +449 val_449 +218 val_218 +228 val_228 +138 val_138 +453 val_453 +30 val_30 +209 val_209 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +69 val_69 +230 val_230 +33 val_33 +368 val_368 +103 val_103 +296 val_296 +113 val_113 +216 val_216 +367 val_367 +344 val_344 +167 val_167 +274 val_274 +219 val_219 +239 val_239 +485 val_485 +116 val_116 +223 val_223 +256 val_256 +263 val_263 +70 val_70 +487 val_487 +480 val_480 +401 val_401 +288 val_288 +191 val_191 +5 val_5 +244 val_244 +438 val_438 +128 val_128 +467 val_467 +432 val_432 +202 val_202 +316 val_316 +229 val_229 +469 val_469 +463 val_463 +280 val_280 +2 val_2 +35 val_35 +283 val_283 +331 val_331 +235 val_235 +80 val_80 +44 val_44 +193 val_193 +321 val_321 +335 val_335 +104 val_104 +466 val_466 +366 val_366 +175 val_175 +403 val_403 +483 val_483 +53 val_53 +105 val_105 +257 val_257 +406 val_406 +409 val_409 +190 val_190 +406 val_406 +401 val_401 +114 val_114 +258 val_258 +90 val_90 +203 val_203 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +201 val_201 +217 val_217 +164 val_164 +431 val_431 +454 val_454 +478 val_478 +298 val_298 +125 val_125 +431 val_431 +164 val_164 +424 val_424 +187 val_187 +382 val_382 +5 val_5 +70 val_70 +397 val_397 +480 val_480 +291 val_291 +24 val_24 +351 val_351 +255 val_255 +104 val_104 +70 val_70 +163 val_163 +438 val_438 +119 val_119 +414 val_414 +200 val_200 +491 val_491 +237 val_237 +439 val_439 +360 val_360 +248 val_248 +479 val_479 +305 val_305 +417 val_417 +199 val_199 +444 val_444 +120 val_120 +429 val_429 +169 val_169 +443 val_443 +323 val_323 +325 val_325 +277 val_277 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +317 val_317 +333 val_333 +493 val_493 +460 val_460 +207 val_207 +249 val_249 +265 val_265 +480 val_480 +83 val_83 +136 val_136 +353 val_353 +172 val_172 +214 val_214 +462 val_462 +233 val_233 +406 val_406 +133 val_133 +175 val_175 +189 val_189 +454 val_454 +375 val_375 +401 val_401 +421 val_421 +407 val_407 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +67 val_67 +384 val_384 +379 val_379 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +9 val_9 +341 val_341 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +285 val_285 +348 val_348 +167 val_167 +18 val_18 +273 val_273 +183 val_183 +281 val_281 +344 val_344 +97 val_97 +469 val_469 +315 val_315 +84 val_84 +28 val_28 +37 val_37 +448 val_448 +152 val_152 +348 val_348 +307 val_307 +194 val_194 +414 val_414 +477 val_477 +222 val_222 +126 val_126 +90 val_90 +169 val_169 +403 val_403 +400 val_400 +200 val_200 +97 val_97 diff --git ql/src/test/results/clientpositive/llap/create_genericudaf.q.out ql/src/test/results/clientpositive/llap/create_genericudaf.q.out new file mode 100644 index 0000000000..a87f04a79f --- /dev/null +++ ql/src/test/results/clientpositive/llap/create_genericudaf.q.out @@ -0,0 +1,114 @@ +PREHOOK: query: EXPLAIN +CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: test_avg +POSTHOOK: query: EXPLAIN +CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: test_avg +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Create Function + class: org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage + name: test_avg + temporary: true + +PREHOOK: query: CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: test_avg +POSTHOOK: query: CREATE TEMPORARY FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: test_avg +PREHOOK: query: EXPLAIN +SELECT + test_avg(1), + test_avg(substr(value,5)) +FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN +SELECT + test_avg(1), + test_avg(substr(value,5)) +FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: substr(value, 5) (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: test_avg(1), test_avg(_col1) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: test_avg(VALUE._col0), test_avg(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT + test_avg(1), + test_avg(substr(value,5)) +FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT + test_avg(1), + test_avg(substr(value,5)) +FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +1.0 260.182 +PREHOOK: query: DROP TEMPORARY FUNCTIOn test_avg +PREHOOK: type: DROPFUNCTION +PREHOOK: Output: test_avg +POSTHOOK: query: DROP TEMPORARY FUNCTIOn test_avg +POSTHOOK: type: DROPFUNCTION +POSTHOOK: Output: test_avg diff --git ql/src/test/results/clientpositive/create_udaf.q.out ql/src/test/results/clientpositive/llap/create_udaf.q.out similarity index 86% rename from ql/src/test/results/clientpositive/create_udaf.q.out rename to ql/src/test/results/clientpositive/llap/create_udaf.q.out index 7bfce125f0..52e5fa6faa 100644 --- ql/src/test/results/clientpositive/create_udaf.q.out +++ ql/src/test/results/clientpositive/llap/create_udaf.q.out @@ -42,56 +42,56 @@ POSTHOOK: Lineage: dest1_n34.col EXPRESSION [(src)src.FieldSchema(name:value, ty PREHOOK: query: SELECT dest1_n34.* FROM dest1_n34 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n34 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n34.* FROM dest1_n34 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n34 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7 PREHOOK: query: SELECT test_max(CAST(length(src.value) AS SMALLINT)) FROM src PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT test_max(CAST(length(src.value) AS SMALLINT)) FROM src POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7 PREHOOK: query: SELECT test_max(CAST(length(src.value) AS BIGINT)) FROM src PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT test_max(CAST(length(src.value) AS BIGINT)) FROM src POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7 PREHOOK: query: SELECT test_max(CAST(length(src.value) AS DOUBLE)) FROM src PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT test_max(CAST(length(src.value) AS DOUBLE)) FROM src POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7.0 PREHOOK: query: SELECT test_max(CAST(length(src.value) AS FLOAT)) FROM src PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT test_max(CAST(length(src.value) AS FLOAT)) FROM src POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7.0 PREHOOK: query: SELECT test_max(substr(src.value,5)) FROM src PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT test_max(substr(src.value,5)) FROM src POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 98 PREHOOK: query: DROP TEMPORARY FUNCTION test_max PREHOOK: type: DROPFUNCTION diff --git ql/src/test/results/clientpositive/create_view.q.out ql/src/test/results/clientpositive/llap/create_view.q.out similarity index 94% rename from ql/src/test/results/clientpositive/create_view.q.out rename to ql/src/test/results/clientpositive/llap/create_view.q.out index 9a251fcd2f..52b77c7505 100644 --- ql/src/test/results/clientpositive/create_view.q.out +++ ql/src/test/results/clientpositive/llap/create_view.q.out @@ -77,11 +77,11 @@ POSTHOOK: type: DROPFUNCTION PREHOOK: query: SELECT * FROM src WHERE key=86 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM src WHERE key=86 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 86 val_86 PREHOOK: query: CREATE VIEW view1 AS SELECT value FROM src WHERE key=86 PREHOOK: type: CREATEVIEW @@ -125,35 +125,35 @@ PREHOOK: query: SELECT * from view1 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * from view1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### val_86 PREHOOK: query: SELECT * from view2 where key=18 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * from view2 where key=18 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 18 val_18 18 val_18 PREHOOK: query: SELECT * from view3 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view3 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * from view3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view3 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### VAL_86 PREHOOK: query: EXPLAIN CREATE VIEW view0(valoo) AS SELECT upper(value) FROM src WHERE key=86 @@ -183,48 +183,32 @@ SELECT * from view2 where key=18 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: EXPLAIN SELECT * from view2 where key=18 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (UDFToDouble(key) = 18.0D) (type: boolean) - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToDouble(key) = 18.0D) (type: boolean) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: src + filterExpr: (UDFToDouble(key) = 18.0D) (type: boolean) + properties: + insideView TRUE + Filter Operator + predicate: (UDFToDouble(key) = 18.0D) (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + ListSink PREHOOK: query: SHOW TABLES 'view%' PREHOOK: type: SHOWTABLES @@ -481,11 +465,11 @@ POSTHOOK: Lineage: table1_n4.key EXPRESSION [(src)src.FieldSchema(name:key, type PREHOOK: query: SELECT * FROM table1_n4 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM table1_n4 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n4 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 86 PREHOOK: query: CREATE VIEW view4 AS SELECT * FROM table1_n4 PREHOOK: type: CREATEVIEW @@ -502,12 +486,12 @@ PREHOOK: query: SELECT * FROM view4 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 PREHOOK: Input: default@view4 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view4 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n4 POSTHOOK: Input: default@view4 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 86 PREHOOK: query: DESCRIBE view4 PREHOOK: type: DESCTABLE @@ -527,22 +511,22 @@ POSTHOOK: Output: default@table1_n4 PREHOOK: query: SELECT * FROM table1_n4 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM table1_n4 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n4 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 86 NULL PREHOOK: query: SELECT * FROM view4 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 PREHOOK: Input: default@view4 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view4 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n4 POSTHOOK: Input: default@view4 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 86 PREHOOK: query: DESCRIBE table1_n4 PREHOOK: type: DESCTABLE @@ -575,19 +559,19 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@view5 POSTHOOK: Lineage: view5.key1 SIMPLE [(table1_n4)table1_n4.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: view5.key2 SIMPLE [(table1_n4)table1_n4.FieldSchema(name:key, type:int, comment:null), ] -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT * FROM view5 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 PREHOOK: Input: default@view4 PREHOOK: Input: default@view5 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view5 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n4 POSTHOOK: Input: default@view4 POSTHOOK: Input: default@view5 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 86 86 PREHOOK: query: DESCRIBE view5 PREHOOK: type: DESCTABLE @@ -641,12 +625,12 @@ PREHOOK: query: SELECT * FROM view7 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view7 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 82 val_82 83 val_83 83 val_83 @@ -661,12 +645,12 @@ PREHOOK: query: SELECT * FROM view7 ORDER BY key DESC, value PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view7 ORDER BY key DESC, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 90 val_90 90 val_90 87 val_87 @@ -681,12 +665,12 @@ PREHOOK: query: SELECT * FROM view7 LIMIT 5 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view7 LIMIT 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 82 val_82 83 val_83 83 val_83 @@ -696,12 +680,12 @@ PREHOOK: query: SELECT * FROM view7 LIMIT 20 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view7 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view7 LIMIT 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view7 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 82 val_82 83 val_83 83 val_83 @@ -780,12 +764,12 @@ PREHOOK: query: SELECT * FROM view8 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 PREHOOK: Input: default@view8 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view8 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n4 POSTHOOK: Input: default@view8 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### bbc PREHOOK: query: CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax' @@ -855,12 +839,12 @@ PREHOOK: query: SELECT * FROM view9 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view9 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view9 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view9 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7 PREHOOK: query: DROP VIEW view9 PREHOOK: type: DROPVIEW @@ -930,12 +914,12 @@ PREHOOK: query: SELECT * FROM view9 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view9 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view9 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view9 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 7 PREHOOK: query: CREATE VIEW view10 AS SELECT slurp.* FROM (SELECT * FROM src WHERE key=86) slurp @@ -996,12 +980,12 @@ PREHOOK: query: SELECT * FROM view10 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view10 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view10 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view10 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 86 val_86 PREHOOK: query: CREATE TEMPORARY FUNCTION test_explode AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode' @@ -1071,12 +1055,12 @@ PREHOOK: query: SELECT * FROM view11 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 PREHOOK: Input: default@view11 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view11 POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n4 POSTHOOK: Input: default@view11 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 2 3 @@ -1143,26 +1127,26 @@ ORDER BY key ASC, myCol ASC LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view12 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view12 ORDER BY key ASC, myCol ASC LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view12 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 1 PREHOOK: query: SELECT * FROM view2 LATERAL VIEW explode(array(1,2,3)) myTable AS myCol ORDER BY key ASC, myCol ASC LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view2 LATERAL VIEW explode(array(1,2,3)) myTable AS myCol ORDER BY key ASC, myCol ASC LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 1 PREHOOK: query: CREATE VIEW view13 AS SELECT s.key @@ -1225,13 +1209,13 @@ ORDER BY key LIMIT 12 PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket PREHOOK: Input: default@view13 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view13 ORDER BY key LIMIT 12 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket POSTHOOK: Input: default@view13 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 8 8 @@ -1346,13 +1330,13 @@ ORDER BY k1 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view14 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view14 ORDER BY k1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view14 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 @@ -1445,14 +1429,14 @@ LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view15 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view15 ORDER BY value_count DESC, key LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view15 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 230 5 348 5 401 5 @@ -1525,14 +1509,14 @@ LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@view16 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM view16 ORDER BY value LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@view16 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### val_0 val_10 val_100 @@ -1589,12 +1573,12 @@ PREHOOK: query: select * from view17 PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Input: default@view17 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from view17 POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Input: default@view17 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 PREHOOK: query: create view view18 as select v+1 from (select 1 as v) t PREHOOK: type: CREATEVIEW @@ -1611,12 +1595,12 @@ PREHOOK: query: select * from view18 PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Input: default@view18 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from view18 POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Input: default@view18 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 2 PREHOOK: query: create view if not exists view18 as select "should be ignored" PREHOOK: type: CREATEVIEW diff --git ql/src/test/results/clientpositive/gen_udf_example_add10.q.out ql/src/test/results/clientpositive/llap/gen_udf_example_add10.q.out similarity index 52% rename from ql/src/test/results/clientpositive/gen_udf_example_add10.q.out rename to ql/src/test/results/clientpositive/llap/gen_udf_example_add10.q.out index bfe313967b..cb7ea6d5c9 100644 --- ql/src/test/results/clientpositive/gen_udf_example_add10.q.out +++ ql/src/test/results/clientpositive/llap/gen_udf_example_add10.q.out @@ -23,48 +23,62 @@ POSTHOOK: Output: default@t1_n102 PREHOOK: query: explain select example_add10(x) as a,example_add10(y) as b from t1_n102 order by a desc,b limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@t1_n102 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select example_add10(x) as a,example_add10(y) as b from t1_n102 order by a desc,b limit 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1_n102 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t1_n102 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: add10(x) (type: int), add10(y) (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: double) - null sort order: zz - sort order: -+ + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1_n102 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Top N Key Operator + sort order: -+ + keys: add10(x) (type: int), add10(y) (type: double) + null sort order: zz + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + top n: 10 + Select Operator + expressions: add10(x) (type: int), add10(y) (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: double) + null sort order: zz + sort order: -+ + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -75,11 +89,11 @@ STAGE PLANS: PREHOOK: query: select example_add10(x) as a,example_add10(y) as b from t1_n102 order by a desc,b limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@t1_n102 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select example_add10(x) as a,example_add10(y) as b from t1_n102 order by a desc,b limit 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1_n102 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 18 28.0 18 38.0 17 27.0 diff --git ql/src/test/results/clientpositive/groupby_bigdata.q.out ql/src/test/results/clientpositive/llap/groupby_bigdata.q.out similarity index 82% rename from ql/src/test/results/clientpositive/groupby_bigdata.q.out rename to ql/src/test/results/clientpositive/llap/groupby_bigdata.q.out index 90ccc8cdfb..9a7ca2f673 100644 --- ql/src/test/results/clientpositive/groupby_bigdata.q.out +++ ql/src/test/results/clientpositive/llap/groupby_bigdata.q.out @@ -2,10 +2,10 @@ PREHOOK: query: select count(distinct subq.key) from (FROM src MAP src.key USING 'python dumpdata_script.py' AS key WHERE src.key = 10) subq PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select count(distinct subq.key) from (FROM src MAP src.key USING 'python dumpdata_script.py' AS key WHERE src.key = 10) subq POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1000022 diff --git ql/src/test/results/clientpositive/llap/input14.q.out ql/src/test/results/clientpositive/llap/input14.q.out new file mode 100644 index 0000000000..f3e5526fb8 --- /dev/null +++ ql/src/test/results/clientpositive/llap/input14.q.out @@ -0,0 +1,252 @@ +PREHOOK: query: CREATE TABLE dest1_n42(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n42 +POSTHOOK: query: CREATE TABLE dest1_n42(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n42 +PREHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n42 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n42 +POSTHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n42 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n42 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0 < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 15770 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 15770 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n42 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 166 Data size: 15770 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n42 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.dest1_n42 + +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n42 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n42 +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n42 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n42 +POSTHOOK: Lineage: dest1_n42.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest1_n42.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT dest1_n42.* FROM dest1_n42 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n42 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT dest1_n42.* FROM dest1_n42 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n42 +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +11 val_11 +12 val_12 +12 val_12 +15 val_15 +15 val_15 +17 val_17 +18 val_18 +18 val_18 +19 val_19 +2 val_2 +20 val_20 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +34 val_34 +35 val_35 +35 val_35 +35 val_35 +37 val_37 +37 val_37 +4 val_4 +41 val_41 +42 val_42 +42 val_42 +43 val_43 +44 val_44 +47 val_47 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +67 val_67 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +74 val_74 +76 val_76 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 diff --git ql/src/test/results/clientpositive/llap/input14_limit.q.out ql/src/test/results/clientpositive/llap/input14_limit.q.out new file mode 100644 index 0000000000..25cfd31cf4 --- /dev/null +++ ql/src/test/results/clientpositive/llap/input14_limit.q.out @@ -0,0 +1,177 @@ +PREHOOK: query: CREATE TABLE dest1_n13(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n13 +POSTHOOK: query: CREATE TABLE dest1_n13(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n13 +PREHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey LIMIT 20 +) tmap +INSERT OVERWRITE TABLE dest1_n13 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n13 +POSTHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey LIMIT 20 +) tmap +INSERT OVERWRITE TABLE dest1_n13 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n13 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Top N Key Operator + sort order: + + keys: _col0 (type: string) + null sort order: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + top n: 20 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Top N Key Operator + sort order: + + keys: _col0 (type: string) + null sort order: a + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + top n: 20 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0 < 100) (type: boolean) + Statistics: Num rows: 6 Data size: 1068 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n13 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n13 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey LIMIT 20 +) tmap +INSERT OVERWRITE TABLE dest1_n13 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n13 +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey LIMIT 20 +) tmap +INSERT OVERWRITE TABLE dest1_n13 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n13 +POSTHOOK: Lineage: dest1_n13.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest1_n13.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT dest1_n13.* FROM dest1_n13 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n13 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT dest1_n13.* FROM dest1_n13 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n13 +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +11 val_11 diff --git ql/src/test/results/clientpositive/llap/input17.q.out ql/src/test/results/clientpositive/llap/input17.q.out new file mode 100644 index 0000000000..307403980c --- /dev/null +++ ql/src/test/results/clientpositive/llap/input17.q.out @@ -0,0 +1,176 @@ +PREHOOK: query: CREATE TABLE dest1_n81(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n81 +POSTHOOK: query: CREATE TABLE dest1_n81(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n81 +PREHOOK: query: EXPLAIN +FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n81 SELECT tmap.tkey, tmap.tvalue +PREHOOK: type: QUERY +PREHOOK: Input: default@src_thrift +PREHOOK: Output: default@dest1_n81 +POSTHOOK: query: EXPLAIN +FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n81 SELECT tmap.tkey, tmap.tvalue +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_thrift +POSTHOOK: Output: default@dest1_n81 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (aint + lint[0]) (type: int), lintstring[0] (type: struct) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n81 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 11 Data size: 29524 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3548 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 3548 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3564 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3564 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n81 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.dest1_n81 + +PREHOOK: query: FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n81 SELECT tmap.tkey, tmap.tvalue +PREHOOK: type: QUERY +PREHOOK: Input: default@src_thrift +PREHOOK: Output: default@dest1_n81 +POSTHOOK: query: FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n81 SELECT tmap.tkey, tmap.tvalue +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_thrift +POSTHOOK: Output: default@dest1_n81 +POSTHOOK: Lineage: dest1_n81.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] +POSTHOOK: Lineage: dest1_n81.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] +PREHOOK: query: SELECT dest1_n81.* FROM dest1_n81 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n81 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT dest1_n81.* FROM dest1_n81 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n81 +POSTHOOK: Output: hdfs://### HDFS PATH ### +-1461153966 {"myint":49,"mystring":"343","underscore_int":7} +-1952710705 {"myint":25,"mystring":"125","underscore_int":5} +-734328905 {"myint":16,"mystring":"64","underscore_int":4} +-751827636 {"myint":4,"mystring":"8","underscore_int":2} +1244525196 {"myint":36,"mystring":"216","underscore_int":6} +1638581586 {"myint":64,"mystring":"512","underscore_int":8} +1712634731 {"myint":0,"mystring":"0","underscore_int":0} +336964422 {"myint":81,"mystring":"729","underscore_int":9} +465985201 {"myint":1,"mystring":"1","underscore_int":1} +477111225 {"myint":9,"mystring":"27","underscore_int":3} +NULL NULL diff --git ql/src/test/results/clientpositive/llap/input18.q.out ql/src/test/results/clientpositive/llap/input18.q.out new file mode 100644 index 0000000000..4623d8b3ac --- /dev/null +++ ql/src/test/results/clientpositive/llap/input18.q.out @@ -0,0 +1,252 @@ +PREHOOK: query: CREATE TABLE dest1_n124(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n124 +POSTHOOK: query: CREATE TABLE dest1_n124(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n124 +PREHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +INSERT OVERWRITE TABLE dest1_n124 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n124 +POSTHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +INSERT OVERWRITE TABLE dest1_n124 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n124 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), 3 (type: int), 7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0 < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), regexp_replace(VALUE._col1, ' ', '+') (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 31208 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 31208 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n124 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 166 Data size: 31208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n124 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.dest1_n124 + +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +INSERT OVERWRITE TABLE dest1_n124 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1_n124 +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +INSERT OVERWRITE TABLE dest1_n124 SELECT tmap.key, regexp_replace(tmap.value,'\t','+') WHERE tmap.key < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1_n124 +POSTHOOK: Lineage: dest1_n124.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest1_n124.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT dest1_n124.* FROM dest1_n124 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n124 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT dest1_n124.* FROM dest1_n124 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n124 +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 val_0+3+7 +0 val_0+3+7 +0 val_0+3+7 +10 val_10+3+7 +11 val_11+3+7 +12 val_12+3+7 +12 val_12+3+7 +15 val_15+3+7 +15 val_15+3+7 +17 val_17+3+7 +18 val_18+3+7 +18 val_18+3+7 +19 val_19+3+7 +2 val_2+3+7 +20 val_20+3+7 +24 val_24+3+7 +24 val_24+3+7 +26 val_26+3+7 +26 val_26+3+7 +27 val_27+3+7 +28 val_28+3+7 +30 val_30+3+7 +33 val_33+3+7 +34 val_34+3+7 +35 val_35+3+7 +35 val_35+3+7 +35 val_35+3+7 +37 val_37+3+7 +37 val_37+3+7 +4 val_4+3+7 +41 val_41+3+7 +42 val_42+3+7 +42 val_42+3+7 +43 val_43+3+7 +44 val_44+3+7 +47 val_47+3+7 +5 val_5+3+7 +5 val_5+3+7 +5 val_5+3+7 +51 val_51+3+7 +51 val_51+3+7 +53 val_53+3+7 +54 val_54+3+7 +57 val_57+3+7 +58 val_58+3+7 +58 val_58+3+7 +64 val_64+3+7 +65 val_65+3+7 +66 val_66+3+7 +67 val_67+3+7 +67 val_67+3+7 +69 val_69+3+7 +70 val_70+3+7 +70 val_70+3+7 +70 val_70+3+7 +72 val_72+3+7 +72 val_72+3+7 +74 val_74+3+7 +76 val_76+3+7 +76 val_76+3+7 +77 val_77+3+7 +78 val_78+3+7 +8 val_8+3+7 +80 val_80+3+7 +82 val_82+3+7 +83 val_83+3+7 +83 val_83+3+7 +84 val_84+3+7 +84 val_84+3+7 +85 val_85+3+7 +86 val_86+3+7 +87 val_87+3+7 +9 val_9+3+7 +90 val_90+3+7 +90 val_90+3+7 +90 val_90+3+7 +92 val_92+3+7 +95 val_95+3+7 +95 val_95+3+7 +96 val_96+3+7 +97 val_97+3+7 +97 val_97+3+7 +98 val_98+3+7 +98 val_98+3+7 diff --git ql/src/test/results/clientpositive/input20.q.out ql/src/test/results/clientpositive/llap/input20.q.out similarity index 54% rename from ql/src/test/results/clientpositive/input20.q.out rename to ql/src/test/results/clientpositive/llap/input20.q.out index d90b9083c3..819d888426 100644 --- ql/src/test/results/clientpositive/input20.q.out +++ ql/src/test/results/clientpositive/llap/input20.q.out @@ -36,74 +36,99 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n138 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Transform Operator + command: python input20_script.py + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: python input20_script.py - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n138 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n138 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -115,7 +140,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n138 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -123,30 +148,6 @@ STAGE PLANS: Column Types: int, string Table: default.dest1_n138 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: FROM ( FROM src MAP src.key, src.key @@ -178,11 +179,11 @@ POSTHOOK: Lineage: dest1_n138.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: SELECT * FROM dest1_n138 ORDER BY key, value PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n138 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM dest1_n138 ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n138 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 105_105 1 10_10 1 111_111 diff --git ql/src/test/results/clientpositive/input33.q.out ql/src/test/results/clientpositive/llap/input33.q.out similarity index 54% rename from ql/src/test/results/clientpositive/input33.q.out rename to ql/src/test/results/clientpositive/llap/input33.q.out index c8df2efede..daf57dea3a 100644 --- ql/src/test/results/clientpositive/input33.q.out +++ ql/src/test/results/clientpositive/llap/input33.q.out @@ -36,74 +36,99 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n135 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Transform Operator + command: python input20_script.py + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: python input20_script.py - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n135 - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n135 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -115,7 +140,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n135 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -123,30 +148,6 @@ STAGE PLANS: Column Types: int, string Table: default.dest1_n135 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: FROM ( FROM src MAP src.key, src.key @@ -178,11 +179,11 @@ POSTHOOK: Lineage: dest1_n135.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: SELECT * FROM dest1_n135 ORDER BY key, value PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n135 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM dest1_n135 ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n135 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 105_105 1 10_10 1 111_111 diff --git ql/src/test/results/clientpositive/input34.q.out ql/src/test/results/clientpositive/llap/input34.q.out similarity index 66% rename from ql/src/test/results/clientpositive/input34.q.out rename to ql/src/test/results/clientpositive/llap/input34.q.out index 00dd35d803..3f702cdf3b 100644 --- ql/src/test/results/clientpositive/input34.q.out +++ ql/src/test/results/clientpositive/llap/input34.q.out @@ -30,81 +30,79 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n161 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n161 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n161 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -116,7 +114,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n161 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -124,36 +122,6 @@ STAGE PLANS: Column Types: int, string Table: default.dest1_n161 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n161 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n161 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' @@ -179,11 +147,11 @@ POSTHOOK: Lineage: dest1_n161.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: SELECT dest1_n161.* FROM dest1_n161 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n161 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n161.* FROM dest1_n161 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n161 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 238 val_238 86 val_86 311 val_311 diff --git ql/src/test/results/clientpositive/input35.q.out ql/src/test/results/clientpositive/llap/input35.q.out similarity index 65% rename from ql/src/test/results/clientpositive/input35.q.out rename to ql/src/test/results/clientpositive/llap/input35.q.out index cee491fc82..85e5412b47 100644 --- ql/src/test/results/clientpositive/input35.q.out +++ ql/src/test/results/clientpositive/llap/input35.q.out @@ -30,81 +30,79 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n25 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n25 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n25 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -116,7 +114,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n25 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -124,36 +122,6 @@ STAGE PLANS: Column Types: int, string Table: default.dest1_n25 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n25 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n25 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002' @@ -179,11 +147,11 @@ POSTHOOK: Lineage: dest1_n25.value SCRIPT [(src)src.FieldSchema(name:key, type:s PREHOOK: query: SELECT dest1_n25.* FROM dest1_n25 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n25 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n25.* FROM dest1_n25 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n25 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 238 val_238 86 val_86 311 val_311 diff --git ql/src/test/results/clientpositive/input36.q.out ql/src/test/results/clientpositive/llap/input36.q.out similarity index 63% rename from ql/src/test/results/clientpositive/input36.q.out rename to ql/src/test/results/clientpositive/llap/input36.q.out index 45289b2143..8a8e30c1ea 100644 --- ql/src/test/results/clientpositive/input36.q.out +++ ql/src/test/results/clientpositive/llap/input36.q.out @@ -30,81 +30,79 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n70 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n70 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n70 + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -116,7 +114,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n70 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -124,36 +122,6 @@ STAGE PLANS: Column Types: int, string Table: default.dest1_n70 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n70 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n70 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002' @@ -179,11 +147,11 @@ POSTHOOK: Lineage: dest1_n70.value SCRIPT [(src)src.FieldSchema(name:key, type:s PREHOOK: query: SELECT dest1_n70.* FROM dest1_n70 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n70 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n70.* FROM dest1_n70 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n70 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### NULL NULL NULL NULL NULL NULL diff --git ql/src/test/results/clientpositive/input38.q.out ql/src/test/results/clientpositive/llap/input38.q.out similarity index 72% rename from ql/src/test/results/clientpositive/input38.q.out rename to ql/src/test/results/clientpositive/llap/input38.q.out index d46ddf03ca..161cda2ac7 100644 --- ql/src/test/results/clientpositive/input38.q.out +++ ql/src/test/results/clientpositive/llap/input38.q.out @@ -28,77 +28,75 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n91 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), 3 (type: int), 7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), 3 (type: int), 7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n91 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n91 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -110,7 +108,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n91 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -118,36 +116,6 @@ STAGE PLANS: Column Types: string, string Table: default.dest1_n91 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n91 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n91 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) @@ -171,11 +139,11 @@ POSTHOOK: Lineage: dest1_n91.value SCRIPT [(src)src.FieldSchema(name:key, type:s PREHOOK: query: SELECT dest1_n91.* FROM dest1_n91 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n91 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n91.* FROM dest1_n91 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n91 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 238 val_238 3 7 86 val_86 3 7 311 val_311 3 7 diff --git ql/src/test/results/clientpositive/llap/input5.q.out ql/src/test/results/clientpositive/llap/input5.q.out new file mode 100644 index 0000000000..ab091fe0df --- /dev/null +++ ql/src/test/results/clientpositive/llap/input5.q.out @@ -0,0 +1,176 @@ +PREHOOK: query: CREATE TABLE dest1_n94(key STRING, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1_n94 +POSTHOOK: query: CREATE TABLE dest1_n94(key STRING, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1_n94 +PREHOOK: query: EXPLAIN +FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.lint, src_thrift.lintstring) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n94 SELECT tmap.tkey, tmap.tvalue +PREHOOK: type: QUERY +PREHOOK: Input: default@src_thrift +PREHOOK: Output: default@dest1_n94 +POSTHOOK: query: EXPLAIN +FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.lint, src_thrift.lintstring) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n94 SELECT tmap.tkey, tmap.tvalue +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_thrift +POSTHOOK: Output: default@dest1_n94 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: lint (type: array), lintstring (type: array>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n94 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 11 Data size: 29480 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n94 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.dest1_n94 + +PREHOOK: query: FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.lint, src_thrift.lintstring) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n94 SELECT tmap.tkey, tmap.tvalue +PREHOOK: type: QUERY +PREHOOK: Input: default@src_thrift +PREHOOK: Output: default@dest1_n94 +POSTHOOK: query: FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.lint, src_thrift.lintstring) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1_n94 SELECT tmap.tkey, tmap.tvalue +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_thrift +POSTHOOK: Output: default@dest1_n94 +POSTHOOK: Lineage: dest1_n94.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] +POSTHOOK: Lineage: dest1_n94.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array>, comment:from deserializer), ] +PREHOOK: query: SELECT dest1_n94.* FROM dest1_n94 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1_n94 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT dest1_n94.* FROM dest1_n94 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1_n94 +POSTHOOK: Output: hdfs://### HDFS PATH ### +NULL NULL +[0,0,0] [{"myint":0,"mystring":"0","underscore_int":0}] +[1,2,3] [{"myint":1,"mystring":"1","underscore_int":1}] +[2,4,6] [{"myint":4,"mystring":"8","underscore_int":2}] +[3,6,9] [{"myint":9,"mystring":"27","underscore_int":3}] +[4,8,12] [{"myint":16,"mystring":"64","underscore_int":4}] +[5,10,15] [{"myint":25,"mystring":"125","underscore_int":5}] +[6,12,18] [{"myint":36,"mystring":"216","underscore_int":6}] +[7,14,21] [{"myint":49,"mystring":"343","underscore_int":7}] +[8,16,24] [{"myint":64,"mystring":"512","underscore_int":8}] +[9,18,27] [{"myint":81,"mystring":"729","underscore_int":9}] diff --git ql/src/test/results/clientpositive/llap/insert_into3.q.out ql/src/test/results/clientpositive/llap/insert_into3.q.out new file mode 100644 index 0000000000..9c9820e661 --- /dev/null +++ ql/src/test/results/clientpositive/llap/insert_into3.q.out @@ -0,0 +1,546 @@ +PREHOOK: query: DROP TABLE insert_into3a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into3a +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE insert_into3b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into3b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE insert_into3a (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into3a +POSTHOOK: query: CREATE TABLE insert_into3a (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into3a +PREHOOK: query: CREATE TABLE insert_into3b (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into3b +POSTHOOK: query: CREATE TABLE insert_into3b (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into3b +PREHOOK: query: EXPLAIN FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 + INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into3a +PREHOOK: Output: default@insert_into3b +POSTHOOK: query: EXPLAIN FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 + INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into3a +POSTHOOK: Output: default@insert_into3b +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Map 1 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Top N Key Operator + sort order: ++ + keys: key (type: string), value (type: string) + null sort order: zz + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + top n: 50 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Top N Key Operator + sort order: ++ + keys: key (type: string), value (type: string) + null sort order: zz + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + top n: 100 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 50 + Statistics: Num rows: 50 Data size: 8900 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 50 Data size: 4750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 50 Data size: 4750 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into3a + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 50 Data size: 4750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.98 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into3b + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into3a + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into3a + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into3b + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into3b + +PREHOOK: query: FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 + INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into3a +PREHOOK: Output: default@insert_into3b +POSTHOOK: query: FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 + INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into3a +POSTHOOK: Output: default@insert_into3b +POSTHOOK: Lineage: insert_into3a.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into3a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into3b.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into3b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into3a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into3a +POSTHOOK: Output: hdfs://### HDFS PATH ### +-1254133670 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into3b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into3b +POSTHOOK: Output: hdfs://### HDFS PATH ### +-1142373758 +PREHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 + INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into3a +PREHOOK: Output: default@insert_into3b +POSTHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 + INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into3a +POSTHOOK: Output: default@insert_into3b +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into3a + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.9 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into3b + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.9 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into3a + + Stage: Stage-4 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into3a + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into3b + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into3b + +PREHOOK: query: FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 + INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into3a +PREHOOK: Output: default@insert_into3b +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 + INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into3a +POSTHOOK: Output: default@insert_into3b +POSTHOOK: Lineage: insert_into3a.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into3a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into3b.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into3b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into3a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into3a +POSTHOOK: Output: hdfs://### HDFS PATH ### +-826625916 +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into3b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into3b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into3b +POSTHOOK: Output: hdfs://### HDFS PATH ### +-1968999674 +PREHOOK: query: DROP TABLE insert_into3a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into3a +PREHOOK: Output: default@insert_into3a +POSTHOOK: query: DROP TABLE insert_into3a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into3a +POSTHOOK: Output: default@insert_into3a +PREHOOK: query: DROP TABLE insert_into3b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into3b +PREHOOK: Output: default@insert_into3b +POSTHOOK: query: DROP TABLE insert_into3b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into3b +POSTHOOK: Output: default@insert_into3b diff --git ql/src/test/results/clientpositive/llap/insert_into4.q.out ql/src/test/results/clientpositive/llap/insert_into4.q.out new file mode 100644 index 0000000000..f2e4bab82c --- /dev/null +++ ql/src/test/results/clientpositive/llap/insert_into4.q.out @@ -0,0 +1,436 @@ +PREHOOK: query: DROP TABLE insert_into4a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into4a +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE insert_into4b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into4b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE insert_into4a (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into4a +POSTHOOK: query: CREATE TABLE insert_into4a (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into4a +PREHOOK: query: CREATE TABLE insert_into4b (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into4b +POSTHOOK: query: CREATE TABLE insert_into4b (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into4b +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into4a +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into4a +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into4a + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.9 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into4a + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into4a + +PREHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into4a +POSTHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into4a +POSTHOOK: Lineage: insert_into4a.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into4a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into4a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into4a +POSTHOOK: Output: hdfs://### HDFS PATH ### +-826625916 +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into4a +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into4a +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into4a + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.9 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into4a + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into4a + +PREHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into4a +POSTHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into4a +POSTHOOK: Lineage: insert_into4a.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into4a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into4a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into4a +POSTHOOK: Output: hdfs://### HDFS PATH ### +-1653251832 +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into4a +PREHOOK: Output: default@insert_into4b +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into4a +POSTHOOK: Output: default@insert_into4b +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: insert_into4a + Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into4b + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.95 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into4b + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into4b + +PREHOOK: query: INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into4a +PREHOOK: Output: default@insert_into4b +POSTHOOK: query: INSERT INTO TABLE insert_into4b SELECT * FROM insert_into4a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into4a +POSTHOOK: Output: default@insert_into4b +POSTHOOK: Lineage: insert_into4b.key SIMPLE [(insert_into4a)insert_into4a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: insert_into4b.value SIMPLE [(insert_into4a)insert_into4a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into4b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into4b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into4b +POSTHOOK: Output: hdfs://### HDFS PATH ### +-1653251832 +PREHOOK: query: DROP TABLE insert_into4a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into4a +PREHOOK: Output: default@insert_into4a +POSTHOOK: query: DROP TABLE insert_into4a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into4a +POSTHOOK: Output: default@insert_into4a +PREHOOK: query: DROP TABLE insert_into4b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into4b +PREHOOK: Output: default@insert_into4b +POSTHOOK: query: DROP TABLE insert_into4b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into4b +POSTHOOK: Output: default@insert_into4b diff --git ql/src/test/results/clientpositive/llap/insert_into5.q.out ql/src/test/results/clientpositive/llap/insert_into5.q.out new file mode 100644 index 0000000000..2573fd2591 --- /dev/null +++ ql/src/test/results/clientpositive/llap/insert_into5.q.out @@ -0,0 +1,550 @@ +PREHOOK: query: DROP TABLE insert_into5a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into5a +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE insert_into5b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into5b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE insert_into5a (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into5a +POSTHOOK: query: CREATE TABLE insert_into5a (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into5a +PREHOOK: query: CREATE TABLE insert_into5b (key int, value string) PARTITIONED BY (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into5b +POSTHOOK: query: CREATE TABLE insert_into5b (key int, value string) PARTITIONED BY (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into5b +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into5a +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into5a +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int), 'one' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5a + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.9 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5a + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into5a + +PREHOOK: query: INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into5a +POSTHOOK: query: INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into5a +POSTHOOK: Lineage: insert_into5a.key SIMPLE [] +POSTHOOK: Lineage: insert_into5a.value SIMPLE [] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5a +POSTHOOK: Output: hdfs://### HDFS PATH ### +481928560 +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5a +PREHOOK: Output: default@insert_into5a +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5a +POSTHOOK: Output: default@insert_into5a +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: insert_into5a + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5a + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, value + Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.9 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5a + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into5a + +PREHOOK: query: INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5a +PREHOOK: Output: default@insert_into5a +POSTHOOK: query: INSERT INTO TABLE insert_into5a SELECT * FROM insert_into5a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5a +POSTHOOK: Output: default@insert_into5a +POSTHOOK: Lineage: insert_into5a.key SIMPLE [(insert_into5a)insert_into5a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: insert_into5a.value SIMPLE [(insert_into5a)insert_into5a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5a +POSTHOOK: Output: hdfs://### HDFS PATH ### +963857120 +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5b PARTITION (ds='1') + SELECT * FROM insert_into5a +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5a +PREHOOK: Output: default@insert_into5b@ds=1 +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5b PARTITION (ds='1') + SELECT * FROM insert_into5a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5a +POSTHOOK: Output: default@insert_into5b@ds=1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: insert_into5a + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5b + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string) + outputColumnNames: key, value, ds + Statistics: Num rows: 20 Data size: 3520 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: ds (type: string) + minReductionHashAggr: 0.95 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5b + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into5b + +PREHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') SELECT * FROM insert_into5a +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5a +PREHOOK: Output: default@insert_into5b@ds=1 +POSTHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') SELECT * FROM insert_into5a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5a +POSTHOOK: Output: default@insert_into5b@ds=1 +POSTHOOK: Lineage: insert_into5b PARTITION(ds=1).key SIMPLE [(insert_into5a)insert_into5a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: insert_into5b PARTITION(ds=1).value SIMPLE [(insert_into5a)insert_into5a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5b +PREHOOK: Input: default@insert_into5b@ds=1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5b +POSTHOOK: Input: default@insert_into5b@ds=1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +-18626052920 +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5b PARTITION (ds='1') + SELECT key, value FROM insert_into5b +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5b +PREHOOK: Input: default@insert_into5b@ds=1 +PREHOOK: Output: default@insert_into5b@ds=1 +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into5b PARTITION (ds='1') + SELECT key, value FROM insert_into5b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5b +POSTHOOK: Input: default@insert_into5b@ds=1 +POSTHOOK: Output: default@insert_into5b@ds=1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: insert_into5b + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5b + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string) + outputColumnNames: key, value, ds + Statistics: Num rows: 20 Data size: 3520 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: ds (type: string) + minReductionHashAggr: 0.95 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into5b + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into5b + +PREHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') + SELECT key, value FROM insert_into5b +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5b +PREHOOK: Input: default@insert_into5b@ds=1 +PREHOOK: Output: default@insert_into5b@ds=1 +POSTHOOK: query: INSERT INTO TABLE insert_into5b PARTITION (ds='1') + SELECT key, value FROM insert_into5b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5b +POSTHOOK: Input: default@insert_into5b@ds=1 +POSTHOOK: Output: default@insert_into5b@ds=1 +POSTHOOK: Lineage: insert_into5b PARTITION(ds=1).key SIMPLE [(insert_into5b)insert_into5b.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: insert_into5b PARTITION(ds=1).value SIMPLE [(insert_into5b)insert_into5b.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into5b +PREHOOK: Input: default@insert_into5b@ds=1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into5b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into5b +POSTHOOK: Input: default@insert_into5b@ds=1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +-37252105840 +PREHOOK: query: DROP TABLE insert_into5a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into5a +PREHOOK: Output: default@insert_into5a +POSTHOOK: query: DROP TABLE insert_into5a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into5a +POSTHOOK: Output: default@insert_into5a diff --git ql/src/test/results/clientpositive/llap/insert_into6.q.out ql/src/test/results/clientpositive/llap/insert_into6.q.out new file mode 100644 index 0000000000..fabcf410ca --- /dev/null +++ ql/src/test/results/clientpositive/llap/insert_into6.q.out @@ -0,0 +1,356 @@ +PREHOOK: query: DROP TABLE insert_into6a +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into6a +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE insert_into6b +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into6b +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE insert_into6a (key int, value string) PARTITIONED BY (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into6a +POSTHOOK: query: CREATE TABLE insert_into6a (key int, value string) PARTITIONED BY (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into6a +PREHOOK: query: CREATE TABLE insert_into6b (key int, value string) PARTITIONED BY (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into6b +POSTHOOK: query: CREATE TABLE insert_into6b (key int, value string) PARTITIONED BY (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into6b +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into6a PARTITION (ds='1') + SELECT * FROM src LIMIT 150 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into6a@ds=1 +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into6a PARTITION (ds='1') + SELECT * FROM src LIMIT 150 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into6a@ds=1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 150 + Statistics: Num rows: 150 Data size: 26700 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 150 Data size: 26700 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 150 Data size: 26700 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 150 + Statistics: Num rows: 150 Data size: 26700 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 150 Data size: 14250 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 150 Data size: 14250 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into6a + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '1' (type: string) + outputColumnNames: key, value, ds + Statistics: Num rows: 150 Data size: 27000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: ds (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 949 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 965 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into6a + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into6a + +PREHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='1') SELECT * FROM src LIMIT 150 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into6a@ds=1 +POSTHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='1') SELECT * FROM src LIMIT 150 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into6a@ds=1 +POSTHOOK: Lineage: insert_into6a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into6a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='2') SELECT * FROM src LIMIT 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into6a@ds=2 +POSTHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='2') SELECT * FROM src LIMIT 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into6a@ds=2 +POSTHOOK: Lineage: insert_into6a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into6a PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into6a +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into6a +PREHOOK: Input: default@insert_into6a@ds=1 +PREHOOK: Input: default@insert_into6a@ds=2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into6a +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into6a +POSTHOOK: Input: default@insert_into6a@ds=1 +POSTHOOK: Input: default@insert_into6a@ds=2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +-35226404960 +PREHOOK: query: EXPLAIN INSERT INTO TABLE insert_into6b PARTITION (ds) + SELECT * FROM insert_into6a +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into6a +PREHOOK: Input: default@insert_into6a@ds=1 +PREHOOK: Input: default@insert_into6a@ds=2 +PREHOOK: Output: default@insert_into6b +POSTHOOK: query: EXPLAIN INSERT INTO TABLE insert_into6b PARTITION (ds) + SELECT * FROM insert_into6a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into6a +POSTHOOK: Input: default@insert_into6a@ds=1 +POSTHOOK: Input: default@insert_into6a@ds=2 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: insert_into6a + Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into6b + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + outputColumnNames: key, value, ds + Statistics: Num rows: 250 Data size: 69750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: ds (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 2096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 2096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 2128 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 2128 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 2128 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into6b + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: int, string + Table: default.insert_into6b + +PREHOOK: query: INSERT INTO TABLE insert_into6b PARTITION (ds) SELECT * FROM insert_into6a +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into6a +PREHOOK: Input: default@insert_into6a@ds=1 +PREHOOK: Input: default@insert_into6a@ds=2 +PREHOOK: Output: default@insert_into6b +POSTHOOK: query: INSERT INTO TABLE insert_into6b PARTITION (ds) SELECT * FROM insert_into6a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into6a +POSTHOOK: Input: default@insert_into6a@ds=1 +POSTHOOK: Input: default@insert_into6a@ds=2 +POSTHOOK: Output: default@insert_into6b@ds=1 +POSTHOOK: Output: default@insert_into6b@ds=2 +POSTHOOK: Lineage: insert_into6b PARTITION(ds=1).key SIMPLE [(insert_into6a)insert_into6a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: insert_into6b PARTITION(ds=1).value SIMPLE [(insert_into6a)insert_into6a.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: insert_into6b PARTITION(ds=2).key SIMPLE [(insert_into6a)insert_into6a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: insert_into6b PARTITION(ds=2).value SIMPLE [(insert_into6a)insert_into6a.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into6b +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into6b +PREHOOK: Input: default@insert_into6b@ds=1 +PREHOOK: Input: default@insert_into6b@ds=2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM insert_into6b +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into6b +POSTHOOK: Input: default@insert_into6b@ds=1 +POSTHOOK: Input: default@insert_into6b@ds=2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +-35226404960 +PREHOOK: query: SHOW PARTITIONS insert_into6b +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@insert_into6b +POSTHOOK: query: SHOW PARTITIONS insert_into6b +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@insert_into6b +ds=1 +ds=2 +PREHOOK: query: DROP TABLE insert_into6a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into6a +PREHOOK: Output: default@insert_into6a +POSTHOOK: query: DROP TABLE insert_into6a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into6a +POSTHOOK: Output: default@insert_into6a +PREHOOK: query: DROP TABLE insert_into6b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into6b +PREHOOK: Output: default@insert_into6b +POSTHOOK: query: DROP TABLE insert_into6b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into6b +POSTHOOK: Output: default@insert_into6b diff --git ql/src/test/results/clientpositive/load_binary_data.q.out ql/src/test/results/clientpositive/llap/load_binary_data.q.out similarity index 96% rename from ql/src/test/results/clientpositive/load_binary_data.q.out rename to ql/src/test/results/clientpositive/llap/load_binary_data.q.out index b0d5c634b5..d3c0d29ef3 100644 --- ql/src/test/results/clientpositive/load_binary_data.q.out +++ ql/src/test/results/clientpositive/llap/load_binary_data.q.out @@ -41,11 +41,11 @@ POSTHOOK: Lineage: dest1_n155.value SCRIPT [(mytable_n2)mytable_n2.FieldSchema(n PREHOOK: query: select key, value, length (key) from dest1_n155 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n155 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select key, value, length (key) from dest1_n155 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n155 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### abc 1 8 test 2 6 test 3 6 diff --git ql/src/test/results/clientpositive/macro_1.q.out ql/src/test/results/clientpositive/llap/macro_1.q.out similarity index 91% rename from ql/src/test/results/clientpositive/macro_1.q.out rename to ql/src/test/results/clientpositive/llap/macro_1.q.out index 28230f90e5..44d6ccab33 100644 --- ql/src/test/results/clientpositive/macro_1.q.out +++ ql/src/test/results/clientpositive/llap/macro_1.q.out @@ -38,13 +38,13 @@ PREHOOK: query: SELECT FROM macro_test PREHOOK: type: QUERY PREHOOK: Input: default@macro_test -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT CONCAT(STRING_LEN(x), ":", STRING_LEN_PLUS_ONE(x), ":", STRING_LEN_PLUS_TWO(x)) a FROM macro_test POSTHOOK: type: QUERY POSTHOOK: Input: default@macro_test -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 2:3:4 1:2:3 3:4:5 @@ -54,14 +54,14 @@ FROM macro_test sort by a PREHOOK: type: QUERY PREHOOK: Input: default@macro_test -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT CONCAT(STRING_LEN(x), ":", STRING_LEN_PLUS_ONE(x), ":", STRING_LEN_PLUS_TWO(x)) a FROM macro_test sort by a POSTHOOK: type: QUERY POSTHOOK: Input: default@macro_test -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1:2:3 2:3:4 3:4:5 @@ -71,14 +71,14 @@ FROM macro_test sort by a desc PREHOOK: type: QUERY PREHOOK: Input: default@macro_test -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT CONCAT(STRING_LEN(x), ":", STRING_LEN_PLUS_ONE(x), ":", STRING_LEN_PLUS_TWO(x)) a FROM macro_test sort by a desc POSTHOOK: type: QUERY POSTHOOK: Input: default@macro_test -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 3:4:5 2:3:4 1:2:3 diff --git ql/src/test/results/clientpositive/macro_duplicate.q.out ql/src/test/results/clientpositive/llap/macro_duplicate.q.out similarity index 96% rename from ql/src/test/results/clientpositive/macro_duplicate.q.out rename to ql/src/test/results/clientpositive/llap/macro_duplicate.q.out index 9598126c92..ccdf735245 100644 --- ql/src/test/results/clientpositive/macro_duplicate.q.out +++ ql/src/test/results/clientpositive/llap/macro_duplicate.q.out @@ -47,10 +47,10 @@ POSTHOOK: Output: database:default PREHOOK: query: select math_square(a), math_square(b),factorial(a), factorial(b), math_add(a), math_add(b),int(c) from macro_testing order by int(c) PREHOOK: type: QUERY PREHOOK: Input: default@macro_testing -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select math_square(a), math_square(b),factorial(a), factorial(b), math_add(a), math_add(b),int(c) from macro_testing order by int(c) POSTHOOK: type: QUERY POSTHOOK: Input: default@macro_testing -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 4 1 2 2 4 3 16 25 24 120 8 10 6 diff --git ql/src/test/results/clientpositive/mapreduce3.q.out ql/src/test/results/clientpositive/llap/mapreduce3.q.out similarity index 69% rename from ql/src/test/results/clientpositive/mapreduce3.q.out rename to ql/src/test/results/clientpositive/llap/mapreduce3.q.out index 9c0157c923..f691d40eb6 100644 --- ql/src/test/results/clientpositive/mapreduce3.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce3.q.out @@ -26,63 +26,89 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n23 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string), _col0 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reduce Operator Tree: - Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), UDFToInteger(VALUE._col1) (type: int), UDFToInteger(VALUE._col2) (type: int), VALUE._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n23 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string) - outputColumnNames: key, ten, one, value - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(ten, 'hll'), compute_stats(one, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 1712 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col3 (type: string), _col0 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), UDFToInteger(VALUE._col1) (type: int), UDFToInteger(VALUE._col2) (type: int), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n23 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string) + outputColumnNames: key, ten, one, value + Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(ten, 'hll'), compute_stats(one, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1712 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1712 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -94,7 +120,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n23 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -102,30 +128,6 @@ STAGE PLANS: Column Types: int, int, int, string Table: default.dest1_n23 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 1712 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n23 MAP src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value @@ -149,11 +151,11 @@ POSTHOOK: Lineage: dest1_n23.value SCRIPT [(src)src.FieldSchema(name:key, type:s PREHOOK: query: SELECT dest1_n23.* FROM dest1_n23 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n23 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n23.* FROM dest1_n23 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n23 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 0 val_0 0 0 0 val_0 0 0 0 val_0 diff --git ql/src/test/results/clientpositive/mapreduce4.q.out ql/src/test/results/clientpositive/llap/mapreduce4.q.out similarity index 69% rename from ql/src/test/results/clientpositive/mapreduce4.q.out rename to ql/src/test/results/clientpositive/llap/mapreduce4.q.out index a606df0894..12882a8684 100644 --- ql/src/test/results/clientpositive/mapreduce4.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce4.q.out @@ -28,64 +28,90 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n93 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: string) - null sort order: zz - sort order: -+ - Map-reduce partition columns: _col3 (type: string), _col0 (type: string) - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reduce Operator Tree: - Select Operator - expressions: UDFToInteger(VALUE._col0) (type: int), UDFToInteger(VALUE._col1) (type: int), UDFToInteger(VALUE._col2) (type: int), VALUE._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n93 - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string) - outputColumnNames: key, ten, one, value - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(ten, 'hll'), compute_stats(one, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 1712 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: string) + null sort order: zz + sort order: -+ + Map-reduce partition columns: _col3 (type: string), _col0 (type: string) + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), UDFToInteger(VALUE._col1) (type: int), UDFToInteger(VALUE._col2) (type: int), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n93 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string) + outputColumnNames: key, ten, one, value + Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(ten, 'hll'), compute_stats(one, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1712 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1712 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -97,7 +123,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n93 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -105,30 +131,6 @@ STAGE PLANS: Column Types: int, int, int, string Table: default.dest1_n93 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 1712 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n93 MAP src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value @@ -154,11 +156,11 @@ POSTHOOK: Lineage: dest1_n93.value SCRIPT [(src)src.FieldSchema(name:key, type:s PREHOOK: query: SELECT dest1_n93.* FROM dest1_n93 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n93 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n93.* FROM dest1_n93 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n93 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 90 9 0 val_90 90 9 0 val_90 90 9 0 val_90 diff --git ql/src/test/results/clientpositive/mapreduce7.q.out ql/src/test/results/clientpositive/llap/mapreduce7.q.out similarity index 76% rename from ql/src/test/results/clientpositive/mapreduce7.q.out rename to ql/src/test/results/clientpositive/llap/mapreduce7.q.out index ab369e667b..10d42cefee 100644 --- ql/src/test/results/clientpositive/mapreduce7.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce7.q.out @@ -26,63 +26,89 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n37 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col5 (type: string), _col2 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), UDFToInteger(VALUE._col2) (type: int), UDFToInteger(VALUE._col3) (type: int), UDFToInteger(VALUE._col4) (type: int), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n37 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) - outputColumnNames: k, v, key, ten, one, value - Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(k, 'hll'), compute_stats(v, 'hll'), compute_stats(key, 'hll'), compute_stats(ten, 'hll'), compute_stats(one, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 2592 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col5 (type: string), _col2 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), UDFToInteger(VALUE._col2) (type: int), UDFToInteger(VALUE._col3) (type: int), UDFToInteger(VALUE._col4) (type: int), VALUE._col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n37 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) + outputColumnNames: k, v, key, ten, one, value + Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(k, 'hll'), compute_stats(v, 'hll'), compute_stats(key, 'hll'), compute_stats(ten, 'hll'), compute_stats(one, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 2592 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 2592 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -94,7 +120,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n37 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -102,30 +128,6 @@ STAGE PLANS: Column Types: string, string, int, int, int, string Table: default.dest1_n37 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 2592 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n37 MAP src.*, src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value @@ -151,11 +153,11 @@ POSTHOOK: Lineage: dest1_n37.value SCRIPT [(src)src.FieldSchema(name:key, type:s PREHOOK: query: SELECT dest1_n37.* FROM dest1_n37 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n37 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n37.* FROM dest1_n37 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n37 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 0 0 val_0 0 val_0 0 0 0 val_0 0 val_0 0 0 0 val_0 diff --git ql/src/test/results/clientpositive/mapreduce8.q.out ql/src/test/results/clientpositive/llap/mapreduce8.q.out similarity index 75% rename from ql/src/test/results/clientpositive/mapreduce8.q.out rename to ql/src/test/results/clientpositive/llap/mapreduce8.q.out index d00ede826b..1a38974d81 100644 --- ql/src/test/results/clientpositive/mapreduce8.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce8.q.out @@ -28,64 +28,90 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1_n158 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col5 (type: string), _col2 (type: string) - null sort order: zz - sort order: ++ - Map-reduce partition columns: rand(3) (type: double) - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), UDFToInteger(VALUE._col2) (type: int), UDFToInteger(VALUE._col3) (type: int), UDFToInteger(VALUE._col4) (type: int), VALUE._col5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1_n158 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) - outputColumnNames: k, v, key, ten, one, value - Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(k, 'hll'), compute_stats(v, 'hll'), compute_stats(key, 'hll'), compute_stats(ten, 'hll'), compute_stats(one, 'hll'), compute_stats(value, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 2592 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col5 (type: string), _col2 (type: string) + null sort order: zz + sort order: ++ + Map-reduce partition columns: rand(3) (type: double) + Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), UDFToInteger(VALUE._col2) (type: int), UDFToInteger(VALUE._col3) (type: int), UDFToInteger(VALUE._col4) (type: int), VALUE._col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1_n158 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: string) + outputColumnNames: k, v, key, ten, one, value + Statistics: Num rows: 500 Data size: 140500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(k, 'hll'), compute_stats(v, 'hll'), compute_stats(key, 'hll'), compute_stats(ten, 'hll'), compute_stats(one, 'hll'), compute_stats(value, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 2592 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 2592 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection Stage: Stage-0 Move Operator @@ -97,7 +123,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1_n158 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -105,30 +131,6 @@ STAGE PLANS: Column Types: string, string, int, int, int, string Table: default.dest1_n158 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 2592 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1_n158 MAP src.*, src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value @@ -156,11 +158,11 @@ POSTHOOK: Lineage: dest1_n158.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: SELECT dest1_n158.* FROM dest1_n158 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n158 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n158.* FROM dest1_n158 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n158 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 0 0 val_0 0 val_0 0 0 0 val_0 0 val_0 0 0 0 val_0 diff --git ql/src/test/results/clientpositive/merge_test_dummy_operator.q.out ql/src/test/results/clientpositive/llap/merge_test_dummy_operator.q.out similarity index 99% rename from ql/src/test/results/clientpositive/merge_test_dummy_operator.q.out rename to ql/src/test/results/clientpositive/llap/merge_test_dummy_operator.q.out index 31d4ae16f7..22c64a16de 100644 --- ql/src/test/results/clientpositive/merge_test_dummy_operator.q.out +++ ql/src/test/results/clientpositive/llap/merge_test_dummy_operator.q.out @@ -317,7 +317,7 @@ SELECT S.* --AND F.temp_result_1 < 200 PREHOOK: type: QUERY PREHOOK: Input: default@common_join_table -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: WITH temp_tbl_1 AS ( SELECT col7 ,col4KEY @@ -407,7 +407,7 @@ SELECT S.* --AND F.temp_result_1 < 200 POSTHOOK: type: QUERY POSTHOOK: Input: default@common_join_table -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### id 109515 1 10 1 PREHOOK: query: drop table common_join_table PREHOOK: type: DROPTABLE diff --git ql/src/test/results/clientpositive/newline.q.out ql/src/test/results/clientpositive/llap/newline.q.out similarity index 89% rename from ql/src/test/results/clientpositive/newline.q.out rename to ql/src/test/results/clientpositive/llap/newline.q.out index bea4e6ce1c..7f7dbb8069 100644 --- ql/src/test/results/clientpositive/newline.q.out +++ ql/src/test/results/clientpositive/llap/newline.q.out @@ -23,11 +23,11 @@ POSTHOOK: Lineage: tmp_tmp_n0.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: select * from tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 2 NULL 1 2 NULL 1 @@ -71,11 +71,11 @@ POSTHOOK: Lineage: tmp_tmp_n0.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: select * from tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1\n2 NULL 1\n2 NULL 1\n2 NULL @@ -85,12 +85,12 @@ PREHOOK: query: SELECT TRANSFORM(key, value) USING 'cat' AS (key, value) FROM tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT TRANSFORM(key, value) USING 'cat' AS (key, value) FROM tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1\n2 NULL 1\n2 NULL 1\n2 NULL @@ -113,11 +113,11 @@ POSTHOOK: Lineage: tmp_tmp_n0.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: select * from tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1\r2 NULL 1\r2 NULL 1\r2 NULL @@ -127,12 +127,12 @@ PREHOOK: query: SELECT TRANSFORM(key, value) USING 'cat' AS (key, value) FROM tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT TRANSFORM(key, value) USING 'cat' AS (key, value) FROM tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1\r2 NULL 1\r2 NULL 1\r2 NULL @@ -155,11 +155,11 @@ POSTHOOK: Lineage: tmp_tmp_n0.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: select * from tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1\t2 NULL 1\t2 NULL 1\t2 NULL @@ -169,12 +169,12 @@ PREHOOK: query: SELECT TRANSFORM(key, value) USING 'cat' AS (key, value) FROM tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT TRANSFORM(key, value) USING 'cat' AS (key, value) FROM tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1\t2 NULL 1\t2 NULL 1\t2 NULL @@ -197,11 +197,11 @@ POSTHOOK: Lineage: tmp_tmp_n0.value SCRIPT [(src)src.FieldSchema(name:key, type: PREHOOK: query: select * from tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1\ 2 NULL 1\ 2 NULL 1\ 2 NULL @@ -211,12 +211,12 @@ PREHOOK: query: SELECT TRANSFORM(key, value) USING 'cat' AS (key, value) FROM tmp_tmp_n0 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT TRANSFORM(key, value) USING 'cat' AS (key, value) FROM tmp_tmp_n0 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_tmp_n0 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1\ 2 NULL 1\ 2 NULL 1\ 2 NULL @@ -225,18 +225,18 @@ POSTHOOK: Input: default@tmp_tmp_n0 PREHOOK: query: SELECT key FROM (SELECT TRANSFORM ('a\tb', 'c') USING 'cat' AS (key, value) FROM src limit 1)a ORDER BY key ASC PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT key FROM (SELECT TRANSFORM ('a\tb', 'c') USING 'cat' AS (key, value) FROM src limit 1)a ORDER BY key ASC POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### a b PREHOOK: query: SELECT value FROM (SELECT TRANSFORM ('a\tb', 'c') USING 'cat' AS (key, value) FROM src limit 1)a ORDER BY value ASC PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT value FROM (SELECT TRANSFORM ('a\tb', 'c') USING 'cat' AS (key, value) FROM src limit 1)a ORDER BY value ASC POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### c diff --git ql/src/test/results/clientpositive/llap/nonreserved_keywords_insert_into1.q.out ql/src/test/results/clientpositive/llap/nonreserved_keywords_insert_into1.q.out new file mode 100644 index 0000000000..dda8546f48 --- /dev/null +++ ql/src/test/results/clientpositive/llap/nonreserved_keywords_insert_into1.q.out @@ -0,0 +1,449 @@ +PREHOOK: query: DROP TABLE `insert` +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE `insert` +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE `insert` (key INT, `as` STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert +POSTHOOK: query: CREATE TABLE `insert` (key INT, `as` STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert +PREHOOK: query: EXPLAIN INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert +POSTHOOK: query: EXPLAIN INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, as + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(as, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, as + Column Types: int, string + Table: default.insert + +PREHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert +POSTHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert +POSTHOOK: Lineage: insert.as SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: insert.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(HASH(hash)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (hash) FROM `insert` +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(hash)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (hash) FROM `insert` +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert +POSTHOOK: Output: hdfs://### HDFS PATH ### +10226524244 +PREHOOK: query: EXPLAIN INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert +POSTHOOK: query: EXPLAIN INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, as + Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(as, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, as + Column Types: int, string + Table: default.insert + +PREHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert +POSTHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert +POSTHOOK: Lineage: insert.as SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: insert.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(HASH(sum)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (sum) FROM `insert` +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(sum)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (sum) FROM `insert` +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert +POSTHOOK: Output: hdfs://### HDFS PATH ### +20453048488 +PREHOOK: query: SELECT COUNT(*) FROM `insert` +PREHOOK: type: QUERY +PREHOOK: Input: default@insert +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM `insert` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert +POSTHOOK: Output: hdfs://### HDFS PATH ### +200 +PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert +POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: key, as + Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(as, 'hll') + minReductionHashAggr: 0.9 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, as + Column Types: int, string + Table: default.insert + +PREHOOK: query: INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert +POSTHOOK: query: INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert +POSTHOOK: Lineage: insert.as SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: insert.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(HASH(add)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (add) FROM `insert` +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@insert +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT SUM(HASH(add)) FROM ( + SELECT TRANSFORM(*) USING 'tr \t _' AS (add) FROM `insert` +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert +POSTHOOK: Output: hdfs://### HDFS PATH ### +-826625916 +PREHOOK: query: DROP TABLE `insert` +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert +PREHOOK: Output: default@insert +POSTHOOK: query: DROP TABLE `insert` +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert +POSTHOOK: Output: default@insert diff --git ql/src/test/results/clientpositive/nullscript.q.out ql/src/test/results/clientpositive/llap/nullscript.q.out similarity index 72% rename from ql/src/test/results/clientpositive/nullscript.q.out rename to ql/src/test/results/clientpositive/llap/nullscript.q.out index cd926aa170..a3aab8da10 100644 --- ql/src/test/results/clientpositive/nullscript.q.out +++ ql/src/test/results/clientpositive/llap/nullscript.q.out @@ -26,41 +26,44 @@ PREHOOK: query: explain select transform(key) using 'cat' as key1 from nullscript PREHOOK: type: QUERY PREHOOK: Input: default@nullscript -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select transform(key) using 'cat' as key1 from nullscript POSTHOOK: type: QUERY POSTHOOK: Input: default@nullscript -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: nullscript - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nullscript Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -71,11 +74,11 @@ STAGE PLANS: PREHOOK: query: select transform(key) using 'cat' as key1 from nullscript PREHOOK: type: QUERY PREHOOK: Input: default@nullscript -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select transform(key) using 'cat' as key1 from nullscript POSTHOOK: type: QUERY POSTHOOK: Input: default@nullscript -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 238 86 311 diff --git ql/src/test/results/clientpositive/orc_createas1.q.out ql/src/test/results/clientpositive/llap/orc_createas1.q.out similarity index 51% rename from ql/src/test/results/clientpositive/orc_createas1.q.out rename to ql/src/test/results/clientpositive/llap/orc_createas1.q.out index 6884e8654e..fe86c273b6 100644 --- ql/src/test/results/clientpositive/orc_createas1.q.out +++ ql/src/test/results/clientpositive/llap/orc_createas1.q.out @@ -60,79 +60,73 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@orc_createas1b STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-8 depends on stages: Stage-0 - Stage-2 depends on stages: Stage-8 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-0, Stage-2 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_createas1b - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: col1, col2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_createas1b + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: col1, col2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-7 - Conditional Operator + Stage: Stage-2 + Dependency Collection Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-8 Create Table columns: key string, value string name: default.orc_createas1b @@ -140,7 +134,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde name: org.apache.hadoop.hive.ql.io.orc.OrcSerde - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -148,25 +142,11 @@ STAGE PLANS: Column Types: string, string Table: default.orc_createas1b - Stage: Stage-3 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-5 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-6 + Stage: Stage-0 Move Operator files: hdfs directory: true -#### A masked pattern was here #### + destination: hdfs://### HDFS PATH ### PREHOOK: query: CREATE TABLE orc_createas1b STORED AS ORC AS @@ -187,48 +167,64 @@ POSTHOOK: Lineage: orc_createas1b.value SIMPLE [(src)src.FieldSchema(name:value, PREHOOK: query: EXPLAIN SELECT * FROM orc_createas1b ORDER BY key LIMIT 5 PREHOOK: type: QUERY PREHOOK: Input: default@orc_createas1b -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: EXPLAIN SELECT * FROM orc_createas1b ORDER BY key LIMIT 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_createas1b -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: orc_createas1b - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orc_createas1b + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Top N Key Operator + sort order: + + keys: key (type: string) + null sort order: z + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + top n: 5 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 5 - Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -239,11 +235,11 @@ STAGE PLANS: PREHOOK: query: SELECT * FROM orc_createas1b ORDER BY key LIMIT 5 PREHOOK: type: QUERY PREHOOK: Input: default@orc_createas1b -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT * FROM orc_createas1b ORDER BY key LIMIT 5 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_createas1b -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 @@ -273,79 +269,73 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@orc_createas1c STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-8 depends on stages: Stage-0 - Stage-2 depends on stages: Stage-8 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-0, Stage-2 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: orc_createas1a - Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: string), (hash(key) pmod 50) (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_createas1c - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) - outputColumnNames: col1, col2, col3 - Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll'), compute_stats(col3, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true + Tez #### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orc_createas1a + Statistics: Num rows: 1000 Data size: 95000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: string), (hash(key) pmod 50) (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_createas1c + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) + outputColumnNames: col1, col2, col3 + Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll'), compute_stats(col3, 'hll') + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1320 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-8 + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 Create Table columns: key int, value string, part int name: default.orc_createas1c @@ -353,7 +343,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde name: org.apache.hadoop.hive.ql.io.orc.OrcSerde - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -361,25 +351,11 @@ STAGE PLANS: Column Types: int, string, int Table: default.orc_createas1c - Stage: Stage-3 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-5 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-6 + Stage: Stage-0 Move Operator files: hdfs directory: true -#### A masked pattern was here #### + destination: hdfs://### HDFS PATH ### PREHOOK: query: CREATE TABLE orc_createas1c STORED AS ORC AS @@ -412,7 +388,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_createas1a PREHOOK: Input: default@orc_createas1a@ds=1 PREHOOK: Input: default@orc_createas1a@ds=2 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) FROM orc_createas1a @@ -421,7 +397,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_createas1a POSTHOOK: Input: default@orc_createas1a@ds=1 POSTHOOK: Input: default@orc_createas1a@ds=2 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 14412220296 PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) @@ -429,14 +405,14 @@ PREHOOK: query: SELECT SUM(HASH(c)) FROM ( ) t PREHOOK: type: QUERY PREHOOK: Input: default@orc_createas1c -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) FROM orc_createas1c ) t POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_createas1c -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 14412220296 PREHOOK: query: DROP TABLE orc_createas1a PREHOOK: type: DROPTABLE diff --git ql/src/test/results/clientpositive/partcols1.q.out ql/src/test/results/clientpositive/llap/partcols1.q.out similarity index 95% rename from ql/src/test/results/clientpositive/partcols1.q.out rename to ql/src/test/results/clientpositive/llap/partcols1.q.out index edd7db2357..58580004af 100644 --- ql/src/test/results/clientpositive/partcols1.q.out +++ ql/src/test/results/clientpositive/llap/partcols1.q.out @@ -33,7 +33,7 @@ SELECT TRANSFORM( PREHOOK: type: QUERY PREHOOK: Input: default@test1_n15 PREHOOK: Input: default@test1_n15@partitionid=1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: FROM ( FROM test1_n15 SELECT partitionId, 111 as col2, 222 as col3, 333 as col4 @@ -50,7 +50,7 @@ SELECT TRANSFORM( POSTHOOK: type: QUERY POSTHOOK: Input: default@test1_n15 POSTHOOK: Input: default@test1_n15@partitionid=1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 111 222 333 1 111 222 333 1 111 222 333 diff --git ql/src/test/results/clientpositive/llap/ppd_transform.q.out ql/src/test/results/clientpositive/llap/ppd_transform.q.out new file mode 100644 index 0000000000..1505011626 --- /dev/null +++ ql/src/test/results/clientpositive/llap/ppd_transform.q.out @@ -0,0 +1,459 @@ +PREHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0 < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +11 val_11 +12 val_12 +12 val_12 +15 val_15 +15 val_15 +17 val_17 +18 val_18 +18 val_18 +19 val_19 +2 val_2 +20 val_20 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +34 val_34 +35 val_35 +35 val_35 +35 val_35 +37 val_37 +37 val_37 +4 val_4 +41 val_41 +42 val_42 +42 val_42 +43 val_43 +44 val_44 +47 val_47 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +67 val_67 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +74 val_74 +76 val_76 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +PREHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0 < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'cat' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +11 val_11 +12 val_12 +12 val_12 +15 val_15 +15 val_15 +17 val_17 +18 val_18 +18 val_18 +19 val_19 +2 val_2 +20 val_20 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +27 val_27 +28 val_28 +30 val_30 +33 val_33 +34 val_34 +35 val_35 +35 val_35 +35 val_35 +37 val_37 +37 val_37 +4 val_4 +41 val_41 +42 val_42 +42 val_42 +43 val_43 +44 val_44 +47 val_47 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +67 val_67 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +74 val_74 +76 val_76 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +PREHOOK: query: EXPLAIN +FROM ( + FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y +) reduced +#### A masked pattern was here #### +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +FROM ( + FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y +) reduced +#### A masked pattern was here #### +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@src +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0) IN ('a', 'b') (type: boolean) + Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col0) IN ('c', 'd') (type: boolean) + Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + files: + hdfs directory: false +#### A masked pattern was here #### + + Stage: Stage-1 + Move Operator + files: + hdfs directory: false +#### A masked pattern was here #### + diff --git ql/src/test/results/clientpositive/query_with_semi.q.out ql/src/test/results/clientpositive/llap/query_with_semi.q.out similarity index 74% rename from ql/src/test/results/clientpositive/query_with_semi.q.out rename to ql/src/test/results/clientpositive/llap/query_with_semi.q.out index 93da006251..5a701ca059 100644 --- ql/src/test/results/clientpositive/query_with_semi.q.out +++ ql/src/test/results/clientpositive/llap/query_with_semi.q.out @@ -2,32 +2,32 @@ PREHOOK: query: from src select transform('aa\;') using 'cat' as a limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: from src select transform('aa\;') using 'cat' as a limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### aa; PREHOOK: query: from src select transform('bb') using 'cat' as b limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: from src select transform('bb') using 'cat' as b limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### bb PREHOOK: query: from src select transform('cc') using 'cat' as c limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: from src select transform('cc') using 'cat' as c limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### cc diff --git ql/src/test/results/clientpositive/rcfile_bigdata.q.out ql/src/test/results/clientpositive/llap/rcfile_bigdata.q.out similarity index 96% rename from ql/src/test/results/clientpositive/rcfile_bigdata.q.out rename to ql/src/test/results/clientpositive/llap/rcfile_bigdata.q.out index c1ada45ad0..9b79077e71 100644 --- ql/src/test/results/clientpositive/rcfile_bigdata.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_bigdata.q.out @@ -39,9 +39,9 @@ value string PREHOOK: query: select count(columnTable_Bigdata.key) from columnTable_Bigdata PREHOOK: type: QUERY PREHOOK: Input: default@columntable_bigdata -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select count(columnTable_Bigdata.key) from columnTable_Bigdata POSTHOOK: type: QUERY POSTHOOK: Input: default@columntable_bigdata -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 5005500 diff --git ql/src/test/results/clientpositive/llap/regexp_extract.q.out ql/src/test/results/clientpositive/llap/regexp_extract.q.out new file mode 100644 index 0000000000..dc2842b0df --- /dev/null +++ ql/src/test/results/clientpositive/llap/regexp_extract.q.out @@ -0,0 +1,544 @@ +PREHOOK: query: EXPLAIN EXTENDED +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN EXTENDED +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string), 3 (type: int), 7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string,string + field.delim 9 + serialization.format 9 + serialization.last.column.takes.rest true + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + isSamplingPred: false + predicate: (_col0 < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: a + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: true + Path -> Alias: + hdfs://### HDFS PATH ### [src] + Path -> Partition: + hdfs://### HDFS PATH ### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), regexp_extract(VALUE._col1, 'val_(\d+\t\d+)', 1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 44986 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 + directory: hdfs://### HDFS PATH ### + NumFilesPerFileSink: 1 + Statistics: Num rows: 166 Data size: 44986 Basic stats: COMPLETE Column stats: COMPLETE + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 3 +0 0 3 +0 0 3 +10 10 3 +11 11 3 +12 12 3 +12 12 3 +15 15 3 +15 15 3 +17 17 3 +18 18 3 +18 18 3 +19 19 3 +2 2 3 +20 20 3 +24 24 3 +24 24 3 +26 26 3 +26 26 3 +27 27 3 +28 28 3 +30 30 3 +33 33 3 +34 34 3 +35 35 3 +35 35 3 +35 35 3 +37 37 3 +37 37 3 +4 4 3 +41 41 3 +42 42 3 +42 42 3 +43 43 3 +44 44 3 +47 47 3 +5 5 3 +5 5 3 +5 5 3 +51 51 3 +51 51 3 +53 53 3 +54 54 3 +57 57 3 +58 58 3 +58 58 3 +64 64 3 +65 65 3 +66 66 3 +67 67 3 +67 67 3 +69 69 3 +70 70 3 +70 70 3 +70 70 3 +72 72 3 +72 72 3 +74 74 3 +76 76 3 +76 76 3 +77 77 3 +78 78 3 +8 8 3 +80 80 3 +82 82 3 +83 83 3 +83 83 3 +84 84 3 +84 84 3 +85 85 3 +86 86 3 +87 87 3 +9 9 3 +90 90 3 +90 90 3 +90 90 3 +92 92 3 +95 95 3 +95 95 3 +96 96 3 +97 97 3 +97 97 3 +98 98 3 +98 98 3 +PREHOOK: query: EXPLAIN EXTENDED +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN EXTENDED +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string), 3 (type: int), 7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string,string + field.delim 9 + serialization.format 9 + serialization.last.column.takes.rest true + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + isSamplingPred: false + predicate: (_col0 < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: a + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: true + Path -> Alias: + hdfs://### HDFS PATH ### [src] + Path -> Partition: + hdfs://### HDFS PATH ### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), regexp_extract(VALUE._col1, 'val_(\d+\t\d+)') (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 44986 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 + directory: hdfs://### HDFS PATH ### + NumFilesPerFileSink: 1 + Statistics: Num rows: 166 Data size: 44986 Basic stats: COMPLETE Column stats: COMPLETE + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) + USING 'cat' + CLUSTER BY key +) tmap +SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 3 +0 0 3 +0 0 3 +10 10 3 +11 11 3 +12 12 3 +12 12 3 +15 15 3 +15 15 3 +17 17 3 +18 18 3 +18 18 3 +19 19 3 +2 2 3 +20 20 3 +24 24 3 +24 24 3 +26 26 3 +26 26 3 +27 27 3 +28 28 3 +30 30 3 +33 33 3 +34 34 3 +35 35 3 +35 35 3 +35 35 3 +37 37 3 +37 37 3 +4 4 3 +41 41 3 +42 42 3 +42 42 3 +43 43 3 +44 44 3 +47 47 3 +5 5 3 +5 5 3 +5 5 3 +51 51 3 +51 51 3 +53 53 3 +54 54 3 +57 57 3 +58 58 3 +58 58 3 +64 64 3 +65 65 3 +66 66 3 +67 67 3 +67 67 3 +69 69 3 +70 70 3 +70 70 3 +70 70 3 +72 72 3 +72 72 3 +74 74 3 +76 76 3 +76 76 3 +77 77 3 +78 78 3 +8 8 3 +80 80 3 +82 82 3 +83 83 3 +83 83 3 +84 84 3 +84 84 3 +85 85 3 +86 86 3 +87 87 3 +9 9 3 +90 90 3 +90 90 3 +90 90 3 +92 92 3 +95 95 3 +95 95 3 +96 96 3 +97 97 3 +97 97 3 +98 98 3 +98 98 3 diff --git ql/src/test/results/clientpositive/script_env_var1.q.out ql/src/test/results/clientpositive/llap/script_env_var1.q.out similarity index 89% rename from ql/src/test/results/clientpositive/script_env_var1.q.out rename to ql/src/test/results/clientpositive/llap/script_env_var1.q.out index c1181b2635..8a0fc7d5da 100644 --- ql/src/test/results/clientpositive/script_env_var1.q.out +++ ql/src/test/results/clientpositive/llap/script_env_var1.q.out @@ -3,12 +3,12 @@ PREHOOK: query: SELECT count(1) FROM SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src order by key LIMIT 1)y ) a GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT count(1) FROM ( SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src order by key LIMIT 1)x UNION ALL SELECT * FROM (SELECT TRANSFORM('echo $HIVE_SCRIPT_OPERATOR_ID') USING 'sh' AS key FROM src order by key LIMIT 1)y ) a GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 1 diff --git ql/src/test/results/clientpositive/script_env_var2.q.out ql/src/test/results/clientpositive/llap/script_env_var2.q.out similarity index 87% rename from ql/src/test/results/clientpositive/script_env_var2.q.out rename to ql/src/test/results/clientpositive/llap/script_env_var2.q.out index 58a0936858..5184c682c8 100644 --- ql/src/test/results/clientpositive/script_env_var2.q.out +++ ql/src/test/results/clientpositive/llap/script_env_var2.q.out @@ -3,12 +3,12 @@ PREHOOK: query: SELECT count(1) FROM SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT count(1) FROM ( SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)a UNION ALL SELECT * FROM (SELECT TRANSFORM('echo $MY_ID') USING 'sh' AS key FROM src LIMIT 1)b ) a GROUP BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 1 diff --git ql/src/test/results/clientpositive/llap/script_pipe.q.out ql/src/test/results/clientpositive/llap/script_pipe.q.out new file mode 100644 index 0000000000..26e8ec04ff --- /dev/null +++ ql/src/test/results/clientpositive/llap/script_pipe.q.out @@ -0,0 +1,134 @@ +PREHOOK: query: EXPLAIN SELECT TRANSFORM(*) USING 'true' AS a, b, c FROM (SELECT * FROM src LIMIT 1) tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN SELECT TRANSFORM(*) USING 'true' AS a, b, c FROM (SELECT * FROM src LIMIT 1) tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: true + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN SELECT TRANSFORM(key, value, key, value, key, value, key, value, key, value, key, value) USING 'head -n 1' as a,b,c,d FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN SELECT TRANSFORM(key, value, key, value, key, value, key, value, key, value, key, value) USING 'head -n 1' as a,b,c,d FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), key (type: string), value (type: string), key (type: string), value (type: string), key (type: string), value (type: string), key (type: string), value (type: string), key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 500 Data size: 534000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: head -n 1 + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 534000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 534000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT TRANSFORM(*) USING 'true' AS a, b, c FROM (SELECT * FROM src LIMIT 1) tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT TRANSFORM(*) USING 'true' AS a, b, c FROM (SELECT * FROM src LIMIT 1) tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: SELECT TRANSFORM(key, value, key, value, key, value, key, value, key, value, key, value) USING 'head -n 1' as a,b,c,d FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT TRANSFORM(key, value, key, value, key, value, key, value, key, value, key, value) USING 'head -n 1' as a,b,c,d FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: hdfs://### HDFS PATH ### +238 val_238 238 val_238 diff --git ql/src/test/results/clientpositive/scriptfile1.q.out ql/src/test/results/clientpositive/llap/scriptfile1.q.out similarity index 94% rename from ql/src/test/results/clientpositive/scriptfile1.q.out rename to ql/src/test/results/clientpositive/llap/scriptfile1.q.out index 29ee2df4b3..f374501df0 100644 --- ql/src/test/results/clientpositive/scriptfile1.q.out +++ ql/src/test/results/clientpositive/llap/scriptfile1.q.out @@ -31,11 +31,11 @@ POSTHOOK: Lineage: dest1_n22.value SCRIPT [(src)src.FieldSchema(name:key, type:s PREHOOK: query: SELECT dest1_n22.* FROM dest1_n22 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n22 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT dest1_n22.* FROM dest1_n22 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n22 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 10 val_10 100 val_100 100 val_100 diff --git ql/src/test/results/clientpositive/select_transform_hint.q.out ql/src/test/results/clientpositive/llap/select_transform_hint.q.out similarity index 82% rename from ql/src/test/results/clientpositive/select_transform_hint.q.out rename to ql/src/test/results/clientpositive/llap/select_transform_hint.q.out index e0c1cea301..f044c22c7a 100644 --- ql/src/test/results/clientpositive/select_transform_hint.q.out +++ ql/src/test/results/clientpositive/llap/select_transform_hint.q.out @@ -6,7 +6,7 @@ on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: cat PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: EXPLAIN SELECT /*+MAPJOIN(a)*/ TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) @@ -15,67 +15,78 @@ on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: cat POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-1 depends on stages: Stage-3 + Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-3 - Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 key (type: string) - 1 key (type: string) - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Reduce Operator Tree: + Merge Join Operator condition map: Inner Join 0 to 1 keys: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE Transform Operator command: cat output info: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Local Work: - Map Reduce Local Work Stage: Stage-0 Fetch Operator @@ -90,7 +101,7 @@ on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: cat PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT /*+MAPJOIN(a)*/ TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) FROM src a join src b @@ -98,1035 +109,1035 @@ on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: cat POSTHOOK: Input: default@src -#### A masked pattern was here #### -238 val_238 -238 val_238 -86 val_86 -311 val_311 -311 val_311 -311 val_311 -27 val_27 -165 val_165 -165 val_165 -409 val_409 -409 val_409 -409 val_409 -255 val_255 -255 val_255 -278 val_278 -278 val_278 -98 val_98 -98 val_98 -484 val_484 -265 val_265 -265 val_265 -193 val_193 -193 val_193 -193 val_193 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -150 val_150 -273 val_273 -273 val_273 -273 val_273 -224 val_224 -224 val_224 -369 val_369 -369 val_369 -369 val_369 -66 val_66 +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +100 val_100 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +104 val_104 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +113 val_113 +113 val_113 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +118 val_118 +118 val_118 +118 val_118 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +12 val_12 +12 val_12 +120 val_120 +120 val_120 +120 val_120 +120 val_120 +125 val_125 +125 val_125 +125 val_125 +125 val_125 +126 val_126 128 val_128 128 val_128 128 val_128 -213 val_213 -213 val_213 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +129 val_129 +129 val_129 +129 val_129 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +134 val_134 +134 val_134 +134 val_134 +136 val_136 +137 val_137 +137 val_137 +137 val_137 +137 val_137 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +143 val_143 +145 val_145 146 val_146 146 val_146 -406 val_406 -406 val_406 -406 val_406 -406 val_406 -429 val_429 -429 val_429 -374 val_374 -152 val_152 -152 val_152 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -145 val_145 -495 val_495 -37 val_37 -37 val_37 -327 val_327 -327 val_327 -327 val_327 -281 val_281 -281 val_281 -277 val_277 -277 val_277 -277 val_277 -277 val_277 -209 val_209 -209 val_209 +146 val_146 +146 val_146 +149 val_149 +149 val_149 +149 val_149 +149 val_149 15 val_15 15 val_15 -82 val_82 -403 val_403 -403 val_403 -403 val_403 -166 val_166 -417 val_417 -417 val_417 -417 val_417 -430 val_430 -430 val_430 -430 val_430 -252 val_252 -292 val_292 -219 val_219 -219 val_219 -287 val_287 +15 val_15 +15 val_15 +150 val_150 +152 val_152 +152 val_152 +152 val_152 +152 val_152 153 val_153 -193 val_193 -193 val_193 -193 val_193 -338 val_338 -446 val_446 -459 val_459 -459 val_459 -394 val_394 -237 val_237 -237 val_237 -482 val_482 -174 val_174 -174 val_174 -413 val_413 -413 val_413 -494 val_494 -207 val_207 -207 val_207 -199 val_199 -199 val_199 -199 val_199 -466 val_466 -466 val_466 -466 val_466 -208 val_208 -208 val_208 -208 val_208 -174 val_174 -174 val_174 -399 val_399 -399 val_399 -396 val_396 -396 val_396 -396 val_396 -247 val_247 -417 val_417 -417 val_417 -417 val_417 -489 val_489 -489 val_489 -489 val_489 -489 val_489 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 162 val_162 -377 val_377 -397 val_397 -397 val_397 -309 val_309 -309 val_309 -365 val_365 -266 val_266 -439 val_439 -439 val_439 -342 val_342 -342 val_342 -367 val_367 -367 val_367 -325 val_325 -325 val_325 +163 val_163 +164 val_164 +164 val_164 +164 val_164 +164 val_164 +165 val_165 +165 val_165 +165 val_165 +165 val_165 +166 val_166 +167 val_167 +167 val_167 167 val_167 167 val_167 167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +168 val_168 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +172 val_172 +172 val_172 +172 val_172 +174 val_174 +174 val_174 +174 val_174 +174 val_174 +175 val_175 +175 val_175 +175 val_175 +175 val_175 +176 val_176 +176 val_176 +176 val_176 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +179 val_179 +179 val_179 +179 val_179 +18 val_18 +18 val_18 +18 val_18 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +191 val_191 +191 val_191 +191 val_191 +192 val_192 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +194 val_194 195 val_195 195 val_195 -475 val_475 -17 val_17 -113 val_113 -113 val_113 -155 val_155 +195 val_195 +195 val_195 +196 val_196 +197 val_197 +197 val_197 +197 val_197 +197 val_197 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +2 val_2 +20 val_20 +200 val_200 +200 val_200 +200 val_200 +200 val_200 +201 val_201 +202 val_202 +203 val_203 +203 val_203 203 val_203 203 val_203 -339 val_339 -0 val_0 -0 val_0 -0 val_0 -455 val_455 -128 val_128 -128 val_128 -128 val_128 -311 val_311 -311 val_311 -311 val_311 -316 val_316 -316 val_316 -316 val_316 -57 val_57 -302 val_302 205 val_205 205 val_205 -149 val_149 -149 val_149 -438 val_438 -438 val_438 -438 val_438 -345 val_345 -129 val_129 -129 val_129 -170 val_170 -20 val_20 -489 val_489 -489 val_489 -489 val_489 -489 val_489 -157 val_157 -378 val_378 -221 val_221 -221 val_221 -92 val_92 -111 val_111 -47 val_47 -72 val_72 -72 val_72 -4 val_4 -280 val_280 -280 val_280 -35 val_35 -35 val_35 -35 val_35 -427 val_427 -277 val_277 -277 val_277 -277 val_277 -277 val_277 +205 val_205 +205 val_205 +207 val_207 +207 val_207 +207 val_207 +207 val_207 208 val_208 208 val_208 208 val_208 -356 val_356 -399 val_399 -399 val_399 -169 val_169 -169 val_169 -169 val_169 -169 val_169 -382 val_382 -382 val_382 -498 val_498 -498 val_498 -498 val_498 -125 val_125 -125 val_125 -386 val_386 -437 val_437 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -192 val_192 -286 val_286 -187 val_187 -187 val_187 -187 val_187 -176 val_176 -176 val_176 -54 val_54 -459 val_459 -459 val_459 -51 val_51 -51 val_51 -138 val_138 -138 val_138 -138 val_138 -138 val_138 -103 val_103 -103 val_103 -239 val_239 -239 val_239 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +209 val_209 +209 val_209 +209 val_209 +209 val_209 +213 val_213 +213 val_213 213 val_213 213 val_213 +214 val_214 216 val_216 216 val_216 -430 val_430 -430 val_430 -430 val_430 -278 val_278 -278 val_278 -176 val_176 -176 val_176 -289 val_289 -221 val_221 -221 val_221 -65 val_65 -318 val_318 -318 val_318 -318 val_318 -332 val_332 -311 val_311 -311 val_311 -311 val_311 -275 val_275 -137 val_137 -137 val_137 -241 val_241 -83 val_83 -83 val_83 -333 val_333 -333 val_333 -180 val_180 -284 val_284 -12 val_12 -12 val_12 +216 val_216 +216 val_216 +217 val_217 +217 val_217 +217 val_217 +217 val_217 +218 val_218 +219 val_219 +219 val_219 +219 val_219 +219 val_219 +221 val_221 +221 val_221 +221 val_221 +221 val_221 +222 val_222 +223 val_223 +223 val_223 +223 val_223 +223 val_223 +224 val_224 +224 val_224 +224 val_224 +224 val_224 +226 val_226 +228 val_228 +229 val_229 +229 val_229 +229 val_229 +229 val_229 230 val_230 230 val_230 230 val_230 230 val_230 230 val_230 -181 val_181 -67 val_67 -67 val_67 -260 val_260 -404 val_404 -404 val_404 -384 val_384 -384 val_384 -384 val_384 -489 val_489 -489 val_489 -489 val_489 -489 val_489 -353 val_353 -353 val_353 -373 val_373 -272 val_272 -272 val_272 -138 val_138 -138 val_138 -138 val_138 -138 val_138 -217 val_217 -217 val_217 -84 val_84 -84 val_84 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -466 val_466 -466 val_466 -466 val_466 -58 val_58 -58 val_58 -8 val_8 -411 val_411 230 val_230 230 val_230 230 val_230 230 val_230 230 val_230 -208 val_208 -208 val_208 -208 val_208 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -24 val_24 -24 val_24 -463 val_463 -463 val_463 -431 val_431 -431 val_431 -431 val_431 -179 val_179 -179 val_179 -172 val_172 -172 val_172 -42 val_42 -42 val_42 -129 val_129 -129 val_129 -158 val_158 -119 val_119 -119 val_119 -119 val_119 -496 val_496 -0 val_0 -0 val_0 -0 val_0 -322 val_322 -322 val_322 -197 val_197 -197 val_197 -468 val_468 -468 val_468 -468 val_468 -468 val_468 -393 val_393 -454 val_454 -454 val_454 -454 val_454 -100 val_100 -100 val_100 -298 val_298 -298 val_298 -298 val_298 -199 val_199 -199 val_199 -199 val_199 -191 val_191 -191 val_191 -418 val_418 -96 val_96 -26 val_26 -26 val_26 -165 val_165 -165 val_165 -327 val_327 -327 val_327 -327 val_327 230 val_230 230 val_230 230 val_230 230 val_230 230 val_230 -205 val_205 -205 val_205 -120 val_120 -120 val_120 -131 val_131 -51 val_51 -51 val_51 -404 val_404 -404 val_404 -43 val_43 -436 val_436 -156 val_156 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -468 val_468 -468 val_468 -468 val_468 -468 val_468 -308 val_308 -95 val_95 -95 val_95 -196 val_196 -288 val_288 -288 val_288 -481 val_481 -457 val_457 -98 val_98 -98 val_98 -282 val_282 -282 val_282 -197 val_197 -197 val_197 -187 val_187 -187 val_187 -187 val_187 -318 val_318 -318 val_318 -318 val_318 -318 val_318 -318 val_318 -318 val_318 -409 val_409 -409 val_409 -409 val_409 -470 val_470 -137 val_137 -137 val_137 -369 val_369 -369 val_369 -369 val_369 -316 val_316 -316 val_316 -316 val_316 -169 val_169 -169 val_169 -169 val_169 -169 val_169 -413 val_413 -413 val_413 -85 val_85 -77 val_77 -0 val_0 -0 val_0 -0 val_0 -490 val_490 -87 val_87 -364 val_364 -179 val_179 -179 val_179 -118 val_118 -118 val_118 -134 val_134 -134 val_134 -395 val_395 -395 val_395 -282 val_282 -282 val_282 -138 val_138 -138 val_138 -138 val_138 -138 val_138 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +233 val_233 +233 val_233 +233 val_233 +233 val_233 +235 val_235 +237 val_237 +237 val_237 +237 val_237 +237 val_237 238 val_238 238 val_238 -419 val_419 -15 val_15 -15 val_15 -118 val_118 -118 val_118 -72 val_72 -72 val_72 -90 val_90 -90 val_90 -90 val_90 -307 val_307 -307 val_307 -19 val_19 -435 val_435 -10 val_10 +238 val_238 +238 val_238 +239 val_239 +239 val_239 +239 val_239 +239 val_239 +24 val_24 +24 val_24 +24 val_24 +24 val_24 +241 val_241 +242 val_242 +242 val_242 +242 val_242 +242 val_242 +244 val_244 +247 val_247 +248 val_248 +249 val_249 +252 val_252 +255 val_255 +255 val_255 +255 val_255 +255 val_255 +256 val_256 +256 val_256 +256 val_256 +256 val_256 +257 val_257 +258 val_258 +26 val_26 +26 val_26 +26 val_26 +26 val_26 +260 val_260 +262 val_262 +263 val_263 +265 val_265 +265 val_265 +265 val_265 +265 val_265 +266 val_266 +27 val_27 +272 val_272 +272 val_272 +272 val_272 +272 val_272 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +274 val_274 +275 val_275 277 val_277 277 val_277 277 val_277 277 val_277 -273 val_273 -273 val_273 -273 val_273 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +278 val_278 +278 val_278 +278 val_278 +278 val_278 +28 val_28 +280 val_280 +280 val_280 +280 val_280 +280 val_280 +281 val_281 +281 val_281 +281 val_281 +281 val_281 +282 val_282 +282 val_282 +282 val_282 +282 val_282 +283 val_283 +284 val_284 +285 val_285 +286 val_286 +287 val_287 +288 val_288 +288 val_288 +288 val_288 +288 val_288 +289 val_289 +291 val_291 +292 val_292 +296 val_296 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +30 val_30 +302 val_302 +305 val_305 306 val_306 -224 val_224 -224 val_224 +307 val_307 +307 val_307 +307 val_307 +307 val_307 +308 val_308 309 val_309 309 val_309 -389 val_389 +309 val_309 +309 val_309 +310 val_310 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +317 val_317 +317 val_317 +317 val_317 +317 val_317 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +321 val_321 +321 val_321 +321 val_321 +321 val_321 +322 val_322 +322 val_322 +322 val_322 +322 val_322 +323 val_323 +325 val_325 +325 val_325 +325 val_325 +325 val_325 327 val_327 327 val_327 327 val_327 -242 val_242 -242 val_242 -369 val_369 -369 val_369 -369 val_369 -392 val_392 -272 val_272 -272 val_272 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +33 val_33 331 val_331 331 val_331 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -242 val_242 -242 val_242 -452 val_452 -177 val_177 -226 val_226 -5 val_5 -5 val_5 -5 val_5 -497 val_497 -402 val_402 -396 val_396 -396 val_396 -396 val_396 -317 val_317 -317 val_317 -395 val_395 -395 val_395 -58 val_58 -58 val_58 -35 val_35 -35 val_35 -35 val_35 +331 val_331 +331 val_331 +332 val_332 +333 val_333 +333 val_333 +333 val_333 +333 val_333 +335 val_335 336 val_336 -95 val_95 -95 val_95 -11 val_11 -168 val_168 +338 val_338 +339 val_339 34 val_34 -229 val_229 -229 val_229 -233 val_233 -233 val_233 -143 val_143 -472 val_472 -322 val_322 -322 val_322 -498 val_498 -498 val_498 -498 val_498 -160 val_160 -195 val_195 -195 val_195 -42 val_42 -42 val_42 -321 val_321 -321 val_321 -430 val_430 -430 val_430 -430 val_430 -119 val_119 -119 val_119 -119 val_119 -489 val_489 -489 val_489 -489 val_489 -489 val_489 -458 val_458 -458 val_458 -78 val_78 -76 val_76 -76 val_76 -41 val_41 -223 val_223 -223 val_223 -492 val_492 -492 val_492 -149 val_149 -149 val_149 -449 val_449 -218 val_218 -228 val_228 -138 val_138 -138 val_138 -138 val_138 -138 val_138 -453 val_453 -30 val_30 -209 val_209 -209 val_209 -64 val_64 -468 val_468 -468 val_468 -468 val_468 -468 val_468 -76 val_76 -76 val_76 -74 val_74 +341 val_341 342 val_342 342 val_342 -69 val_69 -230 val_230 -230 val_230 -230 val_230 -230 val_230 -230 val_230 -33 val_33 -368 val_368 -103 val_103 -103 val_103 -296 val_296 -113 val_113 -113 val_113 -216 val_216 -216 val_216 +342 val_342 +342 val_342 +344 val_344 +344 val_344 +344 val_344 +344 val_344 +345 val_345 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +351 val_351 +353 val_353 +353 val_353 +353 val_353 +353 val_353 +356 val_356 +360 val_360 +362 val_362 +364 val_364 +365 val_365 +366 val_366 367 val_367 367 val_367 -344 val_344 -344 val_344 -167 val_167 -167 val_167 -167 val_167 -274 val_274 -219 val_219 -219 val_219 -239 val_239 -239 val_239 -485 val_485 -116 val_116 -223 val_223 -223 val_223 -256 val_256 -256 val_256 -263 val_263 -70 val_70 -70 val_70 -70 val_70 -487 val_487 -480 val_480 -480 val_480 -480 val_480 +367 val_367 +367 val_367 +368 val_368 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +37 val_37 +37 val_37 +37 val_37 +37 val_37 +373 val_373 +374 val_374 +375 val_375 +377 val_377 +378 val_378 +379 val_379 +382 val_382 +382 val_382 +382 val_382 +382 val_382 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +386 val_386 +389 val_389 +392 val_392 +393 val_393 +394 val_394 +395 val_395 +395 val_395 +395 val_395 +395 val_395 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +397 val_397 +397 val_397 +397 val_397 +397 val_397 +399 val_399 +399 val_399 +399 val_399 +399 val_399 +4 val_4 +400 val_400 401 val_401 401 val_401 401 val_401 401 val_401 401 val_401 -288 val_288 -288 val_288 -191 val_191 -191 val_191 -5 val_5 -5 val_5 -5 val_5 -244 val_244 -438 val_438 -438 val_438 -438 val_438 -128 val_128 -128 val_128 -128 val_128 -467 val_467 -432 val_432 -202 val_202 -316 val_316 -316 val_316 -316 val_316 -229 val_229 -229 val_229 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -469 val_469 -463 val_463 -463 val_463 -280 val_280 -280 val_280 -2 val_2 -35 val_35 -35 val_35 -35 val_35 -283 val_283 -331 val_331 -331 val_331 -235 val_235 -80 val_80 -44 val_44 -193 val_193 -193 val_193 -193 val_193 -321 val_321 -321 val_321 -335 val_335 -104 val_104 -104 val_104 -466 val_466 -466 val_466 -466 val_466 -366 val_366 -175 val_175 -175 val_175 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +402 val_402 403 val_403 403 val_403 403 val_403 -483 val_483 -53 val_53 -105 val_105 -257 val_257 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +404 val_404 +404 val_404 +404 val_404 +404 val_404 406 val_406 406 val_406 406 val_406 406 val_406 -409 val_409 -409 val_409 -409 val_409 -190 val_190 406 val_406 406 val_406 406 val_406 406 val_406 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -114 val_114 -258 val_258 -90 val_90 -90 val_90 -90 val_90 -203 val_203 -203 val_203 -262 val_262 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -348 val_348 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +407 val_407 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +41 val_41 +411 val_411 +413 val_413 +413 val_413 +413 val_413 +413 val_413 +414 val_414 +414 val_414 +414 val_414 +414 val_414 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +418 val_418 +419 val_419 +42 val_42 +42 val_42 +42 val_42 +42 val_42 +421 val_421 424 val_424 424 val_424 -12 val_12 -12 val_12 -396 val_396 -396 val_396 -396 val_396 -201 val_201 -217 val_217 -217 val_217 -164 val_164 -164 val_164 +424 val_424 +424 val_424 +427 val_427 +429 val_429 +429 val_429 +429 val_429 +429 val_429 +43 val_43 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 431 val_431 431 val_431 431 val_431 -454 val_454 -454 val_454 -454 val_454 -478 val_478 -478 val_478 -298 val_298 -298 val_298 -298 val_298 -125 val_125 -125 val_125 431 val_431 431 val_431 431 val_431 -164 val_164 -164 val_164 -424 val_424 -424 val_424 -187 val_187 -187 val_187 -187 val_187 -382 val_382 -382 val_382 -5 val_5 -5 val_5 -5 val_5 -70 val_70 -70 val_70 -70 val_70 -397 val_397 -397 val_397 -480 val_480 -480 val_480 -480 val_480 -291 val_291 -24 val_24 -24 val_24 -351 val_351 -255 val_255 -255 val_255 -104 val_104 -104 val_104 -70 val_70 -70 val_70 -70 val_70 -163 val_163 +431 val_431 +431 val_431 +431 val_431 +432 val_432 +435 val_435 +436 val_436 +437 val_437 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 438 val_438 438 val_438 438 val_438 -119 val_119 -119 val_119 -119 val_119 -414 val_414 -414 val_414 -200 val_200 -200 val_200 -491 val_491 -237 val_237 -237 val_237 439 val_439 439 val_439 -360 val_360 -248 val_248 -479 val_479 -305 val_305 -417 val_417 -417 val_417 -417 val_417 -199 val_199 -199 val_199 -199 val_199 -444 val_444 -120 val_120 -120 val_120 -429 val_429 -429 val_429 -169 val_169 -169 val_169 -169 val_169 -169 val_169 +439 val_439 +439 val_439 +44 val_44 443 val_443 -323 val_323 -325 val_325 -325 val_325 -277 val_277 -277 val_277 -277 val_277 -277 val_277 -230 val_230 -230 val_230 -230 val_230 -230 val_230 -230 val_230 -478 val_478 -478 val_478 -178 val_178 -468 val_468 -468 val_468 -468 val_468 -468 val_468 -310 val_310 -317 val_317 -317 val_317 -333 val_333 -333 val_333 -493 val_493 -460 val_460 -207 val_207 -207 val_207 -249 val_249 -265 val_265 -265 val_265 -480 val_480 -480 val_480 -480 val_480 -83 val_83 -83 val_83 -136 val_136 -353 val_353 -353 val_353 -172 val_172 -172 val_172 -214 val_214 -462 val_462 -462 val_462 -233 val_233 -233 val_233 -406 val_406 -406 val_406 -406 val_406 -406 val_406 -133 val_133 -175 val_175 -175 val_175 -189 val_189 +444 val_444 +446 val_446 +448 val_448 +449 val_449 +452 val_452 +453 val_453 454 val_454 454 val_454 454 val_454 -375 val_375 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -401 val_401 -421 val_421 -407 val_407 -384 val_384 -384 val_384 -384 val_384 -256 val_256 -256 val_256 -26 val_26 -26 val_26 -134 val_134 -134 val_134 -67 val_67 -67 val_67 -384 val_384 -384 val_384 -384 val_384 -379 val_379 -18 val_18 -18 val_18 -462 val_462 -462 val_462 -492 val_492 -492 val_492 -100 val_100 -100 val_100 -298 val_298 -298 val_298 -298 val_298 -9 val_9 -341 val_341 -498 val_498 -498 val_498 -498 val_498 -146 val_146 -146 val_146 -458 val_458 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +455 val_455 +457 val_457 458 val_458 -362 val_362 -186 val_186 -285 val_285 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -167 val_167 -167 val_167 -167 val_167 -18 val_18 -18 val_18 -273 val_273 -273 val_273 -273 val_273 -183 val_183 -281 val_281 -281 val_281 -344 val_344 -344 val_344 -97 val_97 -97 val_97 +458 val_458 +458 val_458 +458 val_458 +459 val_459 +459 val_459 +459 val_459 +459 val_459 +460 val_460 +462 val_462 +462 val_462 +462 val_462 +462 val_462 +463 val_463 +463 val_463 +463 val_463 +463 val_463 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +467 val_467 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 469 val_469 469 val_469 469 val_469 469 val_469 469 val_469 -315 val_315 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +47 val_47 +470 val_470 +472 val_472 +475 val_475 +477 val_477 +478 val_478 +478 val_478 +478 val_478 +478 val_478 +479 val_479 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +481 val_481 +482 val_482 +483 val_483 +484 val_484 +485 val_485 +487 val_487 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +490 val_490 +491 val_491 +492 val_492 +492 val_492 +492 val_492 +492 val_492 +493 val_493 +494 val_494 +495 val_495 +496 val_496 +497 val_497 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +51 val_51 +51 val_51 +53 val_53 +54 val_54 +57 val_57 +58 val_58 +58 val_58 +58 val_58 +58 val_58 +64 val_64 +65 val_65 +66 val_66 +67 val_67 +67 val_67 +67 val_67 +67 val_67 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +72 val_72 +72 val_72 +74 val_74 +76 val_76 +76 val_76 +76 val_76 +76 val_76 +77 val_77 +78 val_78 +8 val_8 +80 val_80 +82 val_82 +83 val_83 +83 val_83 +83 val_83 +83 val_83 84 val_84 84 val_84 -28 val_28 -37 val_37 -37 val_37 -448 val_448 -152 val_152 -152 val_152 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -348 val_348 -307 val_307 -307 val_307 -194 val_194 -414 val_414 -414 val_414 -477 val_477 -222 val_222 -126 val_126 +84 val_84 +84 val_84 +85 val_85 +86 val_86 +87 val_87 +9 val_9 90 val_90 90 val_90 90 val_90 -169 val_169 -169 val_169 -169 val_169 -169 val_169 -403 val_403 -403 val_403 -403 val_403 -400 val_400 -200 val_200 -200 val_200 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 97 val_97 97 val_97 +98 val_98 +98 val_98 +98 val_98 +98 val_98 PREHOOK: query: EXPLAIN SELECT /*+STREAMTABLE(a)*/ TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) @@ -1135,7 +1146,7 @@ on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: cat PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: EXPLAIN SELECT /*+STREAMTABLE(a)*/ TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) @@ -1144,65 +1155,78 @@ on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: cat POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - TableScan - alias: b - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: b + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1217,7 +1241,7 @@ on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: cat PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT /*+STREAMTABLE(a)*/ TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) FROM src a join src b @@ -1225,7 +1249,7 @@ on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: cat POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 diff --git ql/src/test/results/clientpositive/str_to_map.q.out ql/src/test/results/clientpositive/llap/str_to_map.q.out similarity index 66% rename from ql/src/test/results/clientpositive/str_to_map.q.out rename to ql/src/test/results/clientpositive/llap/str_to_map.q.out index f1aa4c14a0..4ddbb62265 100644 --- ql/src/test/results/clientpositive/str_to_map.q.out +++ ql/src/test/results/clientpositive/llap/str_to_map.q.out @@ -14,11 +14,11 @@ Function type:BUILTIN PREHOOK: query: explain select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-0 is a root stage @@ -29,35 +29,32 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: str_to_map('a=1,b=2,c=3',',','=')['a'] (type: string) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 42500 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 - Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 1 1 PREHOOK: query: explain select str_to_map('a:1,b:2,c:3') from src limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select str_to_map('a:1,b:2,c:3') from src limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-0 is a root stage @@ -68,35 +65,32 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: str_to_map('a:1,b:2,c:3') (type: map) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 377000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 - Statistics: Num rows: 3 Data size: 2262 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select str_to_map('a:1,b:2,c:3') from src limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select str_to_map('a:1,b:2,c:3') from src limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### {"a":"1","b":"2","c":"3"} {"a":"1","b":"2","c":"3"} {"a":"1","b":"2","c":"3"} PREHOOK: query: explain select str_to_map('a:1,b:2,c:3',',',':') from src limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select str_to_map('a:1,b:2,c:3',',',':') from src limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-0 is a root stage @@ -107,24 +101,21 @@ STAGE PLANS: Processor Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: str_to_map('a:1,b:2,c:3',',',':') (type: map) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 377000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 - Statistics: Num rows: 3 Data size: 2262 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select str_to_map('a:1,b:2,c:3',',',':') from src limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select str_to_map('a:1,b:2,c:3',',',':') from src limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### {"a":"1","b":"2","c":"3"} {"a":"1","b":"2","c":"3"} {"a":"1","b":"2","c":"3"} @@ -133,49 +124,52 @@ from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select str_to_map(t.ss,',',':')['a'] from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 'a:1,b:2,c:3' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: str_to_map(_col0,',',':')['a'] (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'a:1,b:2,c:3' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: str_to_map(_col0,',',':')['a'] (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -188,13 +182,13 @@ from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select str_to_map(t.ss,',',':')['a'] from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 1 1 @@ -216,11 +210,11 @@ POSTHOOK: Lineage: tbl_s2m.t SIMPLE [] PREHOOK: query: select str_to_map(t,'_','=')['333'] from tbl_s2m PREHOOK: type: QUERY PREHOOK: Input: default@tbl_s2m -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select str_to_map(t,'_','=')['333'] from tbl_s2m POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl_s2m -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 444 444 444 diff --git ql/src/test/results/clientpositive/temp_table_partcols1.q.out ql/src/test/results/clientpositive/llap/temp_table_partcols1.q.out similarity index 95% rename from ql/src/test/results/clientpositive/temp_table_partcols1.q.out rename to ql/src/test/results/clientpositive/llap/temp_table_partcols1.q.out index f2da1bf431..6b438dc5f4 100644 --- ql/src/test/results/clientpositive/temp_table_partcols1.q.out +++ ql/src/test/results/clientpositive/llap/temp_table_partcols1.q.out @@ -33,7 +33,7 @@ USING 'cat' as (a,b,c,d) PREHOOK: type: QUERY PREHOOK: Input: default@test1_n15_temp PREHOOK: Input: default@test1_n15_temp@partitionid=1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: FROM ( FROM test1_n15_temp SELECT partitionid, 111 as col2, 222 as col3, 333 as col4 @@ -50,7 +50,7 @@ USING 'cat' as (a,b,c,d) POSTHOOK: type: QUERY POSTHOOK: Input: default@test1_n15_temp POSTHOOK: Input: default@test1_n15_temp@partitionid=1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 111 222 333 1 111 222 333 1 111 222 333 diff --git ql/src/test/results/clientpositive/llap/transform1.q.out ql/src/test/results/clientpositive/llap/transform1.q.out new file mode 100644 index 0000000000..95fccb4979 --- /dev/null +++ ql/src/test/results/clientpositive/llap/transform1.q.out @@ -0,0 +1,150 @@ +PREHOOK: query: create table transform1_t1(a string, b string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@transform1_t1 +POSTHOOK: query: create table transform1_t1(a string, b string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@transform1_t1 +PREHOOK: query: EXPLAIN +SELECT transform(*) USING 'cat' AS (col array) FROM transform1_t1 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@transform1_t1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN +SELECT transform(*) USING 'cat' AS (col array) FROM transform1_t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@transform1_t1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: transform1_t1 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: string), b (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT transform(*) USING 'cat' AS (col array) FROM transform1_t1 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@transform1_t1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT transform(*) USING 'cat' AS (col array) FROM transform1_t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@transform1_t1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: create table transform1_t2(col array) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@transform1_t2 +POSTHOOK: query: create table transform1_t2(col array) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@transform1_t2 +PREHOOK: query: insert overwrite table transform1_t2 +select array(1,2,3) from src tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@transform1_t2 +POSTHOOK: query: insert overwrite table transform1_t2 +select array(1,2,3) from src tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@transform1_t2 +POSTHOOK: Lineage: transform1_t2.col EXPRESSION [] +PREHOOK: query: EXPLAIN +SELECT transform('0\0021\0022') USING 'cat' AS (col array) FROM transform1_t2 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@transform1_t2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN +SELECT transform('0\0021\0022') USING 'cat' AS (col array) FROM transform1_t2 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@transform1_t2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: transform1_t2 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: '012' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT transform('0\0021\0022') USING 'cat' AS (col array) FROM transform1_t2 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@transform1_t2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT transform('0\0021\0022') USING 'cat' AS (col array) FROM transform1_t2 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@transform1_t2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +[0,1,2] diff --git ql/src/test/results/clientpositive/transform2.q.out ql/src/test/results/clientpositive/llap/transform2.q.out similarity index 76% rename from ql/src/test/results/clientpositive/transform2.q.out rename to ql/src/test/results/clientpositive/llap/transform2.q.out index aeeaebf06b..42e1b272ec 100644 --- ql/src/test/results/clientpositive/transform2.q.out +++ ql/src/test/results/clientpositive/llap/transform2.q.out @@ -1,9 +1,9 @@ PREHOOK: query: SELECT TRANSFORM(substr(key, 1, 2)) USING 'cat' FROM src LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT TRANSFORM(substr(key, 1, 2)) USING 'cat' FROM src LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 23 NULL diff --git ql/src/test/results/clientpositive/transform3.q.out ql/src/test/results/clientpositive/llap/transform3.q.out similarity index 92% rename from ql/src/test/results/clientpositive/transform3.q.out rename to ql/src/test/results/clientpositive/llap/transform3.q.out index e3c55b8466..193f2606b2 100644 --- ql/src/test/results/clientpositive/transform3.q.out +++ ql/src/test/results/clientpositive/llap/transform3.q.out @@ -20,11 +20,11 @@ PREHOOK: query: SELECT t.newCol FROM ( ) t PREHOOK: type: QUERY PREHOOK: Input: default@transform3_t1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT t.newCol FROM ( SELECT TRANSFORM(col) USING 'cat' AS (NewCol string) FROM transform3_t1 ) t POSTHOOK: type: QUERY POSTHOOK: Input: default@transform3_t1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### aaaa diff --git ql/src/test/results/clientpositive/transform_acid.q.out ql/src/test/results/clientpositive/llap/transform_acid.q.out similarity index 95% rename from ql/src/test/results/clientpositive/transform_acid.q.out rename to ql/src/test/results/clientpositive/llap/transform_acid.q.out index bfa9aee0b0..597b5c62ee 100644 --- ql/src/test/results/clientpositive/transform_acid.q.out +++ ql/src/test/results/clientpositive/llap/transform_acid.q.out @@ -20,10 +20,10 @@ PREHOOK: query: SELECT transform(*) USING 'transform_acid_grep.sh' AS (col strin PREHOOK: type: QUERY PREHOOK: Input: default@transform_acid PREHOOK: Input: transform_acid_grep.sh -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT transform(*) USING 'transform_acid_grep.sh' AS (col string) FROM transform_acid POSTHOOK: type: QUERY POSTHOOK: Input: default@transform_acid POSTHOOK: Input: transform_acid_grep.sh -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### a diff --git ql/src/test/results/clientpositive/llap/transform_ppr1.q.out ql/src/test/results/clientpositive/llap/transform_ppr1.q.out new file mode 100644 index 0000000000..afb95a8f28 --- /dev/null +++ ql/src/test/results/clientpositive/llap/transform_ppr1.q.out @@ -0,0 +1,535 @@ +PREHOOK: query: EXPLAIN EXTENDED +FROM ( + FROM srcpart src + SELECT TRANSFORM(src.ds, src.key, src.value) + USING 'cat' AS (ds, tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN EXTENDED +FROM ( + FROM srcpart src + SELECT TRANSFORM(src.ds, src.key, src.value) + USING 'cat' AS (ds, tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 2000 Data size: 724000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: ds (type: string), key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2000 Data size: 724000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1,_col2 + columns.types string,string,string + field.delim 9 + serialization.format 9 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 2000 Data size: 724000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + isSamplingPred: false + predicate: ((_col1 < 100) and (_col0 = '2008-04-08')) (type: boolean) + Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col1 (type: string) + null sort order: a + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: '2008-04-08' (type: string), _col1 (type: string), _col2 (type: string) + auto parallelism: true + Path -> Alias: + hdfs://### HDFS PATH ### [src] + hdfs://### HDFS PATH ### [src] + hdfs://### HDFS PATH ### [src] + hdfs://### HDFS PATH ### [src] + Path -> Partition: + hdfs://### HDFS PATH ### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + hdfs://### HDFS PATH ### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + hdfs://### HDFS PATH ### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + hdfs://### HDFS PATH ### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + hr 12 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [src] + /srcpart/ds=2008-04-08/hr=12 [src] + /srcpart/ds=2008-04-09/hr=11 [src] + /srcpart/ds=2008-04-09/hr=12 [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 333 Data size: 59274 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 + directory: hdfs://### HDFS PATH ### + NumFilesPerFileSink: 1 + Statistics: Num rows: 333 Data size: 59274 Basic stats: COMPLETE Column stats: COMPLETE + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM ( + FROM srcpart src + SELECT TRANSFORM(src.ds, src.key, src.value) + USING 'cat' AS (ds, tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: FROM ( + FROM srcpart src + SELECT TRANSFORM(src.ds, src.key, src.value) + USING 'cat' AS (ds, tkey, tvalue) + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +10 val_10 +11 val_11 +11 val_11 +12 val_12 +12 val_12 +12 val_12 +12 val_12 +15 val_15 +15 val_15 +15 val_15 +15 val_15 +17 val_17 +17 val_17 +18 val_18 +18 val_18 +18 val_18 +18 val_18 +19 val_19 +19 val_19 +2 val_2 +2 val_2 +20 val_20 +20 val_20 +24 val_24 +24 val_24 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +26 val_26 +26 val_26 +27 val_27 +27 val_27 +28 val_28 +28 val_28 +30 val_30 +30 val_30 +33 val_33 +33 val_33 +34 val_34 +34 val_34 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +37 val_37 +37 val_37 +37 val_37 +37 val_37 +4 val_4 +4 val_4 +41 val_41 +41 val_41 +42 val_42 +42 val_42 +42 val_42 +42 val_42 +43 val_43 +43 val_43 +44 val_44 +44 val_44 +47 val_47 +47 val_47 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +51 val_51 +51 val_51 +53 val_53 +53 val_53 +54 val_54 +54 val_54 +57 val_57 +57 val_57 +58 val_58 +58 val_58 +58 val_58 +58 val_58 +64 val_64 +64 val_64 +65 val_65 +65 val_65 +66 val_66 +66 val_66 +67 val_67 +67 val_67 +67 val_67 +67 val_67 +69 val_69 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +72 val_72 +72 val_72 +74 val_74 +74 val_74 +76 val_76 +76 val_76 +76 val_76 +76 val_76 +77 val_77 +77 val_77 +78 val_78 +78 val_78 +8 val_8 +8 val_8 +80 val_80 +80 val_80 +82 val_82 +82 val_82 +83 val_83 +83 val_83 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +84 val_84 +84 val_84 +85 val_85 +85 val_85 +86 val_86 +86 val_86 +87 val_87 +87 val_87 +9 val_9 +9 val_9 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +92 val_92 +95 val_95 +95 val_95 +95 val_95 +95 val_95 +96 val_96 +96 val_96 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +98 val_98 +98 val_98 diff --git ql/src/test/results/clientpositive/llap/transform_ppr2.q.out ql/src/test/results/clientpositive/llap/transform_ppr2.q.out new file mode 100644 index 0000000000..68c46bd466 --- /dev/null +++ ql/src/test/results/clientpositive/llap/transform_ppr2.q.out @@ -0,0 +1,426 @@ +PREHOOK: query: EXPLAIN EXTENDED +FROM ( + FROM srcpart src + SELECT TRANSFORM(src.ds, src.key, src.value) + USING 'cat' AS (ds, tkey, tvalue) + WHERE src.ds = '2008-04-08' + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN EXTENDED +FROM ( + FROM srcpart src + SELECT TRANSFORM(src.ds, src.key, src.value) + USING 'cat' AS (ds, tkey, tvalue) + WHERE src.ds = '2008-04-08' + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: '2008-04-08' (type: string), key (type: string), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1000 Data size: 272000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1,_col2 + columns.types string,string,string + field.delim 9 + serialization.format 9 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1000 Data size: 272000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + isSamplingPred: false + predicate: (_col1 < 100) (type: boolean) + Statistics: Num rows: 333 Data size: 90576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col1 (type: string) + null sort order: a + numBuckets: -1 + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 333 Data size: 90576 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + auto parallelism: true + Path -> Alias: + hdfs://### HDFS PATH ### [src] + hdfs://### HDFS PATH ### [src] + Path -> Partition: + hdfs://### HDFS PATH ### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + hdfs://### HDFS PATH ### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + location hdfs://### HDFS PATH ### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [src] + /srcpart/ds=2008-04-08/hr=12 [src] + Reducer 2 + Execution mode: vectorized, llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 333 Data size: 59274 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + bucketingVersion: 2 + compressed: false + GlobalTableId: 0 + directory: hdfs://### HDFS PATH ### + NumFilesPerFileSink: 1 + Statistics: Num rows: 333 Data size: 59274 Basic stats: COMPLETE Column stats: COMPLETE + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + bucketing_version -1 + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM ( + FROM srcpart src + SELECT TRANSFORM(src.ds, src.key, src.value) + USING 'cat' AS (ds, tkey, tvalue) + WHERE src.ds = '2008-04-08' + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: FROM ( + FROM srcpart src + SELECT TRANSFORM(src.ds, src.key, src.value) + USING 'cat' AS (ds, tkey, tvalue) + WHERE src.ds = '2008-04-08' + CLUSTER BY tkey +) tmap +SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +10 val_10 +11 val_11 +11 val_11 +12 val_12 +12 val_12 +12 val_12 +12 val_12 +15 val_15 +15 val_15 +15 val_15 +15 val_15 +17 val_17 +17 val_17 +18 val_18 +18 val_18 +18 val_18 +18 val_18 +19 val_19 +19 val_19 +2 val_2 +2 val_2 +20 val_20 +20 val_20 +24 val_24 +24 val_24 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +26 val_26 +26 val_26 +27 val_27 +27 val_27 +28 val_28 +28 val_28 +30 val_30 +30 val_30 +33 val_33 +33 val_33 +34 val_34 +34 val_34 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +37 val_37 +37 val_37 +37 val_37 +37 val_37 +4 val_4 +4 val_4 +41 val_41 +41 val_41 +42 val_42 +42 val_42 +42 val_42 +42 val_42 +43 val_43 +43 val_43 +44 val_44 +44 val_44 +47 val_47 +47 val_47 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +51 val_51 +51 val_51 +51 val_51 +51 val_51 +53 val_53 +53 val_53 +54 val_54 +54 val_54 +57 val_57 +57 val_57 +58 val_58 +58 val_58 +58 val_58 +58 val_58 +64 val_64 +64 val_64 +65 val_65 +65 val_65 +66 val_66 +66 val_66 +67 val_67 +67 val_67 +67 val_67 +67 val_67 +69 val_69 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +72 val_72 +72 val_72 +74 val_74 +74 val_74 +76 val_76 +76 val_76 +76 val_76 +76 val_76 +77 val_77 +77 val_77 +78 val_78 +78 val_78 +8 val_8 +8 val_8 +80 val_80 +80 val_80 +82 val_82 +82 val_82 +83 val_83 +83 val_83 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +84 val_84 +84 val_84 +85 val_85 +85 val_85 +86 val_86 +86 val_86 +87 val_87 +87 val_87 +9 val_9 +9 val_9 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +92 val_92 +95 val_95 +95 val_95 +95 val_95 +95 val_95 +96 val_96 +96 val_96 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +98 val_98 +98 val_98 diff --git ql/src/test/results/clientpositive/udaf_sum_list.q.out ql/src/test/results/clientpositive/llap/udaf_sum_list.q.out similarity index 86% rename from ql/src/test/results/clientpositive/udaf_sum_list.q.out rename to ql/src/test/results/clientpositive/llap/udaf_sum_list.q.out index 8afb053261..eb5c041af7 100644 --- ql/src/test/results/clientpositive/udaf_sum_list.q.out +++ ql/src/test/results/clientpositive/llap/udaf_sum_list.q.out @@ -7,9 +7,9 @@ POSTHOOK: Output: sum_list PREHOOK: query: select sum_list(array(key, key)) from src PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select sum_list(array(key, key)) from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 260182 diff --git ql/src/test/results/clientpositive/udf_printf.q.out ql/src/test/results/clientpositive/llap/udf_printf.q.out similarity index 92% rename from ql/src/test/results/clientpositive/udf_printf.q.out rename to ql/src/test/results/clientpositive/llap/udf_printf.q.out index 9809f9567d..a1cf1cc864 100644 --- ql/src/test/results/clientpositive/udf_printf.q.out +++ ql/src/test/results/clientpositive/llap/udf_printf.q.out @@ -23,12 +23,12 @@ PREHOOK: query: EXPLAIN SELECT printf("Hello World %d %s", 100, "days") FROM src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: EXPLAIN SELECT printf("Hello World %d %s", 100, "days") FROM src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-0 is a root stage @@ -40,39 +40,37 @@ STAGE PLANS: TableScan alias: src Row Limit Per Split: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 'Hello World 100 days' (type: string) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 52000 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT printf("Hello World %d %s", 100, "days") FROM src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT printf("Hello World %d %s", 100, "days") FROM src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### Hello World 100 days PREHOOK: query: SELECT printf("All Type Test: %b, %c, %d, %e, %+10.4f, %g, %h, %s, %a", false, 65, 15000, 12.3400, 27183.240051, 2300.41, 50, "corret", 256.125) FROM src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT printf("All Type Test: %b, %c, %d, %e, %+10.4f, %g, %h, %s, %a", false, 65, 15000, 12.3400, 27183.240051, 2300.41, 50, "corret", 256.125) FROM src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### All Type Test: false, A, 15000, 1.234000e+01, +27183.2401, 2300.41, 32, corret, 0x1.002p8 PREHOOK: query: SELECT printf("Color %s, String Null: %s, number1 %d, number2 %05d, Integer Null: %d, hex %#x, float %5.2f Double Null: %f\n", "red", NULL, 123456, 89, NULL, 255, 3.14159, NULL) FROM src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: SELECT printf("Color %s, String Null: %s, number1 %d, number2 %05d, Integer Null: %d, hex %#x, float %5.2f Double Null: %f\n", "red", NULL, 123456, 89, NULL, 255, 3.14159, NULL) FROM src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### Color red, String Null: null, number1 123456, number2 00089, Integer Null: null, hex 0xff, float 3.14 Double Null: null PREHOOK: query: create table timestamp_udf (t timestamp) @@ -99,11 +97,11 @@ POSTHOOK: Lineage: timestamp_udf.t EXPRESSION [] PREHOOK: query: select printf("timestamp: %s", t) from timestamp_udf PREHOOK: type: QUERY PREHOOK: Input: default@timestamp_udf -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select printf("timestamp: %s", t) from timestamp_udf POSTHOOK: type: QUERY POSTHOOK: Input: default@timestamp_udf -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### timestamp: 2011-05-06 07:08:09.1234567 PREHOOK: query: drop table timestamp_udf PREHOOK: type: DROPTABLE @@ -156,11 +154,11 @@ POSTHOOK: Lineage: dest1_n110.value SCRIPT [(binay_udf)binay_udf.FieldSchema(nam PREHOOK: query: select value, printf("format key: %s", key) from dest1_n110 PREHOOK: type: QUERY PREHOOK: Input: default@dest1_n110 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select value, printf("format key: %s", key) from dest1_n110 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1_n110 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 format key: 61 00 62 63 01 02 01 00 2 format key: 00 74 65 73 74 00 3 format key: 01 74 65 73 74 01 diff --git ql/src/test/results/clientpositive/union23.q.out ql/src/test/results/clientpositive/llap/union23.q.out similarity index 81% rename from ql/src/test/results/clientpositive/union23.q.out rename to ql/src/test/results/clientpositive/llap/union23.q.out index 742b282b62..96bdd82361 100644 --- ql/src/test/results/clientpositive/union23.q.out +++ ql/src/test/results/clientpositive/llap/union23.q.out @@ -8,7 +8,7 @@ from ( order by s.key2, s.value2 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select s.key2, s.value2 from ( @@ -19,62 +19,74 @@ from ( order by s.key2, s.value2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Union + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Transform Operator + command: cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ + File Output Operator + compressed: false Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -91,7 +103,7 @@ from ( order by s.key2, s.value2 PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select s.key2, s.value2 from ( select transform(key, value) using 'cat' as (key2, value2) @@ -101,7 +113,7 @@ from ( order by s.key2, s.value2 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 diff --git ql/src/test/results/clientpositive/union_script.q.out ql/src/test/results/clientpositive/llap/union_script.q.out similarity index 97% rename from ql/src/test/results/clientpositive/union_script.q.out rename to ql/src/test/results/clientpositive/llap/union_script.q.out index b8dfeb1319..6994053a0b 100644 --- ql/src/test/results/clientpositive/union_script.q.out +++ ql/src/test/results/clientpositive/llap/union_script.q.out @@ -2,12 +2,12 @@ PREHOOK: query: select * from ( select transform(key) using 'cat' as cola from src)s PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from ( select transform(key) using 'cat' as cola from src)s POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 0 @@ -514,14 +514,14 @@ PREHOOK: query: select * from ( select transform(key) using 'cat' as cola from src) s PREHOOK: type: QUERY PREHOOK: Input: default@src -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from ( select transform(key) using 'cat' as cola from src union all select transform(key) using 'cat' as cola from src) s POSTHOOK: type: QUERY POSTHOOK: Input: default@src -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 0 diff --git ql/src/test/results/clientpositive/vector_custom_udf_configure.q.out ql/src/test/results/clientpositive/llap/vector_custom_udf_configure.q.out similarity index 60% rename from ql/src/test/results/clientpositive/vector_custom_udf_configure.q.out rename to ql/src/test/results/clientpositive/llap/vector_custom_udf_configure.q.out index 738fa32d78..4b6be2c42f 100644 --- ql/src/test/results/clientpositive/vector_custom_udf_configure.q.out +++ ql/src/test/results/clientpositive/llap/vector_custom_udf_configure.q.out @@ -26,35 +26,38 @@ PREHOOK: query: explain select id, UDFHelloTest(name) from testorc1 PREHOOK: type: QUERY PREHOOK: Input: default@testorc1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: explain select id, UDFHelloTest(name) from testorc1 POSTHOOK: type: QUERY POSTHOOK: Input: default@testorc1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: testorc1 - Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: id (type: int), Hello... (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: testorc1 + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), Hello... (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -65,10 +68,10 @@ STAGE PLANS: PREHOOK: query: select id, UDFHelloTest(name) from testorc1 PREHOOK: type: QUERY PREHOOK: Input: default@testorc1 -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select id, UDFHelloTest(name) from testorc1 POSTHOOK: type: QUERY POSTHOOK: Input: default@testorc1 -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 1 Hello a1 2 Hello a2 diff --git ql/src/test/results/clientpositive/llap/vector_udf3.q.out ql/src/test/results/clientpositive/llap/vector_udf3.q.out new file mode 100644 index 0000000000..6bc8745526 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_udf3.q.out @@ -0,0 +1,106 @@ +PREHOOK: query: CREATE TEMPORARY FUNCTION rot13 as 'hive.it.custom.udfs.GenericUDFRot13' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: rot13 +POSTHOOK: query: CREATE TEMPORARY FUNCTION rot13 as 'hive.it.custom.udfs.GenericUDFRot13' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: rot13 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT rot13(cstring1) from alltypesorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT rot13(cstring1) from alltypesorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: hdfs://### HDFS PATH ### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 862450 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: Rot13(cstring1) (type: string) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [13] + selectExpressions: VectorStringRot13(col 6:string) -> 13:string + Statistics: Num rows: 12288 Data size: 2260992 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 12288 Data size: 2260992 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cstring1, rot13(cstring1) from alltypesorc order by cstring1 desc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT cstring1, rot13(cstring1) from alltypesorc order by cstring1 desc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: hdfs://### HDFS PATH ### +yy2GiGM ll2TvTZ +yxN0212hM17E8J8bJj8D7b lkA0212uZ17R8W8oWw8Q7o +ywA68u76Jv06axCv451avL4 ljN68h76Wi06nkPi451niY4 +yvNv1q liAi1d +yv3gnG4a33hD7bIm7oxE5rw li3taT4n33uQ7oVz7bkR5ej +yv1js li1wf +yujO07KWj lhwB07XJw +ytpx1RL8F2I lgck1EY8S2V +ytj7g5W lgw7t5J +ytgaJW1Gvrkv5wFUJU2y1S lgtnWJ1Tiexi5jSHWH2l1F +PREHOOK: query: SELECT cstring1, rot13(cstring1) from alltypesorc order by cstring1 desc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT cstring1, rot13(cstring1) from alltypesorc order by cstring1 desc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: hdfs://### HDFS PATH ### +yy2GiGM Unvectorized +yxN0212hM17E8J8bJj8D7b Unvectorized +ywA68u76Jv06axCv451avL4 Unvectorized +yvNv1q Unvectorized +yv3gnG4a33hD7bIm7oxE5rw Unvectorized +yv1js Unvectorized +yujO07KWj Unvectorized +ytpx1RL8F2I Unvectorized +ytj7g5W Unvectorized +ytgaJW1Gvrkv5wFUJU2y1S Unvectorized diff --git ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out deleted file mode 100644 index 6435e8b5a3..0000000000 --- ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out +++ /dev/null @@ -1,440 +0,0 @@ -PREHOOK: query: DROP TABLE `insert` -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE `insert` -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE `insert` (key INT, `as` STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@insert -POSTHOOK: query: CREATE TABLE `insert` (key INT, `as` STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@insert -PREHOOK: query: EXPLAIN INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert -POSTHOOK: query: EXPLAIN INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 100 - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 100 - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, as - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(as, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, as - Column Types: int, string - Table: default.insert - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert -POSTHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert -POSTHOOK: Lineage: insert.as SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: insert.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: SELECT SUM(HASH(hash)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (hash) FROM `insert` -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(hash)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (hash) FROM `insert` -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert -#### A masked pattern was here #### -10226524244 -PREHOOK: query: EXPLAIN INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert -POSTHOOK: query: EXPLAIN INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 100 - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 100 - Statistics: Num rows: 100 Data size: 17800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, as - Statistics: Num rows: 100 Data size: 9500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(as, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, as - Column Types: int, string - Table: default.insert - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert -POSTHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert -POSTHOOK: Lineage: insert.as SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: insert.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: SELECT SUM(HASH(sum)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (sum) FROM `insert` -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(sum)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (sum) FROM `insert` -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert -#### A masked pattern was here #### -20453048488 -PREHOOK: query: SELECT COUNT(*) FROM `insert` -PREHOOK: type: QUERY -PREHOOK: Input: default@insert -#### A masked pattern was here #### -POSTHOOK: query: SELECT COUNT(*) FROM `insert` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert -#### A masked pattern was here #### -200 -PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert -POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: key, as - Statistics: Num rows: 10 Data size: 950 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(as, 'hll') - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert - - Stage: Stage-2 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: key, as - Column Types: int, string - Table: default.insert - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct), _col1 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@insert -POSTHOOK: query: INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@insert -POSTHOOK: Lineage: insert.as SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: insert.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: SELECT SUM(HASH(add)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (add) FROM `insert` -) t -PREHOOK: type: QUERY -PREHOOK: Input: default@insert -#### A masked pattern was here #### -POSTHOOK: query: SELECT SUM(HASH(add)) FROM ( - SELECT TRANSFORM(*) USING 'tr \t _' AS (add) FROM `insert` -) t -POSTHOOK: type: QUERY -POSTHOOK: Input: default@insert -#### A masked pattern was here #### --826625916 -PREHOOK: query: DROP TABLE `insert` -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@insert -PREHOOK: Output: default@insert -POSTHOOK: query: DROP TABLE `insert` -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@insert -POSTHOOK: Output: default@insert diff --git ql/src/test/results/clientpositive/ppd_transform.q.out ql/src/test/results/clientpositive/ppd_transform.q.out deleted file mode 100644 index a38042c6fe..0000000000 --- ql/src/test/results/clientpositive/ppd_transform.q.out +++ /dev/null @@ -1,436 +0,0 @@ -PREHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -10 val_10 -11 val_11 -12 val_12 -12 val_12 -15 val_15 -15 val_15 -17 val_17 -18 val_18 -18 val_18 -19 val_19 -2 val_2 -20 val_20 -24 val_24 -24 val_24 -26 val_26 -26 val_26 -27 val_27 -28 val_28 -30 val_30 -33 val_33 -34 val_34 -35 val_35 -35 val_35 -35 val_35 -37 val_37 -37 val_37 -4 val_4 -41 val_41 -42 val_42 -42 val_42 -43 val_43 -44 val_44 -47 val_47 -5 val_5 -5 val_5 -5 val_5 -51 val_51 -51 val_51 -53 val_53 -54 val_54 -57 val_57 -58 val_58 -58 val_58 -64 val_64 -65 val_65 -66 val_66 -67 val_67 -67 val_67 -69 val_69 -70 val_70 -70 val_70 -70 val_70 -72 val_72 -72 val_72 -74 val_74 -76 val_76 -76 val_76 -77 val_77 -78 val_78 -8 val_8 -80 val_80 -82 val_82 -83 val_83 -83 val_83 -84 val_84 -84 val_84 -85 val_85 -86 val_86 -87 val_87 -9 val_9 -90 val_90 -90 val_90 -90 val_90 -92 val_92 -95 val_95 -95 val_95 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'cat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -10 val_10 -11 val_11 -12 val_12 -12 val_12 -15 val_15 -15 val_15 -17 val_17 -18 val_18 -18 val_18 -19 val_19 -2 val_2 -20 val_20 -24 val_24 -24 val_24 -26 val_26 -26 val_26 -27 val_27 -28 val_28 -30 val_30 -33 val_33 -34 val_34 -35 val_35 -35 val_35 -35 val_35 -37 val_37 -37 val_37 -4 val_4 -41 val_41 -42 val_42 -42 val_42 -43 val_43 -44 val_44 -47 val_47 -5 val_5 -5 val_5 -5 val_5 -51 val_51 -51 val_51 -53 val_53 -54 val_54 -57 val_57 -58 val_58 -58 val_58 -64 val_64 -65 val_65 -66 val_66 -67 val_67 -67 val_67 -69 val_69 -70 val_70 -70 val_70 -70 val_70 -72 val_72 -72 val_72 -74 val_74 -76 val_76 -76 val_76 -77 val_77 -78 val_78 -8 val_8 -80 val_80 -82 val_82 -83 val_83 -83 val_83 -84 val_84 -84 val_84 -85 val_85 -86 val_86 -87 val_87 -9 val_9 -90 val_90 -90 val_90 -90 val_90 -92 val_92 -95 val_95 -95 val_95 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: EXPLAIN -FROM ( - FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y -) reduced -#### A masked pattern was here #### -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -FROM ( - FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y -) reduced -#### A masked pattern was here #### -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-0 depends on stages: Stage-2 - Stage-1 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col0) IN ('a', 'b') (type: boolean) - Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Filter Operator - predicate: (_col0) IN ('c', 'd') (type: boolean) - Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Move Operator - files: - hdfs directory: false -#### A masked pattern was here #### - - Stage: Stage-1 - Move Operator - files: - hdfs directory: false -#### A masked pattern was here #### - diff --git ql/src/test/results/clientpositive/regexp_extract.q.out ql/src/test/results/clientpositive/regexp_extract.q.out deleted file mode 100644 index 95f7c22bc9..0000000000 --- ql/src/test/results/clientpositive/regexp_extract.q.out +++ /dev/null @@ -1,524 +0,0 @@ -PREHOOK: query: EXPLAIN EXTENDED -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: key (type: string), value (type: string), 3 (type: int), 7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string,string - field.delim 9 - serialization.format 9 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - isSamplingPred: false - predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: a - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [tmap:src] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), regexp_extract(VALUE._col1, 'val_(\d+\t\d+)', 1) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 44986 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 166 Data size: 44986 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 0 3 -0 0 3 -0 0 3 -10 10 3 -11 11 3 -12 12 3 -12 12 3 -15 15 3 -15 15 3 -17 17 3 -18 18 3 -18 18 3 -19 19 3 -2 2 3 -20 20 3 -24 24 3 -24 24 3 -26 26 3 -26 26 3 -27 27 3 -28 28 3 -30 30 3 -33 33 3 -34 34 3 -35 35 3 -35 35 3 -35 35 3 -37 37 3 -37 37 3 -4 4 3 -41 41 3 -42 42 3 -42 42 3 -43 43 3 -44 44 3 -47 47 3 -5 5 3 -5 5 3 -5 5 3 -51 51 3 -51 51 3 -53 53 3 -54 54 3 -57 57 3 -58 58 3 -58 58 3 -64 64 3 -65 65 3 -66 66 3 -67 67 3 -67 67 3 -69 69 3 -70 70 3 -70 70 3 -70 70 3 -72 72 3 -72 72 3 -74 74 3 -76 76 3 -76 76 3 -77 77 3 -78 78 3 -8 8 3 -80 80 3 -82 82 3 -83 83 3 -83 83 3 -84 84 3 -84 84 3 -85 85 3 -86 86 3 -87 87 3 -9 9 3 -90 90 3 -90 90 3 -90 90 3 -92 92 3 -95 95 3 -95 95 3 -96 96 3 -97 97 3 -97 97 3 -98 98 3 -98 98 3 -PREHOOK: query: EXPLAIN EXTENDED -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: key (type: string), value (type: string), 3 (type: int), 7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string,string - field.delim 9 - serialization.format 9 - serialization.last.column.takes.rest true - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - isSamplingPred: false - predicate: (_col0 < 100) (type: boolean) - Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: a - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 30876 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col0 (type: string), _col1 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [tmap:src] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), regexp_extract(VALUE._col1, 'val_(\d+\t\d+)') (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 44986 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 166 Data size: 44986 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) - USING 'cat' - CLUSTER BY key -) tmap -SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 0 3 -0 0 3 -0 0 3 -10 10 3 -11 11 3 -12 12 3 -12 12 3 -15 15 3 -15 15 3 -17 17 3 -18 18 3 -18 18 3 -19 19 3 -2 2 3 -20 20 3 -24 24 3 -24 24 3 -26 26 3 -26 26 3 -27 27 3 -28 28 3 -30 30 3 -33 33 3 -34 34 3 -35 35 3 -35 35 3 -35 35 3 -37 37 3 -37 37 3 -4 4 3 -41 41 3 -42 42 3 -42 42 3 -43 43 3 -44 44 3 -47 47 3 -5 5 3 -5 5 3 -5 5 3 -51 51 3 -51 51 3 -53 53 3 -54 54 3 -57 57 3 -58 58 3 -58 58 3 -64 64 3 -65 65 3 -66 66 3 -67 67 3 -67 67 3 -69 69 3 -70 70 3 -70 70 3 -70 70 3 -72 72 3 -72 72 3 -74 74 3 -76 76 3 -76 76 3 -77 77 3 -78 78 3 -8 8 3 -80 80 3 -82 82 3 -83 83 3 -83 83 3 -84 84 3 -84 84 3 -85 85 3 -86 86 3 -87 87 3 -9 9 3 -90 90 3 -90 90 3 -90 90 3 -92 92 3 -95 95 3 -95 95 3 -96 96 3 -97 97 3 -97 97 3 -98 98 3 -98 98 3 diff --git ql/src/test/results/clientpositive/script_pipe.q.out ql/src/test/results/clientpositive/script_pipe.q.out deleted file mode 100644 index f56107ebb1..0000000000 --- ql/src/test/results/clientpositive/script_pipe.q.out +++ /dev/null @@ -1,123 +0,0 @@ -PREHOOK: query: EXPLAIN SELECT TRANSFORM(*) USING 'true' AS a, b, c FROM (SELECT * FROM src LIMIT 1) tmp -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN SELECT TRANSFORM(*) USING 'true' AS a, b, c FROM (SELECT * FROM src LIMIT 1) tmp -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: true - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN SELECT TRANSFORM(key, value, key, value, key, value, key, value, key, value, key, value) USING 'head -n 1' as a,b,c,d FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN SELECT TRANSFORM(key, value, key, value, key, value, key, value, key, value, key, value) USING 'head -n 1' as a,b,c,d FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), key (type: string), value (type: string), key (type: string), value (type: string), key (type: string), value (type: string), key (type: string), value (type: string), key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 500 Data size: 534000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: head -n 1 - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 500 Data size: 534000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 534000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT TRANSFORM(*) USING 'true' AS a, b, c FROM (SELECT * FROM src LIMIT 1) tmp -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT TRANSFORM(*) USING 'true' AS a, b, c FROM (SELECT * FROM src LIMIT 1) tmp -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -PREHOOK: query: SELECT TRANSFORM(key, value, key, value, key, value, key, value, key, value, key, value) USING 'head -n 1' as a,b,c,d FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT TRANSFORM(key, value, key, value, key, value, key, value, key, value, key, value) USING 'head -n 1' as a,b,c,d FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -238 val_238 238 val_238 diff --git ql/src/test/results/clientpositive/transform1.q.out ql/src/test/results/clientpositive/transform1.q.out deleted file mode 100644 index d8d22443b8..0000000000 --- ql/src/test/results/clientpositive/transform1.q.out +++ /dev/null @@ -1,144 +0,0 @@ -PREHOOK: query: create table transform1_t1(a string, b string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@transform1_t1 -POSTHOOK: query: create table transform1_t1(a string, b string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@transform1_t1 -PREHOOK: query: EXPLAIN -SELECT transform(*) USING 'cat' AS (col array) FROM transform1_t1 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@transform1_t1 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT transform(*) USING 'cat' AS (col array) FROM transform1_t1 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@transform1_t1 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: transform1_t1 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: string), b (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT transform(*) USING 'cat' AS (col array) FROM transform1_t1 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@transform1_t1 -#### A masked pattern was here #### -POSTHOOK: query: SELECT transform(*) USING 'cat' AS (col array) FROM transform1_t1 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@transform1_t1 -#### A masked pattern was here #### -PREHOOK: query: create table transform1_t2(col array) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@transform1_t2 -POSTHOOK: query: create table transform1_t2(col array) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@transform1_t2 -PREHOOK: query: insert overwrite table transform1_t2 -select array(1,2,3) from src tablesample (1 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@transform1_t2 -POSTHOOK: query: insert overwrite table transform1_t2 -select array(1,2,3) from src tablesample (1 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@transform1_t2 -POSTHOOK: Lineage: transform1_t2.col EXPRESSION [] -PREHOOK: query: EXPLAIN -SELECT transform('0\0021\0022') USING 'cat' AS (col array) FROM transform1_t2 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@transform1_t2 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT transform('0\0021\0022') USING 'cat' AS (col array) FROM transform1_t2 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@transform1_t2 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: transform1_t2 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: '012' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT transform('0\0021\0022') USING 'cat' AS (col array) FROM transform1_t2 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@transform1_t2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT transform('0\0021\0022') USING 'cat' AS (col array) FROM transform1_t2 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@transform1_t2 -#### A masked pattern was here #### -[0,1,2] diff --git ql/src/test/results/clientpositive/transform_ppr1.q.out ql/src/test/results/clientpositive/transform_ppr1.q.out deleted file mode 100644 index 25468bcd9c..0000000000 --- ql/src/test/results/clientpositive/transform_ppr1.q.out +++ /dev/null @@ -1,516 +0,0 @@ -PREHOOK: query: EXPLAIN EXTENDED -FROM ( - FROM srcpart src - SELECT TRANSFORM(src.ds, src.key, src.value) - USING 'cat' AS (ds, tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED -FROM ( - FROM srcpart src - SELECT TRANSFORM(src.ds, src.key, src.value) - USING 'cat' AS (ds, tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 2000 Data size: 724000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: ds (type: string), key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2000 Data size: 724000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1,_col2 - columns.types string,string,string - field.delim 9 - serialization.format 9 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 2000 Data size: 724000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - isSamplingPred: false - predicate: ((_col1 < 100) and (_col0 = '2008-04-08')) (type: boolean) - Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col1 (type: string) - null sort order: a - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 333 Data size: 120546 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: '2008-04-08' (type: string), _col1 (type: string), _col2 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### - Partition - base file name: hr=12 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 12 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - hr 11 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### - Partition - base file name: hr=12 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - hr 12 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [tmap:src] - /srcpart/ds=2008-04-08/hr=12 [tmap:src] - /srcpart/ds=2008-04-09/hr=11 [tmap:src] - /srcpart/ds=2008-04-09/hr=12 [tmap:src] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 333 Data size: 59274 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 333 Data size: 59274 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: FROM ( - FROM srcpart src - SELECT TRANSFORM(src.ds, src.key, src.value) - USING 'cat' AS (ds, tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: FROM ( - FROM srcpart src - SELECT TRANSFORM(src.ds, src.key, src.value) - USING 'cat' AS (ds, tkey, tvalue) - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 AND tmap.ds = '2008-04-08' -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -10 val_10 -10 val_10 -11 val_11 -11 val_11 -12 val_12 -12 val_12 -12 val_12 -12 val_12 -15 val_15 -15 val_15 -15 val_15 -15 val_15 -17 val_17 -17 val_17 -18 val_18 -18 val_18 -18 val_18 -18 val_18 -19 val_19 -19 val_19 -2 val_2 -2 val_2 -20 val_20 -20 val_20 -24 val_24 -24 val_24 -24 val_24 -24 val_24 -26 val_26 -26 val_26 -26 val_26 -26 val_26 -27 val_27 -27 val_27 -28 val_28 -28 val_28 -30 val_30 -30 val_30 -33 val_33 -33 val_33 -34 val_34 -34 val_34 -35 val_35 -35 val_35 -35 val_35 -35 val_35 -35 val_35 -35 val_35 -37 val_37 -37 val_37 -37 val_37 -37 val_37 -4 val_4 -4 val_4 -41 val_41 -41 val_41 -42 val_42 -42 val_42 -42 val_42 -42 val_42 -43 val_43 -43 val_43 -44 val_44 -44 val_44 -47 val_47 -47 val_47 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -51 val_51 -51 val_51 -51 val_51 -51 val_51 -53 val_53 -53 val_53 -54 val_54 -54 val_54 -57 val_57 -57 val_57 -58 val_58 -58 val_58 -58 val_58 -58 val_58 -64 val_64 -64 val_64 -65 val_65 -65 val_65 -66 val_66 -66 val_66 -67 val_67 -67 val_67 -67 val_67 -67 val_67 -69 val_69 -69 val_69 -70 val_70 -70 val_70 -70 val_70 -70 val_70 -70 val_70 -70 val_70 -72 val_72 -72 val_72 -72 val_72 -72 val_72 -74 val_74 -74 val_74 -76 val_76 -76 val_76 -76 val_76 -76 val_76 -77 val_77 -77 val_77 -78 val_78 -78 val_78 -8 val_8 -8 val_8 -80 val_80 -80 val_80 -82 val_82 -82 val_82 -83 val_83 -83 val_83 -83 val_83 -83 val_83 -84 val_84 -84 val_84 -84 val_84 -84 val_84 -85 val_85 -85 val_85 -86 val_86 -86 val_86 -87 val_87 -87 val_87 -9 val_9 -9 val_9 -90 val_90 -90 val_90 -90 val_90 -90 val_90 -90 val_90 -90 val_90 -92 val_92 -92 val_92 -95 val_95 -95 val_95 -95 val_95 -95 val_95 -96 val_96 -96 val_96 -97 val_97 -97 val_97 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -98 val_98 -98 val_98 diff --git ql/src/test/results/clientpositive/transform_ppr2.q.out ql/src/test/results/clientpositive/transform_ppr2.q.out deleted file mode 100644 index 8aeb688513..0000000000 --- ql/src/test/results/clientpositive/transform_ppr2.q.out +++ /dev/null @@ -1,413 +0,0 @@ -PREHOOK: query: EXPLAIN EXTENDED -FROM ( - FROM srcpart src - SELECT TRANSFORM(src.ds, src.key, src.value) - USING 'cat' AS (ds, tkey, tvalue) - WHERE src.ds = '2008-04-08' - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN EXTENDED -FROM ( - FROM srcpart src - SELECT TRANSFORM(src.ds, src.key, src.value) - USING 'cat' AS (ds, tkey, tvalue) - WHERE src.ds = '2008-04-08' - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Select Operator - expressions: '2008-04-08' (type: string), key (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 272000 Basic stats: COMPLETE Column stats: COMPLETE - Transform Operator - command: cat - output info: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1,_col2 - columns.types string,string,string - field.delim 9 - serialization.format 9 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Statistics: Num rows: 1000 Data size: 272000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - isSamplingPred: false - predicate: (_col1 < 100) (type: boolean) - Statistics: Num rows: 333 Data size: 90576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col1 (type: string) - null sort order: a - numBuckets: -1 - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 333 Data size: 90576 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - auto parallelism: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### - Partition - base file name: hr=12 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 12 - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - bucketing_version 2 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [tmap:src] - /srcpart/ds=2008-04-08/hr=12 [tmap:src] - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 333 Data size: 59274 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - bucketingVersion: 2 - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 333 Data size: 59274 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - bucketing_version -1 - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: FROM ( - FROM srcpart src - SELECT TRANSFORM(src.ds, src.key, src.value) - USING 'cat' AS (ds, tkey, tvalue) - WHERE src.ds = '2008-04-08' - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: FROM ( - FROM srcpart src - SELECT TRANSFORM(src.ds, src.key, src.value) - USING 'cat' AS (ds, tkey, tvalue) - WHERE src.ds = '2008-04-08' - CLUSTER BY tkey -) tmap -SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100 -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -10 val_10 -10 val_10 -11 val_11 -11 val_11 -12 val_12 -12 val_12 -12 val_12 -12 val_12 -15 val_15 -15 val_15 -15 val_15 -15 val_15 -17 val_17 -17 val_17 -18 val_18 -18 val_18 -18 val_18 -18 val_18 -19 val_19 -19 val_19 -2 val_2 -2 val_2 -20 val_20 -20 val_20 -24 val_24 -24 val_24 -24 val_24 -24 val_24 -26 val_26 -26 val_26 -26 val_26 -26 val_26 -27 val_27 -27 val_27 -28 val_28 -28 val_28 -30 val_30 -30 val_30 -33 val_33 -33 val_33 -34 val_34 -34 val_34 -35 val_35 -35 val_35 -35 val_35 -35 val_35 -35 val_35 -35 val_35 -37 val_37 -37 val_37 -37 val_37 -37 val_37 -4 val_4 -4 val_4 -41 val_41 -41 val_41 -42 val_42 -42 val_42 -42 val_42 -42 val_42 -43 val_43 -43 val_43 -44 val_44 -44 val_44 -47 val_47 -47 val_47 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -5 val_5 -51 val_51 -51 val_51 -51 val_51 -51 val_51 -53 val_53 -53 val_53 -54 val_54 -54 val_54 -57 val_57 -57 val_57 -58 val_58 -58 val_58 -58 val_58 -58 val_58 -64 val_64 -64 val_64 -65 val_65 -65 val_65 -66 val_66 -66 val_66 -67 val_67 -67 val_67 -67 val_67 -67 val_67 -69 val_69 -69 val_69 -70 val_70 -70 val_70 -70 val_70 -70 val_70 -70 val_70 -70 val_70 -72 val_72 -72 val_72 -72 val_72 -72 val_72 -74 val_74 -74 val_74 -76 val_76 -76 val_76 -76 val_76 -76 val_76 -77 val_77 -77 val_77 -78 val_78 -78 val_78 -8 val_8 -8 val_8 -80 val_80 -80 val_80 -82 val_82 -82 val_82 -83 val_83 -83 val_83 -83 val_83 -83 val_83 -84 val_84 -84 val_84 -84 val_84 -84 val_84 -85 val_85 -85 val_85 -86 val_86 -86 val_86 -87 val_87 -87 val_87 -9 val_9 -9 val_9 -90 val_90 -90 val_90 -90 val_90 -90 val_90 -90 val_90 -90 val_90 -92 val_92 -92 val_92 -95 val_95 -95 val_95 -95 val_95 -95 val_95 -96 val_96 -96 val_96 -97 val_97 -97 val_97 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -98 val_98 -98 val_98 diff --git ql/src/test/results/clientpositive/vector_udf3.q.out ql/src/test/results/clientpositive/vector_udf3.q.out deleted file mode 100644 index 16ce5f3142..0000000000 --- ql/src/test/results/clientpositive/vector_udf3.q.out +++ /dev/null @@ -1,103 +0,0 @@ -PREHOOK: query: CREATE TEMPORARY FUNCTION rot13 as 'hive.it.custom.udfs.GenericUDFRot13' -PREHOOK: type: CREATEFUNCTION -PREHOOK: Output: rot13 -POSTHOOK: query: CREATE TEMPORARY FUNCTION rot13 as 'hive.it.custom.udfs.GenericUDFRot13' -POSTHOOK: type: CREATEFUNCTION -POSTHOOK: Output: rot13 -PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT rot13(cstring1) from alltypesorc -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT rot13(cstring1) from alltypesorc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypesorc - Statistics: Num rows: 12288 Data size: 862450 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Select Operator - expressions: Rot13(cstring1) (type: string) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [13] - selectExpressions: VectorStringRot13(col 6:string) -> 13:string - Statistics: Num rows: 12288 Data size: 2260992 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - Statistics: Num rows: 12288 Data size: 2260992 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cstring1, rot13(cstring1) from alltypesorc order by cstring1 desc limit 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: SELECT cstring1, rot13(cstring1) from alltypesorc order by cstring1 desc limit 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -yy2GiGM ll2TvTZ -yxN0212hM17E8J8bJj8D7b lkA0212uZ17R8W8oWw8Q7o -ywA68u76Jv06axCv451avL4 ljN68h76Wi06nkPi451niY4 -yvNv1q liAi1d -yv3gnG4a33hD7bIm7oxE5rw li3taT4n33uQ7oVz7bkR5ej -yv1js li1wf -yujO07KWj lhwB07XJw -ytpx1RL8F2I lgck1EY8S2V -ytj7g5W lgw7t5J -ytgaJW1Gvrkv5wFUJU2y1S lgtnWJ1Tiexi5jSHWH2l1F -PREHOOK: query: SELECT cstring1, rot13(cstring1) from alltypesorc order by cstring1 desc limit 10 -PREHOOK: type: QUERY -PREHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -POSTHOOK: query: SELECT cstring1, rot13(cstring1) from alltypesorc order by cstring1 desc limit 10 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@alltypesorc -#### A masked pattern was here #### -yy2GiGM Unvectorized -yxN0212hM17E8J8bJj8D7b Unvectorized -ywA68u76Jv06axCv451avL4 Unvectorized -yvNv1q Unvectorized -yv3gnG4a33hD7bIm7oxE5rw Unvectorized -yv1js Unvectorized -yujO07KWj Unvectorized -ytpx1RL8F2I Unvectorized -ytj7g5W Unvectorized -ytgaJW1Gvrkv5wFUJU2y1S Unvectorized