From 75eebc7c61eeaac253b0b248d3292624275e9812 Mon Sep 17 00:00:00 2001 From: Na Yang Date: Wed, 5 Nov 2014 17:59:39 -0800 Subject: [PATCH] HIVE-7729 --- .../test/resources/testconfiguration.properties | 22 + .../test/results/clientpositive/spark/stats0.q.out | 2642 ++++++++++++++++++++ .../results/clientpositive/spark/stats10.q.out | 527 ++++ .../results/clientpositive/spark/stats12.q.out | 395 +++ .../results/clientpositive/spark/stats13.q.out | 390 +++ .../results/clientpositive/spark/stats14.q.out | 318 +++ .../results/clientpositive/spark/stats15.q.out | 318 +++ .../results/clientpositive/spark/stats16.q.out | 105 + .../results/clientpositive/spark/stats18.q.out | 128 + .../test/results/clientpositive/spark/stats2.q.out | 205 ++ .../results/clientpositive/spark/stats20.q.out | 108 + .../test/results/clientpositive/spark/stats3.q.out | 262 ++ .../test/results/clientpositive/spark/stats5.q.out | 76 + .../test/results/clientpositive/spark/stats6.q.out | 264 ++ .../test/results/clientpositive/spark/stats7.q.out | 194 ++ .../test/results/clientpositive/spark/stats8.q.out | 627 +++++ .../test/results/clientpositive/spark/stats9.q.out | 84 + .../clientpositive/spark/stats_counter.q.out | 104 + .../clientpositive/spark/stats_noscan_1.q.out | 537 ++++ .../clientpositive/spark/stats_noscan_2.q.out | 315 +++ .../clientpositive/spark/stats_only_null.q.out | 430 ++++ .../clientpositive/spark/stats_partscan_1_23.q.out | 222 ++ .../results/clientpositive/spark/statsfs.q.out | 529 ++++ 23 files changed, 8802 insertions(+) create mode 100644 ql/src/test/results/clientpositive/spark/stats0.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats10.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats12.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats13.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats14.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats15.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats16.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats18.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats2.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats20.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats3.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats5.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats6.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats7.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats8.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats9.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats_counter.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats_only_null.q.out create mode 100644 ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out create mode 100644 ql/src/test/results/clientpositive/spark/statsfs.q.out diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index c777e45..52f3f11 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -789,6 +789,28 @@ spark.query.files=add_part_multiple.q, \ sort_merge_join_desc_7.q, \ sort_merge_join_desc_8.q, \ spark_test.q, \ + stats_counter.q, \ + stats_noscan_1.q, \ + stats_noscan_2.q, \ + stats_only_null.q, \ + stats_partscan_1_23.q, \ + stats0.q, \ + stats10.q, \ + stats12.q, \ + stats13.q, \ + stats14.q, \ + stats15.q, \ + stats16.q, \ + stats18.q, \ + stats2.q, \ + stats20.q, \ + stats3.q, \ + stats5.q, \ + stats6.q, \ + stats7.q, \ + stats8.q, \ + stats9.q, \ + statsfs.q, \ subquery_exists.q, \ subquery_in.q, \ subquery_multiinsert.q, \ diff --git ql/src/test/results/clientpositive/spark/stats0.q.out ql/src/test/results/clientpositive/spark/stats0.q.out new file mode 100644 index 0000000..eda5706 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats0.q.out @@ -0,0 +1,2642 @@ +PREHOOK: query: CREATE TABLE stats_non_partitioned (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_non_partitioned +POSTHOOK: query: CREATE TABLE stats_non_partitioned (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_non_partitioned +PREHOOK: query: explain extended +insert overwrite table stats_non_partitioned +select * from src +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +insert overwrite table stats_non_partitioned +select * from src +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + stats_non_partitioned + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.stats_non_partitioned + serialization.ddl struct stats_non_partitioned { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_non_partitioned + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.stats_non_partitioned + serialization.ddl struct stats_non_partitioned { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_non_partitioned + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: insert overwrite table stats_non_partitioned +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_non_partitioned +POSTHOOK: query: insert overwrite table stats_non_partitioned +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_non_partitioned +POSTHOOK: Lineage: stats_non_partitioned.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_non_partitioned.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc extended stats_non_partitioned +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_non_partitioned +POSTHOOK: query: desc extended stats_non_partitioned +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_non_partitioned +key string +value string + +#### A masked pattern was here #### +PREHOOK: query: select * from stats_non_partitioned +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_non_partitioned +#### A masked pattern was here #### +POSTHOOK: query: select * from stats_non_partitioned +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_non_partitioned +#### A masked pattern was here #### +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +409 val_409 +255 val_255 +278 val_278 +98 val_98 +484 val_484 +265 val_265 +193 val_193 +401 val_401 +150 val_150 +273 val_273 +224 val_224 +369 val_369 +66 val_66 +128 val_128 +213 val_213 +146 val_146 +406 val_406 +429 val_429 +374 val_374 +152 val_152 +469 val_469 +145 val_145 +495 val_495 +37 val_37 +327 val_327 +281 val_281 +277 val_277 +209 val_209 +15 val_15 +82 val_82 +403 val_403 +166 val_166 +417 val_417 +430 val_430 +252 val_252 +292 val_292 +219 val_219 +287 val_287 +153 val_153 +193 val_193 +338 val_338 +446 val_446 +459 val_459 +394 val_394 +237 val_237 +482 val_482 +174 val_174 +413 val_413 +494 val_494 +207 val_207 +199 val_199 +466 val_466 +208 val_208 +174 val_174 +399 val_399 +396 val_396 +247 val_247 +417 val_417 +489 val_489 +162 val_162 +377 val_377 +397 val_397 +309 val_309 +365 val_365 +266 val_266 +439 val_439 +342 val_342 +367 val_367 +325 val_325 +167 val_167 +195 val_195 +475 val_475 +17 val_17 +113 val_113 +155 val_155 +203 val_203 +339 val_339 +0 val_0 +455 val_455 +128 val_128 +311 val_311 +316 val_316 +57 val_57 +302 val_302 +205 val_205 +149 val_149 +438 val_438 +345 val_345 +129 val_129 +170 val_170 +20 val_20 +489 val_489 +157 val_157 +378 val_378 +221 val_221 +92 val_92 +111 val_111 +47 val_47 +72 val_72 +4 val_4 +280 val_280 +35 val_35 +427 val_427 +277 val_277 +208 val_208 +356 val_356 +399 val_399 +169 val_169 +382 val_382 +498 val_498 +125 val_125 +386 val_386 +437 val_437 +469 val_469 +192 val_192 +286 val_286 +187 val_187 +176 val_176 +54 val_54 +459 val_459 +51 val_51 +138 val_138 +103 val_103 +239 val_239 +213 val_213 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +289 val_289 +221 val_221 +65 val_65 +318 val_318 +332 val_332 +311 val_311 +275 val_275 +137 val_137 +241 val_241 +83 val_83 +333 val_333 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +181 val_181 +67 val_67 +260 val_260 +404 val_404 +384 val_384 +489 val_489 +353 val_353 +373 val_373 +272 val_272 +138 val_138 +217 val_217 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +411 val_411 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +463 val_463 +431 val_431 +179 val_179 +172 val_172 +42 val_42 +129 val_129 +158 val_158 +119 val_119 +496 val_496 +0 val_0 +322 val_322 +197 val_197 +468 val_468 +393 val_393 +454 val_454 +100 val_100 +298 val_298 +199 val_199 +191 val_191 +418 val_418 +96 val_96 +26 val_26 +165 val_165 +327 val_327 +230 val_230 +205 val_205 +120 val_120 +131 val_131 +51 val_51 +404 val_404 +43 val_43 +436 val_436 +156 val_156 +469 val_469 +468 val_468 +308 val_308 +95 val_95 +196 val_196 +288 val_288 +481 val_481 +457 val_457 +98 val_98 +282 val_282 +197 val_197 +187 val_187 +318 val_318 +318 val_318 +409 val_409 +470 val_470 +137 val_137 +369 val_369 +316 val_316 +169 val_169 +413 val_413 +85 val_85 +77 val_77 +0 val_0 +490 val_490 +87 val_87 +364 val_364 +179 val_179 +118 val_118 +134 val_134 +395 val_395 +282 val_282 +138 val_138 +238 val_238 +419 val_419 +15 val_15 +118 val_118 +72 val_72 +90 val_90 +307 val_307 +19 val_19 +435 val_435 +10 val_10 +277 val_277 +273 val_273 +306 val_306 +224 val_224 +309 val_309 +389 val_389 +327 val_327 +242 val_242 +369 val_369 +392 val_392 +272 val_272 +331 val_331 +401 val_401 +242 val_242 +452 val_452 +177 val_177 +226 val_226 +5 val_5 +497 val_497 +402 val_402 +396 val_396 +317 val_317 +395 val_395 +58 val_58 +35 val_35 +336 val_336 +95 val_95 +11 val_11 +168 val_168 +34 val_34 +229 val_229 +233 val_233 +143 val_143 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +195 val_195 +42 val_42 +321 val_321 +430 val_430 +119 val_119 +489 val_489 +458 val_458 +78 val_78 +76 val_76 +41 val_41 +223 val_223 +492 val_492 +149 val_149 +449 val_449 +218 val_218 +228 val_228 +138 val_138 +453 val_453 +30 val_30 +209 val_209 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +69 val_69 +230 val_230 +33 val_33 +368 val_368 +103 val_103 +296 val_296 +113 val_113 +216 val_216 +367 val_367 +344 val_344 +167 val_167 +274 val_274 +219 val_219 +239 val_239 +485 val_485 +116 val_116 +223 val_223 +256 val_256 +263 val_263 +70 val_70 +487 val_487 +480 val_480 +401 val_401 +288 val_288 +191 val_191 +5 val_5 +244 val_244 +438 val_438 +128 val_128 +467 val_467 +432 val_432 +202 val_202 +316 val_316 +229 val_229 +469 val_469 +463 val_463 +280 val_280 +2 val_2 +35 val_35 +283 val_283 +331 val_331 +235 val_235 +80 val_80 +44 val_44 +193 val_193 +321 val_321 +335 val_335 +104 val_104 +466 val_466 +366 val_366 +175 val_175 +403 val_403 +483 val_483 +53 val_53 +105 val_105 +257 val_257 +406 val_406 +409 val_409 +190 val_190 +406 val_406 +401 val_401 +114 val_114 +258 val_258 +90 val_90 +203 val_203 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +201 val_201 +217 val_217 +164 val_164 +431 val_431 +454 val_454 +478 val_478 +298 val_298 +125 val_125 +431 val_431 +164 val_164 +424 val_424 +187 val_187 +382 val_382 +5 val_5 +70 val_70 +397 val_397 +480 val_480 +291 val_291 +24 val_24 +351 val_351 +255 val_255 +104 val_104 +70 val_70 +163 val_163 +438 val_438 +119 val_119 +414 val_414 +200 val_200 +491 val_491 +237 val_237 +439 val_439 +360 val_360 +248 val_248 +479 val_479 +305 val_305 +417 val_417 +199 val_199 +444 val_444 +120 val_120 +429 val_429 +169 val_169 +443 val_443 +323 val_323 +325 val_325 +277 val_277 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +317 val_317 +333 val_333 +493 val_493 +460 val_460 +207 val_207 +249 val_249 +265 val_265 +480 val_480 +83 val_83 +136 val_136 +353 val_353 +172 val_172 +214 val_214 +462 val_462 +233 val_233 +406 val_406 +133 val_133 +175 val_175 +189 val_189 +454 val_454 +375 val_375 +401 val_401 +421 val_421 +407 val_407 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +67 val_67 +384 val_384 +379 val_379 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +9 val_9 +341 val_341 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +285 val_285 +348 val_348 +167 val_167 +18 val_18 +273 val_273 +183 val_183 +281 val_281 +344 val_344 +97 val_97 +469 val_469 +315 val_315 +84 val_84 +28 val_28 +37 val_37 +448 val_448 +152 val_152 +348 val_348 +307 val_307 +194 val_194 +414 val_414 +477 val_477 +222 val_222 +126 val_126 +90 val_90 +169 val_169 +403 val_403 +400 val_400 +200 val_200 +97 val_97 +PREHOOK: query: CREATE TABLE stats_partitioned(key string, value string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_partitioned +POSTHOOK: query: CREATE TABLE stats_partitioned(key string, value string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_partitioned +PREHOOK: query: explain +insert overwrite table stats_partitioned partition (ds='1') +select * from src +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table stats_partitioned partition (ds='1') +select * from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_partitioned + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_partitioned + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_partitioned@ds=1 +POSTHOOK: query: insert overwrite table stats_partitioned partition (ds='1') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_partitioned@ds=1 +POSTHOOK: Lineage: stats_partitioned PARTITION(ds=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_partitioned PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show partitions stats_partitioned +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@stats_partitioned +POSTHOOK: query: show partitions stats_partitioned +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@stats_partitioned +ds=1 +PREHOOK: query: select * from stats_partitioned where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_partitioned +PREHOOK: Input: default@stats_partitioned@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: select * from stats_partitioned where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_partitioned +POSTHOOK: Input: default@stats_partitioned@ds=1 +#### A masked pattern was here #### +238 val_238 1 +86 val_86 1 +311 val_311 1 +27 val_27 1 +165 val_165 1 +409 val_409 1 +255 val_255 1 +278 val_278 1 +98 val_98 1 +484 val_484 1 +265 val_265 1 +193 val_193 1 +401 val_401 1 +150 val_150 1 +273 val_273 1 +224 val_224 1 +369 val_369 1 +66 val_66 1 +128 val_128 1 +213 val_213 1 +146 val_146 1 +406 val_406 1 +429 val_429 1 +374 val_374 1 +152 val_152 1 +469 val_469 1 +145 val_145 1 +495 val_495 1 +37 val_37 1 +327 val_327 1 +281 val_281 1 +277 val_277 1 +209 val_209 1 +15 val_15 1 +82 val_82 1 +403 val_403 1 +166 val_166 1 +417 val_417 1 +430 val_430 1 +252 val_252 1 +292 val_292 1 +219 val_219 1 +287 val_287 1 +153 val_153 1 +193 val_193 1 +338 val_338 1 +446 val_446 1 +459 val_459 1 +394 val_394 1 +237 val_237 1 +482 val_482 1 +174 val_174 1 +413 val_413 1 +494 val_494 1 +207 val_207 1 +199 val_199 1 +466 val_466 1 +208 val_208 1 +174 val_174 1 +399 val_399 1 +396 val_396 1 +247 val_247 1 +417 val_417 1 +489 val_489 1 +162 val_162 1 +377 val_377 1 +397 val_397 1 +309 val_309 1 +365 val_365 1 +266 val_266 1 +439 val_439 1 +342 val_342 1 +367 val_367 1 +325 val_325 1 +167 val_167 1 +195 val_195 1 +475 val_475 1 +17 val_17 1 +113 val_113 1 +155 val_155 1 +203 val_203 1 +339 val_339 1 +0 val_0 1 +455 val_455 1 +128 val_128 1 +311 val_311 1 +316 val_316 1 +57 val_57 1 +302 val_302 1 +205 val_205 1 +149 val_149 1 +438 val_438 1 +345 val_345 1 +129 val_129 1 +170 val_170 1 +20 val_20 1 +489 val_489 1 +157 val_157 1 +378 val_378 1 +221 val_221 1 +92 val_92 1 +111 val_111 1 +47 val_47 1 +72 val_72 1 +4 val_4 1 +280 val_280 1 +35 val_35 1 +427 val_427 1 +277 val_277 1 +208 val_208 1 +356 val_356 1 +399 val_399 1 +169 val_169 1 +382 val_382 1 +498 val_498 1 +125 val_125 1 +386 val_386 1 +437 val_437 1 +469 val_469 1 +192 val_192 1 +286 val_286 1 +187 val_187 1 +176 val_176 1 +54 val_54 1 +459 val_459 1 +51 val_51 1 +138 val_138 1 +103 val_103 1 +239 val_239 1 +213 val_213 1 +216 val_216 1 +430 val_430 1 +278 val_278 1 +176 val_176 1 +289 val_289 1 +221 val_221 1 +65 val_65 1 +318 val_318 1 +332 val_332 1 +311 val_311 1 +275 val_275 1 +137 val_137 1 +241 val_241 1 +83 val_83 1 +333 val_333 1 +180 val_180 1 +284 val_284 1 +12 val_12 1 +230 val_230 1 +181 val_181 1 +67 val_67 1 +260 val_260 1 +404 val_404 1 +384 val_384 1 +489 val_489 1 +353 val_353 1 +373 val_373 1 +272 val_272 1 +138 val_138 1 +217 val_217 1 +84 val_84 1 +348 val_348 1 +466 val_466 1 +58 val_58 1 +8 val_8 1 +411 val_411 1 +230 val_230 1 +208 val_208 1 +348 val_348 1 +24 val_24 1 +463 val_463 1 +431 val_431 1 +179 val_179 1 +172 val_172 1 +42 val_42 1 +129 val_129 1 +158 val_158 1 +119 val_119 1 +496 val_496 1 +0 val_0 1 +322 val_322 1 +197 val_197 1 +468 val_468 1 +393 val_393 1 +454 val_454 1 +100 val_100 1 +298 val_298 1 +199 val_199 1 +191 val_191 1 +418 val_418 1 +96 val_96 1 +26 val_26 1 +165 val_165 1 +327 val_327 1 +230 val_230 1 +205 val_205 1 +120 val_120 1 +131 val_131 1 +51 val_51 1 +404 val_404 1 +43 val_43 1 +436 val_436 1 +156 val_156 1 +469 val_469 1 +468 val_468 1 +308 val_308 1 +95 val_95 1 +196 val_196 1 +288 val_288 1 +481 val_481 1 +457 val_457 1 +98 val_98 1 +282 val_282 1 +197 val_197 1 +187 val_187 1 +318 val_318 1 +318 val_318 1 +409 val_409 1 +470 val_470 1 +137 val_137 1 +369 val_369 1 +316 val_316 1 +169 val_169 1 +413 val_413 1 +85 val_85 1 +77 val_77 1 +0 val_0 1 +490 val_490 1 +87 val_87 1 +364 val_364 1 +179 val_179 1 +118 val_118 1 +134 val_134 1 +395 val_395 1 +282 val_282 1 +138 val_138 1 +238 val_238 1 +419 val_419 1 +15 val_15 1 +118 val_118 1 +72 val_72 1 +90 val_90 1 +307 val_307 1 +19 val_19 1 +435 val_435 1 +10 val_10 1 +277 val_277 1 +273 val_273 1 +306 val_306 1 +224 val_224 1 +309 val_309 1 +389 val_389 1 +327 val_327 1 +242 val_242 1 +369 val_369 1 +392 val_392 1 +272 val_272 1 +331 val_331 1 +401 val_401 1 +242 val_242 1 +452 val_452 1 +177 val_177 1 +226 val_226 1 +5 val_5 1 +497 val_497 1 +402 val_402 1 +396 val_396 1 +317 val_317 1 +395 val_395 1 +58 val_58 1 +35 val_35 1 +336 val_336 1 +95 val_95 1 +11 val_11 1 +168 val_168 1 +34 val_34 1 +229 val_229 1 +233 val_233 1 +143 val_143 1 +472 val_472 1 +322 val_322 1 +498 val_498 1 +160 val_160 1 +195 val_195 1 +42 val_42 1 +321 val_321 1 +430 val_430 1 +119 val_119 1 +489 val_489 1 +458 val_458 1 +78 val_78 1 +76 val_76 1 +41 val_41 1 +223 val_223 1 +492 val_492 1 +149 val_149 1 +449 val_449 1 +218 val_218 1 +228 val_228 1 +138 val_138 1 +453 val_453 1 +30 val_30 1 +209 val_209 1 +64 val_64 1 +468 val_468 1 +76 val_76 1 +74 val_74 1 +342 val_342 1 +69 val_69 1 +230 val_230 1 +33 val_33 1 +368 val_368 1 +103 val_103 1 +296 val_296 1 +113 val_113 1 +216 val_216 1 +367 val_367 1 +344 val_344 1 +167 val_167 1 +274 val_274 1 +219 val_219 1 +239 val_239 1 +485 val_485 1 +116 val_116 1 +223 val_223 1 +256 val_256 1 +263 val_263 1 +70 val_70 1 +487 val_487 1 +480 val_480 1 +401 val_401 1 +288 val_288 1 +191 val_191 1 +5 val_5 1 +244 val_244 1 +438 val_438 1 +128 val_128 1 +467 val_467 1 +432 val_432 1 +202 val_202 1 +316 val_316 1 +229 val_229 1 +469 val_469 1 +463 val_463 1 +280 val_280 1 +2 val_2 1 +35 val_35 1 +283 val_283 1 +331 val_331 1 +235 val_235 1 +80 val_80 1 +44 val_44 1 +193 val_193 1 +321 val_321 1 +335 val_335 1 +104 val_104 1 +466 val_466 1 +366 val_366 1 +175 val_175 1 +403 val_403 1 +483 val_483 1 +53 val_53 1 +105 val_105 1 +257 val_257 1 +406 val_406 1 +409 val_409 1 +190 val_190 1 +406 val_406 1 +401 val_401 1 +114 val_114 1 +258 val_258 1 +90 val_90 1 +203 val_203 1 +262 val_262 1 +348 val_348 1 +424 val_424 1 +12 val_12 1 +396 val_396 1 +201 val_201 1 +217 val_217 1 +164 val_164 1 +431 val_431 1 +454 val_454 1 +478 val_478 1 +298 val_298 1 +125 val_125 1 +431 val_431 1 +164 val_164 1 +424 val_424 1 +187 val_187 1 +382 val_382 1 +5 val_5 1 +70 val_70 1 +397 val_397 1 +480 val_480 1 +291 val_291 1 +24 val_24 1 +351 val_351 1 +255 val_255 1 +104 val_104 1 +70 val_70 1 +163 val_163 1 +438 val_438 1 +119 val_119 1 +414 val_414 1 +200 val_200 1 +491 val_491 1 +237 val_237 1 +439 val_439 1 +360 val_360 1 +248 val_248 1 +479 val_479 1 +305 val_305 1 +417 val_417 1 +199 val_199 1 +444 val_444 1 +120 val_120 1 +429 val_429 1 +169 val_169 1 +443 val_443 1 +323 val_323 1 +325 val_325 1 +277 val_277 1 +230 val_230 1 +478 val_478 1 +178 val_178 1 +468 val_468 1 +310 val_310 1 +317 val_317 1 +333 val_333 1 +493 val_493 1 +460 val_460 1 +207 val_207 1 +249 val_249 1 +265 val_265 1 +480 val_480 1 +83 val_83 1 +136 val_136 1 +353 val_353 1 +172 val_172 1 +214 val_214 1 +462 val_462 1 +233 val_233 1 +406 val_406 1 +133 val_133 1 +175 val_175 1 +189 val_189 1 +454 val_454 1 +375 val_375 1 +401 val_401 1 +421 val_421 1 +407 val_407 1 +384 val_384 1 +256 val_256 1 +26 val_26 1 +134 val_134 1 +67 val_67 1 +384 val_384 1 +379 val_379 1 +18 val_18 1 +462 val_462 1 +492 val_492 1 +100 val_100 1 +298 val_298 1 +9 val_9 1 +341 val_341 1 +498 val_498 1 +146 val_146 1 +458 val_458 1 +362 val_362 1 +186 val_186 1 +285 val_285 1 +348 val_348 1 +167 val_167 1 +18 val_18 1 +273 val_273 1 +183 val_183 1 +281 val_281 1 +344 val_344 1 +97 val_97 1 +469 val_469 1 +315 val_315 1 +84 val_84 1 +28 val_28 1 +37 val_37 1 +448 val_448 1 +152 val_152 1 +348 val_348 1 +307 val_307 1 +194 val_194 1 +414 val_414 1 +477 val_477 1 +222 val_222 1 +126 val_126 1 +90 val_90 1 +169 val_169 1 +403 val_403 1 +400 val_400 1 +200 val_200 1 +97 val_97 1 +PREHOOK: query: describe extended stats_partitioned partition (ds='1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_partitioned +POSTHOOK: query: describe extended stats_partitioned partition (ds='1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_partitioned +key string +value string +ds string + +# Partition Information +# col_name data_type comment + +ds string + +#### A masked pattern was here #### +PREHOOK: query: describe extended stats_partitioned +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_partitioned +POSTHOOK: query: describe extended stats_partitioned +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_partitioned +key string +value string +ds string + +# Partition Information +# col_name data_type comment + +ds string + +#### A masked pattern was here #### +PREHOOK: query: drop table stats_non_partitioned +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_non_partitioned +PREHOOK: Output: default@stats_non_partitioned +POSTHOOK: query: drop table stats_non_partitioned +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_non_partitioned +POSTHOOK: Output: default@stats_non_partitioned +PREHOOK: query: drop table stats_partitioned +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_partitioned +PREHOOK: Output: default@stats_partitioned +POSTHOOK: query: drop table stats_partitioned +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_partitioned +POSTHOOK: Output: default@stats_partitioned +PREHOOK: query: CREATE TABLE stats_non_partitioned (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_non_partitioned +POSTHOOK: query: CREATE TABLE stats_non_partitioned (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_non_partitioned +PREHOOK: query: explain extended +insert overwrite table stats_non_partitioned +select * from src +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +insert overwrite table stats_non_partitioned +select * from src +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + stats_non_partitioned + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.stats_non_partitioned + serialization.ddl struct stats_non_partitioned { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_non_partitioned + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.stats_non_partitioned + serialization.ddl struct stats_non_partitioned { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_non_partitioned + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: insert overwrite table stats_non_partitioned +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_non_partitioned +POSTHOOK: query: insert overwrite table stats_non_partitioned +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_non_partitioned +POSTHOOK: Lineage: stats_non_partitioned.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_non_partitioned.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc extended stats_non_partitioned +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_non_partitioned +POSTHOOK: query: desc extended stats_non_partitioned +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_non_partitioned +key string +value string + +#### A masked pattern was here #### +PREHOOK: query: select * from stats_non_partitioned +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_non_partitioned +#### A masked pattern was here #### +POSTHOOK: query: select * from stats_non_partitioned +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_non_partitioned +#### A masked pattern was here #### +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +409 val_409 +255 val_255 +278 val_278 +98 val_98 +484 val_484 +265 val_265 +193 val_193 +401 val_401 +150 val_150 +273 val_273 +224 val_224 +369 val_369 +66 val_66 +128 val_128 +213 val_213 +146 val_146 +406 val_406 +429 val_429 +374 val_374 +152 val_152 +469 val_469 +145 val_145 +495 val_495 +37 val_37 +327 val_327 +281 val_281 +277 val_277 +209 val_209 +15 val_15 +82 val_82 +403 val_403 +166 val_166 +417 val_417 +430 val_430 +252 val_252 +292 val_292 +219 val_219 +287 val_287 +153 val_153 +193 val_193 +338 val_338 +446 val_446 +459 val_459 +394 val_394 +237 val_237 +482 val_482 +174 val_174 +413 val_413 +494 val_494 +207 val_207 +199 val_199 +466 val_466 +208 val_208 +174 val_174 +399 val_399 +396 val_396 +247 val_247 +417 val_417 +489 val_489 +162 val_162 +377 val_377 +397 val_397 +309 val_309 +365 val_365 +266 val_266 +439 val_439 +342 val_342 +367 val_367 +325 val_325 +167 val_167 +195 val_195 +475 val_475 +17 val_17 +113 val_113 +155 val_155 +203 val_203 +339 val_339 +0 val_0 +455 val_455 +128 val_128 +311 val_311 +316 val_316 +57 val_57 +302 val_302 +205 val_205 +149 val_149 +438 val_438 +345 val_345 +129 val_129 +170 val_170 +20 val_20 +489 val_489 +157 val_157 +378 val_378 +221 val_221 +92 val_92 +111 val_111 +47 val_47 +72 val_72 +4 val_4 +280 val_280 +35 val_35 +427 val_427 +277 val_277 +208 val_208 +356 val_356 +399 val_399 +169 val_169 +382 val_382 +498 val_498 +125 val_125 +386 val_386 +437 val_437 +469 val_469 +192 val_192 +286 val_286 +187 val_187 +176 val_176 +54 val_54 +459 val_459 +51 val_51 +138 val_138 +103 val_103 +239 val_239 +213 val_213 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +289 val_289 +221 val_221 +65 val_65 +318 val_318 +332 val_332 +311 val_311 +275 val_275 +137 val_137 +241 val_241 +83 val_83 +333 val_333 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +181 val_181 +67 val_67 +260 val_260 +404 val_404 +384 val_384 +489 val_489 +353 val_353 +373 val_373 +272 val_272 +138 val_138 +217 val_217 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +411 val_411 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +463 val_463 +431 val_431 +179 val_179 +172 val_172 +42 val_42 +129 val_129 +158 val_158 +119 val_119 +496 val_496 +0 val_0 +322 val_322 +197 val_197 +468 val_468 +393 val_393 +454 val_454 +100 val_100 +298 val_298 +199 val_199 +191 val_191 +418 val_418 +96 val_96 +26 val_26 +165 val_165 +327 val_327 +230 val_230 +205 val_205 +120 val_120 +131 val_131 +51 val_51 +404 val_404 +43 val_43 +436 val_436 +156 val_156 +469 val_469 +468 val_468 +308 val_308 +95 val_95 +196 val_196 +288 val_288 +481 val_481 +457 val_457 +98 val_98 +282 val_282 +197 val_197 +187 val_187 +318 val_318 +318 val_318 +409 val_409 +470 val_470 +137 val_137 +369 val_369 +316 val_316 +169 val_169 +413 val_413 +85 val_85 +77 val_77 +0 val_0 +490 val_490 +87 val_87 +364 val_364 +179 val_179 +118 val_118 +134 val_134 +395 val_395 +282 val_282 +138 val_138 +238 val_238 +419 val_419 +15 val_15 +118 val_118 +72 val_72 +90 val_90 +307 val_307 +19 val_19 +435 val_435 +10 val_10 +277 val_277 +273 val_273 +306 val_306 +224 val_224 +309 val_309 +389 val_389 +327 val_327 +242 val_242 +369 val_369 +392 val_392 +272 val_272 +331 val_331 +401 val_401 +242 val_242 +452 val_452 +177 val_177 +226 val_226 +5 val_5 +497 val_497 +402 val_402 +396 val_396 +317 val_317 +395 val_395 +58 val_58 +35 val_35 +336 val_336 +95 val_95 +11 val_11 +168 val_168 +34 val_34 +229 val_229 +233 val_233 +143 val_143 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +195 val_195 +42 val_42 +321 val_321 +430 val_430 +119 val_119 +489 val_489 +458 val_458 +78 val_78 +76 val_76 +41 val_41 +223 val_223 +492 val_492 +149 val_149 +449 val_449 +218 val_218 +228 val_228 +138 val_138 +453 val_453 +30 val_30 +209 val_209 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +69 val_69 +230 val_230 +33 val_33 +368 val_368 +103 val_103 +296 val_296 +113 val_113 +216 val_216 +367 val_367 +344 val_344 +167 val_167 +274 val_274 +219 val_219 +239 val_239 +485 val_485 +116 val_116 +223 val_223 +256 val_256 +263 val_263 +70 val_70 +487 val_487 +480 val_480 +401 val_401 +288 val_288 +191 val_191 +5 val_5 +244 val_244 +438 val_438 +128 val_128 +467 val_467 +432 val_432 +202 val_202 +316 val_316 +229 val_229 +469 val_469 +463 val_463 +280 val_280 +2 val_2 +35 val_35 +283 val_283 +331 val_331 +235 val_235 +80 val_80 +44 val_44 +193 val_193 +321 val_321 +335 val_335 +104 val_104 +466 val_466 +366 val_366 +175 val_175 +403 val_403 +483 val_483 +53 val_53 +105 val_105 +257 val_257 +406 val_406 +409 val_409 +190 val_190 +406 val_406 +401 val_401 +114 val_114 +258 val_258 +90 val_90 +203 val_203 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +201 val_201 +217 val_217 +164 val_164 +431 val_431 +454 val_454 +478 val_478 +298 val_298 +125 val_125 +431 val_431 +164 val_164 +424 val_424 +187 val_187 +382 val_382 +5 val_5 +70 val_70 +397 val_397 +480 val_480 +291 val_291 +24 val_24 +351 val_351 +255 val_255 +104 val_104 +70 val_70 +163 val_163 +438 val_438 +119 val_119 +414 val_414 +200 val_200 +491 val_491 +237 val_237 +439 val_439 +360 val_360 +248 val_248 +479 val_479 +305 val_305 +417 val_417 +199 val_199 +444 val_444 +120 val_120 +429 val_429 +169 val_169 +443 val_443 +323 val_323 +325 val_325 +277 val_277 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +317 val_317 +333 val_333 +493 val_493 +460 val_460 +207 val_207 +249 val_249 +265 val_265 +480 val_480 +83 val_83 +136 val_136 +353 val_353 +172 val_172 +214 val_214 +462 val_462 +233 val_233 +406 val_406 +133 val_133 +175 val_175 +189 val_189 +454 val_454 +375 val_375 +401 val_401 +421 val_421 +407 val_407 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +67 val_67 +384 val_384 +379 val_379 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +9 val_9 +341 val_341 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +285 val_285 +348 val_348 +167 val_167 +18 val_18 +273 val_273 +183 val_183 +281 val_281 +344 val_344 +97 val_97 +469 val_469 +315 val_315 +84 val_84 +28 val_28 +37 val_37 +448 val_448 +152 val_152 +348 val_348 +307 val_307 +194 val_194 +414 val_414 +477 val_477 +222 val_222 +126 val_126 +90 val_90 +169 val_169 +403 val_403 +400 val_400 +200 val_200 +97 val_97 +PREHOOK: query: CREATE TABLE stats_partitioned(key string, value string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_partitioned +POSTHOOK: query: CREATE TABLE stats_partitioned(key string, value string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_partitioned +PREHOOK: query: explain +insert overwrite table stats_partitioned partition (ds='1') +select * from src +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table stats_partitioned partition (ds='1') +select * from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_partitioned + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.stats_partitioned + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_partitioned@ds=1 +POSTHOOK: query: insert overwrite table stats_partitioned partition (ds='1') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_partitioned@ds=1 +POSTHOOK: Lineage: stats_partitioned PARTITION(ds=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_partitioned PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show partitions stats_partitioned +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@stats_partitioned +POSTHOOK: query: show partitions stats_partitioned +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@stats_partitioned +ds=1 +PREHOOK: query: select * from stats_partitioned where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_partitioned +PREHOOK: Input: default@stats_partitioned@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: select * from stats_partitioned where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_partitioned +POSTHOOK: Input: default@stats_partitioned@ds=1 +#### A masked pattern was here #### +238 val_238 1 +86 val_86 1 +311 val_311 1 +27 val_27 1 +165 val_165 1 +409 val_409 1 +255 val_255 1 +278 val_278 1 +98 val_98 1 +484 val_484 1 +265 val_265 1 +193 val_193 1 +401 val_401 1 +150 val_150 1 +273 val_273 1 +224 val_224 1 +369 val_369 1 +66 val_66 1 +128 val_128 1 +213 val_213 1 +146 val_146 1 +406 val_406 1 +429 val_429 1 +374 val_374 1 +152 val_152 1 +469 val_469 1 +145 val_145 1 +495 val_495 1 +37 val_37 1 +327 val_327 1 +281 val_281 1 +277 val_277 1 +209 val_209 1 +15 val_15 1 +82 val_82 1 +403 val_403 1 +166 val_166 1 +417 val_417 1 +430 val_430 1 +252 val_252 1 +292 val_292 1 +219 val_219 1 +287 val_287 1 +153 val_153 1 +193 val_193 1 +338 val_338 1 +446 val_446 1 +459 val_459 1 +394 val_394 1 +237 val_237 1 +482 val_482 1 +174 val_174 1 +413 val_413 1 +494 val_494 1 +207 val_207 1 +199 val_199 1 +466 val_466 1 +208 val_208 1 +174 val_174 1 +399 val_399 1 +396 val_396 1 +247 val_247 1 +417 val_417 1 +489 val_489 1 +162 val_162 1 +377 val_377 1 +397 val_397 1 +309 val_309 1 +365 val_365 1 +266 val_266 1 +439 val_439 1 +342 val_342 1 +367 val_367 1 +325 val_325 1 +167 val_167 1 +195 val_195 1 +475 val_475 1 +17 val_17 1 +113 val_113 1 +155 val_155 1 +203 val_203 1 +339 val_339 1 +0 val_0 1 +455 val_455 1 +128 val_128 1 +311 val_311 1 +316 val_316 1 +57 val_57 1 +302 val_302 1 +205 val_205 1 +149 val_149 1 +438 val_438 1 +345 val_345 1 +129 val_129 1 +170 val_170 1 +20 val_20 1 +489 val_489 1 +157 val_157 1 +378 val_378 1 +221 val_221 1 +92 val_92 1 +111 val_111 1 +47 val_47 1 +72 val_72 1 +4 val_4 1 +280 val_280 1 +35 val_35 1 +427 val_427 1 +277 val_277 1 +208 val_208 1 +356 val_356 1 +399 val_399 1 +169 val_169 1 +382 val_382 1 +498 val_498 1 +125 val_125 1 +386 val_386 1 +437 val_437 1 +469 val_469 1 +192 val_192 1 +286 val_286 1 +187 val_187 1 +176 val_176 1 +54 val_54 1 +459 val_459 1 +51 val_51 1 +138 val_138 1 +103 val_103 1 +239 val_239 1 +213 val_213 1 +216 val_216 1 +430 val_430 1 +278 val_278 1 +176 val_176 1 +289 val_289 1 +221 val_221 1 +65 val_65 1 +318 val_318 1 +332 val_332 1 +311 val_311 1 +275 val_275 1 +137 val_137 1 +241 val_241 1 +83 val_83 1 +333 val_333 1 +180 val_180 1 +284 val_284 1 +12 val_12 1 +230 val_230 1 +181 val_181 1 +67 val_67 1 +260 val_260 1 +404 val_404 1 +384 val_384 1 +489 val_489 1 +353 val_353 1 +373 val_373 1 +272 val_272 1 +138 val_138 1 +217 val_217 1 +84 val_84 1 +348 val_348 1 +466 val_466 1 +58 val_58 1 +8 val_8 1 +411 val_411 1 +230 val_230 1 +208 val_208 1 +348 val_348 1 +24 val_24 1 +463 val_463 1 +431 val_431 1 +179 val_179 1 +172 val_172 1 +42 val_42 1 +129 val_129 1 +158 val_158 1 +119 val_119 1 +496 val_496 1 +0 val_0 1 +322 val_322 1 +197 val_197 1 +468 val_468 1 +393 val_393 1 +454 val_454 1 +100 val_100 1 +298 val_298 1 +199 val_199 1 +191 val_191 1 +418 val_418 1 +96 val_96 1 +26 val_26 1 +165 val_165 1 +327 val_327 1 +230 val_230 1 +205 val_205 1 +120 val_120 1 +131 val_131 1 +51 val_51 1 +404 val_404 1 +43 val_43 1 +436 val_436 1 +156 val_156 1 +469 val_469 1 +468 val_468 1 +308 val_308 1 +95 val_95 1 +196 val_196 1 +288 val_288 1 +481 val_481 1 +457 val_457 1 +98 val_98 1 +282 val_282 1 +197 val_197 1 +187 val_187 1 +318 val_318 1 +318 val_318 1 +409 val_409 1 +470 val_470 1 +137 val_137 1 +369 val_369 1 +316 val_316 1 +169 val_169 1 +413 val_413 1 +85 val_85 1 +77 val_77 1 +0 val_0 1 +490 val_490 1 +87 val_87 1 +364 val_364 1 +179 val_179 1 +118 val_118 1 +134 val_134 1 +395 val_395 1 +282 val_282 1 +138 val_138 1 +238 val_238 1 +419 val_419 1 +15 val_15 1 +118 val_118 1 +72 val_72 1 +90 val_90 1 +307 val_307 1 +19 val_19 1 +435 val_435 1 +10 val_10 1 +277 val_277 1 +273 val_273 1 +306 val_306 1 +224 val_224 1 +309 val_309 1 +389 val_389 1 +327 val_327 1 +242 val_242 1 +369 val_369 1 +392 val_392 1 +272 val_272 1 +331 val_331 1 +401 val_401 1 +242 val_242 1 +452 val_452 1 +177 val_177 1 +226 val_226 1 +5 val_5 1 +497 val_497 1 +402 val_402 1 +396 val_396 1 +317 val_317 1 +395 val_395 1 +58 val_58 1 +35 val_35 1 +336 val_336 1 +95 val_95 1 +11 val_11 1 +168 val_168 1 +34 val_34 1 +229 val_229 1 +233 val_233 1 +143 val_143 1 +472 val_472 1 +322 val_322 1 +498 val_498 1 +160 val_160 1 +195 val_195 1 +42 val_42 1 +321 val_321 1 +430 val_430 1 +119 val_119 1 +489 val_489 1 +458 val_458 1 +78 val_78 1 +76 val_76 1 +41 val_41 1 +223 val_223 1 +492 val_492 1 +149 val_149 1 +449 val_449 1 +218 val_218 1 +228 val_228 1 +138 val_138 1 +453 val_453 1 +30 val_30 1 +209 val_209 1 +64 val_64 1 +468 val_468 1 +76 val_76 1 +74 val_74 1 +342 val_342 1 +69 val_69 1 +230 val_230 1 +33 val_33 1 +368 val_368 1 +103 val_103 1 +296 val_296 1 +113 val_113 1 +216 val_216 1 +367 val_367 1 +344 val_344 1 +167 val_167 1 +274 val_274 1 +219 val_219 1 +239 val_239 1 +485 val_485 1 +116 val_116 1 +223 val_223 1 +256 val_256 1 +263 val_263 1 +70 val_70 1 +487 val_487 1 +480 val_480 1 +401 val_401 1 +288 val_288 1 +191 val_191 1 +5 val_5 1 +244 val_244 1 +438 val_438 1 +128 val_128 1 +467 val_467 1 +432 val_432 1 +202 val_202 1 +316 val_316 1 +229 val_229 1 +469 val_469 1 +463 val_463 1 +280 val_280 1 +2 val_2 1 +35 val_35 1 +283 val_283 1 +331 val_331 1 +235 val_235 1 +80 val_80 1 +44 val_44 1 +193 val_193 1 +321 val_321 1 +335 val_335 1 +104 val_104 1 +466 val_466 1 +366 val_366 1 +175 val_175 1 +403 val_403 1 +483 val_483 1 +53 val_53 1 +105 val_105 1 +257 val_257 1 +406 val_406 1 +409 val_409 1 +190 val_190 1 +406 val_406 1 +401 val_401 1 +114 val_114 1 +258 val_258 1 +90 val_90 1 +203 val_203 1 +262 val_262 1 +348 val_348 1 +424 val_424 1 +12 val_12 1 +396 val_396 1 +201 val_201 1 +217 val_217 1 +164 val_164 1 +431 val_431 1 +454 val_454 1 +478 val_478 1 +298 val_298 1 +125 val_125 1 +431 val_431 1 +164 val_164 1 +424 val_424 1 +187 val_187 1 +382 val_382 1 +5 val_5 1 +70 val_70 1 +397 val_397 1 +480 val_480 1 +291 val_291 1 +24 val_24 1 +351 val_351 1 +255 val_255 1 +104 val_104 1 +70 val_70 1 +163 val_163 1 +438 val_438 1 +119 val_119 1 +414 val_414 1 +200 val_200 1 +491 val_491 1 +237 val_237 1 +439 val_439 1 +360 val_360 1 +248 val_248 1 +479 val_479 1 +305 val_305 1 +417 val_417 1 +199 val_199 1 +444 val_444 1 +120 val_120 1 +429 val_429 1 +169 val_169 1 +443 val_443 1 +323 val_323 1 +325 val_325 1 +277 val_277 1 +230 val_230 1 +478 val_478 1 +178 val_178 1 +468 val_468 1 +310 val_310 1 +317 val_317 1 +333 val_333 1 +493 val_493 1 +460 val_460 1 +207 val_207 1 +249 val_249 1 +265 val_265 1 +480 val_480 1 +83 val_83 1 +136 val_136 1 +353 val_353 1 +172 val_172 1 +214 val_214 1 +462 val_462 1 +233 val_233 1 +406 val_406 1 +133 val_133 1 +175 val_175 1 +189 val_189 1 +454 val_454 1 +375 val_375 1 +401 val_401 1 +421 val_421 1 +407 val_407 1 +384 val_384 1 +256 val_256 1 +26 val_26 1 +134 val_134 1 +67 val_67 1 +384 val_384 1 +379 val_379 1 +18 val_18 1 +462 val_462 1 +492 val_492 1 +100 val_100 1 +298 val_298 1 +9 val_9 1 +341 val_341 1 +498 val_498 1 +146 val_146 1 +458 val_458 1 +362 val_362 1 +186 val_186 1 +285 val_285 1 +348 val_348 1 +167 val_167 1 +18 val_18 1 +273 val_273 1 +183 val_183 1 +281 val_281 1 +344 val_344 1 +97 val_97 1 +469 val_469 1 +315 val_315 1 +84 val_84 1 +28 val_28 1 +37 val_37 1 +448 val_448 1 +152 val_152 1 +348 val_348 1 +307 val_307 1 +194 val_194 1 +414 val_414 1 +477 val_477 1 +222 val_222 1 +126 val_126 1 +90 val_90 1 +169 val_169 1 +403 val_403 1 +400 val_400 1 +200 val_200 1 +97 val_97 1 +PREHOOK: query: describe extended stats_partitioned partition (ds='1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_partitioned +POSTHOOK: query: describe extended stats_partitioned partition (ds='1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_partitioned +key string +value string +ds string + +# Partition Information +# col_name data_type comment + +ds string + +#### A masked pattern was here #### +PREHOOK: query: describe extended stats_partitioned +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_partitioned +POSTHOOK: query: describe extended stats_partitioned +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_partitioned +key string +value string +ds string + +# Partition Information +# col_name data_type comment + +ds string + +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/stats10.q.out ql/src/test/results/clientpositive/spark/stats10.q.out new file mode 100644 index 0000000..0dd89a2 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats10.q.out @@ -0,0 +1,527 @@ +PREHOOK: query: CREATE TABLE bucket3_1(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@bucket3_1 +POSTHOOK: query: CREATE TABLE bucket3_1(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@bucket3_1 +PREHOOK: query: explain +insert overwrite table bucket3_1 partition (ds='1') +select * from src +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table bucket3_1 partition (ds='1') +select * from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket3_1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 1 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket3_1 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: query: insert overwrite table bucket3_1 partition (ds='1') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: query: insert overwrite table bucket3_1 partition (ds='1') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table bucket3_1 partition (ds='2') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@bucket3_1@ds=2 +POSTHOOK: query: insert overwrite table bucket3_1 partition (ds='2') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@bucket3_1@ds=2 +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: bucket3_1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from bucket3_1 tablesample (bucket 1 out of 2) s where ds = '1' order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@bucket3_1 +PREHOOK: Input: default@bucket3_1@ds=1 +#### A masked pattern was here #### +POSTHOOK: query: select * from bucket3_1 tablesample (bucket 1 out of 2) s where ds = '1' order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@bucket3_1 +POSTHOOK: Input: default@bucket3_1@ds=1 +#### A masked pattern was here #### +0 val_0 1 +0 val_0 1 +0 val_0 1 +2 val_2 1 +4 val_4 1 +8 val_8 1 +10 val_10 1 +12 val_12 1 +12 val_12 1 +18 val_18 1 +18 val_18 1 +20 val_20 1 +24 val_24 1 +24 val_24 1 +26 val_26 1 +26 val_26 1 +28 val_28 1 +30 val_30 1 +34 val_34 1 +42 val_42 1 +42 val_42 1 +44 val_44 1 +54 val_54 1 +58 val_58 1 +58 val_58 1 +64 val_64 1 +66 val_66 1 +70 val_70 1 +70 val_70 1 +70 val_70 1 +72 val_72 1 +72 val_72 1 +74 val_74 1 +76 val_76 1 +76 val_76 1 +78 val_78 1 +80 val_80 1 +82 val_82 1 +84 val_84 1 +84 val_84 1 +86 val_86 1 +90 val_90 1 +90 val_90 1 +90 val_90 1 +92 val_92 1 +96 val_96 1 +98 val_98 1 +98 val_98 1 +100 val_100 1 +100 val_100 1 +104 val_104 1 +104 val_104 1 +114 val_114 1 +116 val_116 1 +118 val_118 1 +118 val_118 1 +120 val_120 1 +120 val_120 1 +126 val_126 1 +128 val_128 1 +128 val_128 1 +128 val_128 1 +134 val_134 1 +134 val_134 1 +136 val_136 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +138 val_138 1 +146 val_146 1 +146 val_146 1 +150 val_150 1 +152 val_152 1 +152 val_152 1 +156 val_156 1 +158 val_158 1 +160 val_160 1 +162 val_162 1 +164 val_164 1 +164 val_164 1 +166 val_166 1 +168 val_168 1 +170 val_170 1 +172 val_172 1 +172 val_172 1 +174 val_174 1 +174 val_174 1 +176 val_176 1 +176 val_176 1 +178 val_178 1 +180 val_180 1 +186 val_186 1 +190 val_190 1 +192 val_192 1 +194 val_194 1 +196 val_196 1 +200 val_200 1 +200 val_200 1 +202 val_202 1 +208 val_208 1 +208 val_208 1 +208 val_208 1 +214 val_214 1 +216 val_216 1 +216 val_216 1 +218 val_218 1 +222 val_222 1 +224 val_224 1 +224 val_224 1 +226 val_226 1 +228 val_228 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +230 val_230 1 +238 val_238 1 +238 val_238 1 +242 val_242 1 +242 val_242 1 +244 val_244 1 +248 val_248 1 +252 val_252 1 +256 val_256 1 +256 val_256 1 +258 val_258 1 +260 val_260 1 +262 val_262 1 +266 val_266 1 +272 val_272 1 +272 val_272 1 +274 val_274 1 +278 val_278 1 +278 val_278 1 +280 val_280 1 +280 val_280 1 +282 val_282 1 +282 val_282 1 +284 val_284 1 +286 val_286 1 +288 val_288 1 +288 val_288 1 +292 val_292 1 +296 val_296 1 +298 val_298 1 +298 val_298 1 +298 val_298 1 +302 val_302 1 +306 val_306 1 +308 val_308 1 +310 val_310 1 +316 val_316 1 +316 val_316 1 +316 val_316 1 +318 val_318 1 +318 val_318 1 +318 val_318 1 +322 val_322 1 +322 val_322 1 +332 val_332 1 +336 val_336 1 +338 val_338 1 +342 val_342 1 +342 val_342 1 +344 val_344 1 +344 val_344 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +348 val_348 1 +356 val_356 1 +360 val_360 1 +362 val_362 1 +364 val_364 1 +366 val_366 1 +368 val_368 1 +374 val_374 1 +378 val_378 1 +382 val_382 1 +382 val_382 1 +384 val_384 1 +384 val_384 1 +384 val_384 1 +386 val_386 1 +392 val_392 1 +394 val_394 1 +396 val_396 1 +396 val_396 1 +396 val_396 1 +400 val_400 1 +402 val_402 1 +404 val_404 1 +404 val_404 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +406 val_406 1 +414 val_414 1 +414 val_414 1 +418 val_418 1 +424 val_424 1 +424 val_424 1 +430 val_430 1 +430 val_430 1 +430 val_430 1 +432 val_432 1 +436 val_436 1 +438 val_438 1 +438 val_438 1 +438 val_438 1 +444 val_444 1 +446 val_446 1 +448 val_448 1 +452 val_452 1 +454 val_454 1 +454 val_454 1 +454 val_454 1 +458 val_458 1 +458 val_458 1 +460 val_460 1 +462 val_462 1 +462 val_462 1 +466 val_466 1 +466 val_466 1 +466 val_466 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +468 val_468 1 +470 val_470 1 +472 val_472 1 +478 val_478 1 +478 val_478 1 +480 val_480 1 +480 val_480 1 +480 val_480 1 +482 val_482 1 +484 val_484 1 +490 val_490 1 +492 val_492 1 +492 val_492 1 +494 val_494 1 +496 val_496 1 +498 val_498 1 +498 val_498 1 +498 val_498 1 +PREHOOK: query: explain analyze table bucket3_1 partition (ds) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table bucket3_1 partition (ds) compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: bucket3_1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table bucket3_1 partition (ds) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@bucket3_1 +PREHOOK: Input: default@bucket3_1@ds=1 +PREHOOK: Input: default@bucket3_1@ds=2 +PREHOOK: Output: default@bucket3_1 +PREHOOK: Output: default@bucket3_1@ds=1 +PREHOOK: Output: default@bucket3_1@ds=2 +POSTHOOK: query: analyze table bucket3_1 partition (ds) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@bucket3_1 +POSTHOOK: Input: default@bucket3_1@ds=1 +POSTHOOK: Input: default@bucket3_1@ds=2 +POSTHOOK: Output: default@bucket3_1 +POSTHOOK: Output: default@bucket3_1@ds=1 +POSTHOOK: Output: default@bucket3_1@ds=2 +PREHOOK: query: describe formatted bucket3_1 partition (ds='1') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@bucket3_1 +POSTHOOK: query: describe formatted bucket3_1 partition (ds='1') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@bucket3_1 +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: bucket3_1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 2 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted bucket3_1 partition (ds='2') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@bucket3_1 +POSTHOOK: query: describe formatted bucket3_1 partition (ds='2') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@bucket3_1 +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2] +Database: default +Table: bucket3_1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 2 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted bucket3_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@bucket3_1 +POSTHOOK: query: describe formatted bucket3_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@bucket3_1 +# col_name data_type comment + +key int +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats12.q.out ql/src/test/results/clientpositive/spark/stats12.q.out new file mode 100644 index 0000000..8c85371 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats12.q.out @@ -0,0 +1,395 @@ +PREHOOK: query: create table analyze_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: create table analyze_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart +PREHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain extended +analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_ANALYZE + TOK_TAB + TOK_TABNAME + analyze_srcpart + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + + +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcpart + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: NONE + Statistics Aggregation Key Prefix: default.analyze_srcpart/ + GatherStats: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE false + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.analyze_srcpart + numFiles 1 + numRows -1 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize -1 + serialization.ddl struct analyze_srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.analyze_srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct analyze_srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.analyze_srcpart + name: default.analyze_srcpart +#### A masked pattern was here #### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + COLUMN_STATS_ACCURATE false + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.analyze_srcpart + numFiles 1 + numRows -1 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize -1 + serialization.ddl struct analyze_srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.analyze_srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct analyze_srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.analyze_srcpart + name: default.analyze_srcpart + Truncated Path -> Alias: + /analyze_srcpart/ds=2008-04-08/hr=11 [analyze_srcpart] + /analyze_srcpart/ds=2008-04-08/hr=12 [analyze_srcpart] + + Stage: Stage-2 + Stats-Aggr Operator + Stats Aggregation Key Prefix: default.analyze_srcpart/ + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: query: desc formatted analyze_srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats13.q.out ql/src/test/results/clientpositive/spark/stats13.q.out new file mode 100644 index 0000000..da584e0 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats13.q.out @@ -0,0 +1,390 @@ +PREHOOK: query: create table analyze_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: create table analyze_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart +PREHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain extended +analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_ANALYZE + TOK_TAB + TOK_TABNAME + analyze_srcpart + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + 11 + + +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcpart + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: NONE + Statistics Aggregation Key Prefix: default.analyze_srcpart/ + GatherStats: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE false + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.analyze_srcpart + numFiles 1 + numRows -1 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize -1 + serialization.ddl struct analyze_srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.analyze_srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct analyze_srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.analyze_srcpart + name: default.analyze_srcpart + Truncated Path -> Alias: + /analyze_srcpart/ds=2008-04-08/hr=11 [analyze_srcpart] + + Stage: Stage-2 + Stats-Aggr Operator + Stats Aggregation Key Prefix: default.analyze_srcpart/ + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: query: desc formatted analyze_srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-08', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: desc formatted analyze_srcpart partition (ds='2008-04-09', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table analyze_srcpart2 like analyze_srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart2 +POSTHOOK: query: create table analyze_srcpart2 like analyze_srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart2 +PREHOOK: query: desc formatted analyze_srcpart2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart2 +POSTHOOK: query: desc formatted analyze_srcpart2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart2 +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats14.q.out ql/src/test/results/clientpositive/spark/stats14.q.out new file mode 100644 index 0000000..f12b136 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats14.q.out @@ -0,0 +1,318 @@ +PREHOOK: query: create table stats_src like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_src +POSTHOOK: query: create table stats_src like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_src +PREHOOK: query: insert overwrite table stats_src select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_src +POSTHOOK: query: insert overwrite table stats_src select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_src +POSTHOOK: Lineage: stats_src.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: analyze table stats_src compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_src +PREHOOK: Output: default@stats_src +POSTHOOK: query: analyze table stats_src compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_src +POSTHOOK: Output: default@stats_src +PREHOOK: query: desc formatted stats_src +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_src +POSTHOOK: query: desc formatted stats_src +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_src +# col_name data_type comment + +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table stats_part like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_part +POSTHOOK: query: create table stats_part like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_part +PREHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '11') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '11') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=11).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '12') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '12') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=12).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=12).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: analyze table stats_part partition(ds='2010-04-08', hr='11') compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=11 +PREHOOK: Output: default@stats_part +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: query: analyze table stats_part partition(ds='2010-04-08', hr='11') compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: Output: default@stats_part +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +PREHOOK: query: analyze table stats_part partition(ds='2010-04-08', hr='12') compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=12 +PREHOOK: Output: default@stats_part +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: query: analyze table stats_part partition(ds='2010-04-08', hr='12') compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: Output: default@stats_part +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +PREHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=13).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=13).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted stats_part +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr = '11') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr = '11') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-04-08, 11] +Database: default +Table: stats_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr = '12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr = '12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-04-08, 12] +Database: default +Table: stats_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table stats_part partition(ds, hr) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=11 +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=12 +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=13 +PREHOOK: Output: default@stats_part +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: query: analyze table stats_part partition(ds, hr) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: Output: default@stats_part +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +PREHOOK: query: desc formatted stats_part +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table stats_src +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_src +PREHOOK: Output: default@stats_src +POSTHOOK: query: drop table stats_src +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_src +POSTHOOK: Output: default@stats_src +PREHOOK: query: drop table stats_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_part +PREHOOK: Output: default@stats_part +POSTHOOK: query: drop table stats_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_part +POSTHOOK: Output: default@stats_part diff --git ql/src/test/results/clientpositive/spark/stats15.q.out ql/src/test/results/clientpositive/spark/stats15.q.out new file mode 100644 index 0000000..a60dee2 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats15.q.out @@ -0,0 +1,318 @@ +PREHOOK: query: create table stats_src like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_src +POSTHOOK: query: create table stats_src like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_src +PREHOOK: query: insert overwrite table stats_src select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_src +POSTHOOK: query: insert overwrite table stats_src select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_src +POSTHOOK: Lineage: stats_src.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: analyze table stats_src compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_src +PREHOOK: Output: default@stats_src +POSTHOOK: query: analyze table stats_src compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_src +POSTHOOK: Output: default@stats_src +PREHOOK: query: desc formatted stats_src +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_src +POSTHOOK: query: desc formatted stats_src +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_src +# col_name data_type comment + +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table stats_part like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_part +POSTHOOK: query: create table stats_part like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_part +PREHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '11') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '11') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=11).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '12') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '12') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=12).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=12).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: analyze table stats_part partition(ds='2010-04-08', hr='11') compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=11 +PREHOOK: Output: default@stats_part +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: query: analyze table stats_part partition(ds='2010-04-08', hr='11') compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: Output: default@stats_part +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +PREHOOK: query: analyze table stats_part partition(ds='2010-04-08', hr='12') compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=12 +PREHOOK: Output: default@stats_part +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: query: analyze table stats_part partition(ds='2010-04-08', hr='12') compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: Output: default@stats_part +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +PREHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=13).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=13).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted stats_part +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr = '11') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr = '11') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-04-08, 11] +Database: default +Table: stats_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr = '12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr = '12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-04-08, 12] +Database: default +Table: stats_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table stats_part partition(ds, hr) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=11 +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=12 +PREHOOK: Input: default@stats_part@ds=2010-04-08/hr=13 +PREHOOK: Output: default@stats_part +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: query: analyze table stats_part partition(ds, hr) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: Input: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: Output: default@stats_part +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=11 +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=12 +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +PREHOOK: query: desc formatted stats_part +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table stats_src +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_src +PREHOOK: Output: default@stats_src +POSTHOOK: query: drop table stats_src +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_src +POSTHOOK: Output: default@stats_src +PREHOOK: query: drop table stats_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_part +PREHOOK: Output: default@stats_part +POSTHOOK: query: drop table stats_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_part +POSTHOOK: Output: default@stats_part diff --git ql/src/test/results/clientpositive/spark/stats16.q.out ql/src/test/results/clientpositive/spark/stats16.q.out new file mode 100644 index 0000000..3f0f2ea --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats16.q.out @@ -0,0 +1,105 @@ +PREHOOK: query: drop table stats16 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table stats16 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table stats16 (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats16 +POSTHOOK: query: create table stats16 (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats16 +PREHOOK: query: desc formatted stats16 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats16 +POSTHOOK: query: desc formatted stats16 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats16 +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table stats16 select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats16 +POSTHOOK: query: insert into table stats16 select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats16 +POSTHOOK: Lineage: stats16.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats16.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: analyze table stats16 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats16 +PREHOOK: Output: default@stats16 +POSTHOOK: query: analyze table stats16 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats16 +POSTHOOK: Output: default@stats16 +PREHOOK: query: desc formatted stats16 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats16 +POSTHOOK: query: desc formatted stats16 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats16 +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table stats16 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats16 +PREHOOK: Output: default@stats16 +POSTHOOK: query: drop table stats16 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats16 +POSTHOOK: Output: default@stats16 diff --git ql/src/test/results/clientpositive/spark/stats18.q.out ql/src/test/results/clientpositive/spark/stats18.q.out new file mode 100644 index 0000000..a061846 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats18.q.out @@ -0,0 +1,128 @@ +PREHOOK: query: create table stats_part like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_part +POSTHOOK: query: create table stats_part like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_part +PREHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: query: insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=13).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_part PARTITION(ds=2010-04-08,hr=13).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- Load a file into a existing partition +-- Some stats (numFiles, totalSize) should be updated correctly +-- Some other stats (numRows, rawDataSize) should be cleared +desc formatted stats_part partition (ds='2010-04-08', hr='13') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: -- Load a file into a existing partition +-- Some stats (numFiles, totalSize) should be updated correctly +-- Some other stats (numRows, rawDataSize) should be cleared +desc formatted stats_part partition (ds='2010-04-08', hr='13') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-04-08, 13] +Database: default +Table: stats_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE stats_part partition (ds='2010-04-08', hr='13') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE stats_part partition (ds='2010-04-08', hr='13') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@stats_part@ds=2010-04-08/hr=13 +PREHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr='13') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_part +POSTHOOK: query: desc formatted stats_part partition (ds='2010-04-08', hr='13') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_part +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-04-08, 13] +Database: default +Table: stats_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 2 + numRows 0 + rawDataSize 0 + totalSize 7170 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table stats_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table stats_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table stats_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_part +PREHOOK: Output: default@stats_part +POSTHOOK: query: drop table stats_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_part +POSTHOOK: Output: default@stats_part diff --git ql/src/test/results/clientpositive/spark/stats2.q.out ql/src/test/results/clientpositive/spark/stats2.q.out new file mode 100644 index 0000000..fc7916f --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats2.q.out @@ -0,0 +1,205 @@ +PREHOOK: query: create table analyze_t1 like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_t1 +POSTHOOK: query: create table analyze_t1 like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_t1 +PREHOOK: query: explain +insert overwrite table analyze_t1 partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table analyze_t1 partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.analyze_t1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.analyze_t1 + +PREHOOK: query: insert overwrite table analyze_t1 partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_t1 +POSTHOOK: query: insert overwrite table analyze_t1 partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_t1@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_t1@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_t1@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_t1@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_t1 PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted analyze_t1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_t1 +POSTHOOK: query: desc formatted analyze_t1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_t1 +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain analyze table analyze_t1 partition (ds, hr) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_t1 partition (ds, hr) compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_t1 + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: NONE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_t1 partition (ds, hr) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_t1 +PREHOOK: Input: default@analyze_t1@ds=2008-04-08/hr=11 +PREHOOK: Input: default@analyze_t1@ds=2008-04-08/hr=12 +PREHOOK: Input: default@analyze_t1@ds=2008-04-09/hr=11 +PREHOOK: Input: default@analyze_t1@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_t1 +PREHOOK: Output: default@analyze_t1@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_t1@ds=2008-04-08/hr=12 +PREHOOK: Output: default@analyze_t1@ds=2008-04-09/hr=11 +PREHOOK: Output: default@analyze_t1@ds=2008-04-09/hr=12 +POSTHOOK: query: analyze table analyze_t1 partition (ds, hr) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_t1 +POSTHOOK: Input: default@analyze_t1@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@analyze_t1@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@analyze_t1@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@analyze_t1@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_t1 +POSTHOOK: Output: default@analyze_t1@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_t1@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_t1@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_t1@ds=2008-04-09/hr=12 +PREHOOK: query: describe formatted analyze_t1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_t1 +POSTHOOK: query: describe formatted analyze_t1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_t1 +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats20.q.out ql/src/test/results/clientpositive/spark/stats20.q.out new file mode 100644 index 0000000..4ac7bc5 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats20.q.out @@ -0,0 +1,108 @@ +PREHOOK: query: CREATE TABLE stats_partitioned(key string, value string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_partitioned +POSTHOOK: query: CREATE TABLE stats_partitioned(key string, value string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_partitioned +PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_partitioned@ds=1 +POSTHOOK: query: insert overwrite table stats_partitioned partition (ds='1') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_partitioned@ds=1 +POSTHOOK: Lineage: stats_partitioned PARTITION(ds=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_partitioned PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- rawDataSize is 5312 after config is turned on +describe formatted stats_partitioned +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_partitioned +POSTHOOK: query: -- rawDataSize is 5312 after config is turned on +describe formatted stats_partitioned +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_partitioned +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') +select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@stats_partitioned@ds=1 +POSTHOOK: query: insert overwrite table stats_partitioned partition (ds='1') +select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@stats_partitioned@ds=1 +POSTHOOK: Lineage: stats_partitioned PARTITION(ds=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: stats_partitioned PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- rawDataSize is 0 after config is turned off +describe formatted stats_partitioned +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_partitioned +POSTHOOK: query: -- rawDataSize is 0 after config is turned off +describe formatted stats_partitioned +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_partitioned +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats3.q.out ql/src/test/results/clientpositive/spark/stats3.q.out new file mode 100644 index 0000000..dd3a95b --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats3.q.out @@ -0,0 +1,262 @@ +PREHOOK: query: drop table hive_test_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table hive_test_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table hive_test_dst +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table hive_test_dst +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table hive_test_src ( col1 string ) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hive_test_src +POSTHOOK: query: create table hive_test_src ( col1 string ) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hive_test_src +PREHOOK: query: explain extended +load data local inpath '../../data/files/test.dat' overwrite into table hive_test_src +PREHOOK: type: LOAD +POSTHOOK: query: explain extended +load data local inpath '../../data/files/test.dat' overwrite into table hive_test_src +POSTHOOK: type: LOAD +ABSTRACT SYNTAX TREE: + +TOK_LOAD + '../../data/files/test.dat' + TOK_TAB + TOK_TABNAME + hive_test_src + local + overwrite + + +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-1 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns col1 + columns.comments + columns.types string +#### A masked pattern was here #### + name default.hive_test_src + serialization.ddl struct hive_test_src { string col1} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.hive_test_src + + Stage: Stage-1 + Stats-Aggr Operator + +PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test_src +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@hive_test_src +POSTHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test_src +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@hive_test_src +PREHOOK: query: desc formatted hive_test_src +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@hive_test_src +POSTHOOK: query: desc formatted hive_test_src +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@hive_test_src +# col_name data_type comment + +col1 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 11 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as sequencefile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hive_test_dst +POSTHOOK: query: create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string) stored as sequencefile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hive_test_dst +PREHOOK: query: insert overwrite table hive_test_dst partition ( pcol1='test_part', pCol2='test_Part') select col1 from hive_test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_src +PREHOOK: Output: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: query: insert overwrite table hive_test_dst partition ( pcol1='test_part', pCol2='test_Part') select col1 from hive_test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_src +POSTHOOK: Output: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_Part' +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst +PREHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +#### A masked pattern was here #### +POSTHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_Part' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst +POSTHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +#### A masked pattern was here #### +1 test_part test_Part +2 test_part test_Part +3 test_part test_Part +4 test_part test_Part +5 test_part test_Part +6 test_part test_Part +PREHOOK: query: select count(1) from hive_test_dst +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst +PREHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from hive_test_dst +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst +POSTHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +#### A masked pattern was here #### +6 +PREHOOK: query: insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_src +PREHOOK: Output: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: query: insert overwrite table hive_test_dst partition ( pCol1='test_part', pcol2='test_Part') select col1 from hive_test_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_src +POSTHOOK: Output: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +POSTHOOK: Lineage: hive_test_dst PARTITION(pcol1=test_part,pcol2=test_Part).col1 SIMPLE [(hive_test_src)hive_test_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part' +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst +#### A masked pattern was here #### +POSTHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst +#### A masked pattern was here #### +PREHOOK: query: select count(1) from hive_test_dst +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst +PREHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from hive_test_dst +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst +POSTHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +#### A masked pattern was here #### +6 +PREHOOK: query: select * from hive_test_dst where pcol1='test_part' +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst +PREHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +#### A masked pattern was here #### +POSTHOOK: query: select * from hive_test_dst where pcol1='test_part' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst +POSTHOOK: Input: default@hive_test_dst@pcol1=test_part/pcol2=test_Part +#### A masked pattern was here #### +1 test_part test_Part +2 test_part test_Part +3 test_part test_Part +4 test_part test_Part +5 test_part test_Part +6 test_part test_Part +PREHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part' +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst +#### A masked pattern was here #### +POSTHOOK: query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst +#### A masked pattern was here #### +PREHOOK: query: select * from hive_test_dst where pcol1='test_Part' +PREHOOK: type: QUERY +PREHOOK: Input: default@hive_test_dst +#### A masked pattern was here #### +POSTHOOK: query: select * from hive_test_dst where pcol1='test_Part' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive_test_dst +#### A masked pattern was here #### +PREHOOK: query: describe formatted hive_test_dst +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@hive_test_dst +POSTHOOK: query: describe formatted hive_test_dst +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@hive_test_dst +# col_name data_type comment + +col1 string + +# Partition Information +# col_name data_type comment + +pcol1 string +pcol2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table hive_test_src +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hive_test_src +PREHOOK: Output: default@hive_test_src +POSTHOOK: query: drop table hive_test_src +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hive_test_src +POSTHOOK: Output: default@hive_test_src +PREHOOK: query: drop table hive_test_dst +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hive_test_dst +PREHOOK: Output: default@hive_test_dst +POSTHOOK: query: drop table hive_test_dst +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hive_test_dst +POSTHOOK: Output: default@hive_test_dst diff --git ql/src/test/results/clientpositive/spark/stats5.q.out ql/src/test/results/clientpositive/spark/stats5.q.out new file mode 100644 index 0000000..0004aee --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats5.q.out @@ -0,0 +1,76 @@ +PREHOOK: query: create table analyze_src as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_src +POSTHOOK: query: create table analyze_src as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_src +PREHOOK: query: explain analyze table analyze_src compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_src compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_src + Statistics: Num rows: -1 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_src compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_src +PREHOOK: Output: default@analyze_src +POSTHOOK: query: analyze table analyze_src compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_src +POSTHOOK: Output: default@analyze_src +PREHOOK: query: describe formatted analyze_src +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_src +POSTHOOK: query: describe formatted analyze_src +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_src +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats6.q.out ql/src/test/results/clientpositive/spark/stats6.q.out new file mode 100644 index 0000000..b4435f2 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats6.q.out @@ -0,0 +1,264 @@ +PREHOOK: query: create table analyze_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: create table analyze_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart +PREHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats7.q.out ql/src/test/results/clientpositive/spark/stats7.q.out new file mode 100644 index 0000000..bce3272 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats7.q.out @@ -0,0 +1,194 @@ +PREHOOK: query: create table analyze_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: create table analyze_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart +PREHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcpart + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: NONE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats8.q.out ql/src/test/results/clientpositive/spark/stats8.q.out new file mode 100644 index 0000000..d618616 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats8.q.out @@ -0,0 +1,627 @@ +PREHOOK: query: create table analyze_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: create table analyze_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart +PREHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcpart + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: NONE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcpart + Statistics: Num rows: 500 Data size: 5312 Basic stats: PARTIAL Column stats: NONE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: PARTIAL Column stats: NONE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-09/hr=11 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcpart + Statistics: Num rows: 1500 Data size: 15936 Basic stats: PARTIAL Column stats: NONE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain analyze table analyze_srcpart PARTITION(ds, hr) compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_srcpart PARTITION(ds, hr) compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds, hr) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds, hr) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats9.q.out ql/src/test/results/clientpositive/spark/stats9.q.out new file mode 100644 index 0000000..3eef1bb --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats9.q.out @@ -0,0 +1,84 @@ +PREHOOK: query: create table analyze_srcbucket like srcbucket +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcbucket +POSTHOOK: query: create table analyze_srcbucket like srcbucket +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcbucket +PREHOOK: query: insert overwrite table analyze_srcbucket select * from srcbucket +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@analyze_srcbucket +POSTHOOK: query: insert overwrite table analyze_srcbucket select * from srcbucket +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@analyze_srcbucket +POSTHOOK: Lineage: analyze_srcbucket.key SIMPLE [(srcbucket)srcbucket.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: analyze_srcbucket.value SIMPLE [(srcbucket)srcbucket.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: explain analyze table analyze_srcbucket compute statistics +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze table analyze_srcbucket compute statistics +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-0 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: analyze_srcbucket + Statistics: Num rows: -1 Data size: 11603 Basic stats: PARTIAL Column stats: COMPLETE + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcbucket compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcbucket +PREHOOK: Output: default@analyze_srcbucket +POSTHOOK: query: analyze table analyze_srcbucket compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcbucket +POSTHOOK: Output: default@analyze_srcbucket +PREHOOK: query: describe formatted analyze_srcbucket +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcbucket +POSTHOOK: query: describe formatted analyze_srcbucket +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcbucket +# col_name data_type comment + +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 1000 + rawDataSize 10603 + totalSize 11603 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats_counter.q.out ql/src/test/results/clientpositive/spark/stats_counter.q.out new file mode 100644 index 0000000..e2980e8 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats_counter.q.out @@ -0,0 +1,104 @@ +PREHOOK: query: -- by analyze +create table dummy1 as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@dummy1 +POSTHOOK: query: -- by analyze +create table dummy1 as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dummy1 +PREHOOK: query: analyze table dummy1 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@dummy1 +PREHOOK: Output: default@dummy1 +POSTHOOK: query: analyze table dummy1 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dummy1 +POSTHOOK: Output: default@dummy1 +PREHOOK: query: desc formatted dummy1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@dummy1 +POSTHOOK: query: desc formatted dummy1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@dummy1 +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: -- by autogather +create table dummy2 as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@dummy2 +POSTHOOK: query: -- by autogather +create table dummy2 as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dummy2 +PREHOOK: query: desc formatted dummy2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@dummy2 +POSTHOOK: query: desc formatted dummy2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@dummy2 +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out new file mode 100644 index 0000000..aa49526 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out @@ -0,0 +1,537 @@ +PREHOOK: query: -- test analyze table ... compute statistics noscan + +-- 1. test full spec +create table analyze_srcpart like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: -- test analyze table ... compute statistics noscan + +-- 1. test full spec +create table analyze_srcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart +PREHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: insert overwrite table analyze_srcpart partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan +PREHOOK: type: QUERY +POSTHOOK: query: explain +analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + +STAGE PLANS: + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=11 +PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Input: default@analyze_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart@ds=2008-04-08/hr=12 +PREHOOK: query: -- confirm result +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: -- confirm result +describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart +POSTHOOK: query: describe formatted analyze_srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table analyze_srcpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@analyze_srcpart +PREHOOK: Output: default@analyze_srcpart +POSTHOOK: query: drop table analyze_srcpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@analyze_srcpart +POSTHOOK: Output: default@analyze_srcpart +PREHOOK: query: -- 2. test partial spec +create table analyze_srcpart_partial like srcpart +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart_partial +POSTHOOK: query: -- 2. test partial spec +create table analyze_srcpart_partial like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart_partial +PREHOOK: query: insert overwrite table analyze_srcpart_partial partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart_partial +POSTHOOK: query: insert overwrite table analyze_srcpart_partial partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart_partial PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan +PREHOOK: type: QUERY +POSTHOOK: query: explain +analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + +STAGE PLANS: + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart_partial +PREHOOK: Input: default@analyze_srcpart_partial@ds=2008-04-08/hr=11 +PREHOOK: Input: default@analyze_srcpart_partial@ds=2008-04-08/hr=12 +PREHOOK: Output: default@analyze_srcpart_partial +PREHOOK: Output: default@analyze_srcpart_partial@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart_partial@ds=2008-04-08/hr=12 +POSTHOOK: query: analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart_partial +POSTHOOK: Input: default@analyze_srcpart_partial@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@analyze_srcpart_partial@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial +POSTHOOK: Output: default@analyze_srcpart_partial@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial@ds=2008-04-08/hr=12 +PREHOOK: query: -- confirm result +describe formatted analyze_srcpart_partial PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial +POSTHOOK: query: -- confirm result +describe formatted analyze_srcpart_partial PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart_partial +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart_partial PARTITION(ds='2008-04-08',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial +POSTHOOK: query: describe formatted analyze_srcpart_partial PARTITION(ds='2008-04-08',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: analyze_srcpart_partial +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart_partial PARTITION(ds='2008-04-09',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial +POSTHOOK: query: describe formatted analyze_srcpart_partial PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart_partial +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart_partial PARTITION(ds='2008-04-09',hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial +POSTHOOK: query: describe formatted analyze_srcpart_partial PARTITION(ds='2008-04-09',hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: analyze_srcpart_partial +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table analyze_srcpart_partial +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@analyze_srcpart_partial +PREHOOK: Output: default@analyze_srcpart_partial +POSTHOOK: query: drop table analyze_srcpart_partial +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@analyze_srcpart_partial +POSTHOOK: Output: default@analyze_srcpart_partial diff --git ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out new file mode 100644 index 0000000..238727d --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats_noscan_2.q.out @@ -0,0 +1,315 @@ +PREHOOK: query: -- test analyze table compute statistiscs [noscan] on external table +-- 1 test table +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@anaylyze_external +POSTHOOK: query: -- test analyze table compute statistiscs [noscan] on external table +-- 1 test table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@anaylyze_external +PREHOOK: query: SELECT * FROM anaylyze_external +PREHOOK: type: QUERY +PREHOOK: Input: default@anaylyze_external +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM anaylyze_external +POSTHOOK: type: QUERY +POSTHOOK: Input: default@anaylyze_external +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +PREHOOK: query: analyze table anaylyze_external compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Input: default@anaylyze_external +PREHOOK: Output: default@anaylyze_external +POSTHOOK: query: analyze table anaylyze_external compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Input: default@anaylyze_external +POSTHOOK: Output: default@anaylyze_external +PREHOOK: query: describe formatted anaylyze_external +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@anaylyze_external +POSTHOOK: query: describe formatted anaylyze_external +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@anaylyze_external +# col_name data_type comment + +a int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + EXTERNAL TRUE + numFiles 0 + numRows -1 + rawDataSize -1 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table anaylyze_external compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@anaylyze_external +PREHOOK: Output: default@anaylyze_external +POSTHOOK: query: analyze table anaylyze_external compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@anaylyze_external +POSTHOOK: Output: default@anaylyze_external +PREHOOK: query: describe formatted anaylyze_external +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@anaylyze_external +POSTHOOK: query: describe formatted anaylyze_external +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@anaylyze_external +# col_name data_type comment + +a int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + EXTERNAL TRUE + numFiles 0 + numRows 6 + rawDataSize 6 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table anaylyze_external +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@anaylyze_external +PREHOOK: Output: default@anaylyze_external +POSTHOOK: query: drop table anaylyze_external +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@anaylyze_external +POSTHOOK: Output: default@anaylyze_external +PREHOOK: query: -- 2 test partition +-- prepare data +create table texternal(key string, val string) partitioned by (insertdate string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@texternal +POSTHOOK: query: -- 2 test partition +-- prepare data +create table texternal(key string, val string) partitioned by (insertdate string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@texternal +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@texternal +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@texternal +POSTHOOK: Output: default@texternal@insertdate=2008-01-01 +PREHOOK: query: from src insert overwrite table texternal partition (insertdate='2008-01-01') select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@texternal@insertdate=2008-01-01 +POSTHOOK: query: from src insert overwrite table texternal partition (insertdate='2008-01-01') select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@texternal@insertdate=2008-01-01 +POSTHOOK: Lineage: texternal PARTITION(insertdate=2008-01-01).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: texternal PARTITION(insertdate=2008-01-01).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from texternal where insertdate='2008-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@texternal +PREHOOK: Input: default@texternal@insertdate=2008-01-01 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from texternal where insertdate='2008-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@texternal +POSTHOOK: Input: default@texternal@insertdate=2008-01-01 +#### A masked pattern was here #### +500 +PREHOOK: query: -- create external table +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@anaylyze_external +POSTHOOK: query: -- create external table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@anaylyze_external +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@anaylyze_external +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@anaylyze_external +POSTHOOK: Output: default@anaylyze_external@insertdate=2008-01-01 +PREHOOK: query: select count(*) from anaylyze_external where insertdate='2008-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@anaylyze_external +PREHOOK: Input: default@anaylyze_external@insertdate=2008-01-01 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from anaylyze_external where insertdate='2008-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@anaylyze_external +POSTHOOK: Input: default@anaylyze_external@insertdate=2008-01-01 +#### A masked pattern was here #### +500 +PREHOOK: query: -- analyze +analyze table anaylyze_external PARTITION (insertdate='2008-01-01') compute statistics noscan +PREHOOK: type: QUERY +PREHOOK: Input: default@anaylyze_external +PREHOOK: Input: default@anaylyze_external@insertdate=2008-01-01 +PREHOOK: Output: default@anaylyze_external +PREHOOK: Output: default@anaylyze_external@insertdate=2008-01-01 +POSTHOOK: query: -- analyze +analyze table anaylyze_external PARTITION (insertdate='2008-01-01') compute statistics noscan +POSTHOOK: type: QUERY +POSTHOOK: Input: default@anaylyze_external +POSTHOOK: Input: default@anaylyze_external@insertdate=2008-01-01 +POSTHOOK: Output: default@anaylyze_external +POSTHOOK: Output: default@anaylyze_external@insertdate=2008-01-01 +PREHOOK: query: describe formatted anaylyze_external PARTITION (insertdate='2008-01-01') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@anaylyze_external +POSTHOOK: query: describe formatted anaylyze_external PARTITION (insertdate='2008-01-01') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@anaylyze_external +# col_name data_type comment + +key string +val string + +# Partition Information +# col_name data_type comment + +insertdate string + +# Detailed Partition Information +Partition Value: [2008-01-01] +Database: default +Table: anaylyze_external +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table anaylyze_external PARTITION (insertdate='2008-01-01') compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@anaylyze_external +PREHOOK: Input: default@anaylyze_external@insertdate=2008-01-01 +PREHOOK: Output: default@anaylyze_external +PREHOOK: Output: default@anaylyze_external@insertdate=2008-01-01 +POSTHOOK: query: analyze table anaylyze_external PARTITION (insertdate='2008-01-01') compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@anaylyze_external +POSTHOOK: Input: default@anaylyze_external@insertdate=2008-01-01 +POSTHOOK: Output: default@anaylyze_external +POSTHOOK: Output: default@anaylyze_external@insertdate=2008-01-01 +PREHOOK: query: describe formatted anaylyze_external PARTITION (insertdate='2008-01-01') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@anaylyze_external +POSTHOOK: query: describe formatted anaylyze_external PARTITION (insertdate='2008-01-01') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@anaylyze_external +# col_name data_type comment + +key string +val string + +# Partition Information +# col_name data_type comment + +insertdate string + +# Detailed Partition Information +Partition Value: [2008-01-01] +Database: default +Table: anaylyze_external +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +#### A masked pattern was here #### +PREHOOK: query: drop table anaylyze_external +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@anaylyze_external +PREHOOK: Output: default@anaylyze_external +POSTHOOK: query: drop table anaylyze_external +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@anaylyze_external +POSTHOOK: Output: default@anaylyze_external diff --git ql/src/test/results/clientpositive/spark/stats_only_null.q.out ql/src/test/results/clientpositive/spark/stats_only_null.q.out new file mode 100644 index 0000000..1fd672e --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats_only_null.q.out @@ -0,0 +1,430 @@ +PREHOOK: query: CREATE TABLE temps_null(a double, b int, c STRING, d smallint) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@temps_null +POSTHOOK: query: CREATE TABLE temps_null(a double, b int, c STRING, d smallint) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@temps_null +PREHOOK: query: CREATE TABLE stats_null(a double, b int, c STRING, d smallint) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_null +POSTHOOK: query: CREATE TABLE stats_null(a double, b int, c STRING, d smallint) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_null +PREHOOK: query: CREATE TABLE stats_null_part(a double, b int, c STRING, d smallint) partitioned by (dt string) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_null_part +POSTHOOK: query: CREATE TABLE stats_null_part(a double, b int, c STRING, d smallint) partitioned by (dt string) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_null_part +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/null.txt' INTO TABLE temps_null +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@temps_null +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/null.txt' INTO TABLE temps_null +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@temps_null +PREHOOK: query: insert overwrite table stats_null select * from temps_null +PREHOOK: type: QUERY +PREHOOK: Input: default@temps_null +PREHOOK: Output: default@stats_null +POSTHOOK: query: insert overwrite table stats_null select * from temps_null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@temps_null +POSTHOOK: Output: default@stats_null +POSTHOOK: Lineage: stats_null.a SIMPLE [(temps_null)temps_null.FieldSchema(name:a, type:double, comment:null), ] +POSTHOOK: Lineage: stats_null.b SIMPLE [(temps_null)temps_null.FieldSchema(name:b, type:int, comment:null), ] +POSTHOOK: Lineage: stats_null.c SIMPLE [(temps_null)temps_null.FieldSchema(name:c, type:string, comment:null), ] +POSTHOOK: Lineage: stats_null.d SIMPLE [(temps_null)temps_null.FieldSchema(name:d, type:smallint, comment:null), ] +PREHOOK: query: insert into table stats_null_part partition(dt='2010') select * from temps_null where d <=5 +PREHOOK: type: QUERY +PREHOOK: Input: default@temps_null +PREHOOK: Output: default@stats_null_part@dt=2010 +POSTHOOK: query: insert into table stats_null_part partition(dt='2010') select * from temps_null where d <=5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@temps_null +POSTHOOK: Output: default@stats_null_part@dt=2010 +POSTHOOK: Lineage: stats_null_part PARTITION(dt=2010).a SIMPLE [(temps_null)temps_null.FieldSchema(name:a, type:double, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=2010).b SIMPLE [(temps_null)temps_null.FieldSchema(name:b, type:int, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=2010).c SIMPLE [(temps_null)temps_null.FieldSchema(name:c, type:string, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=2010).d SIMPLE [(temps_null)temps_null.FieldSchema(name:d, type:smallint, comment:null), ] +PREHOOK: query: insert into table stats_null_part partition(dt='2011') select * from temps_null where d > 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@temps_null +PREHOOK: Output: default@stats_null_part@dt=2011 +POSTHOOK: query: insert into table stats_null_part partition(dt='2011') select * from temps_null where d > 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@temps_null +POSTHOOK: Output: default@stats_null_part@dt=2011 +POSTHOOK: Lineage: stats_null_part PARTITION(dt=2011).a SIMPLE [(temps_null)temps_null.FieldSchema(name:a, type:double, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=2011).b SIMPLE [(temps_null)temps_null.FieldSchema(name:b, type:int, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=2011).c SIMPLE [(temps_null)temps_null.FieldSchema(name:c, type:string, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=2011).d SIMPLE [(temps_null)temps_null.FieldSchema(name:d, type:smallint, comment:null), ] +PREHOOK: query: explain +select count(*), count(a), count(b), count(c), count(d) from stats_null +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*), count(a), count(b), count(c), count(d) from stats_null +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: stats_null + Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: double), b (type: int), c (type: string), d (type: smallint) + outputColumnNames: a, b, c, d + Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(a), count(b), count(c), count(d) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1), count(VALUE._col2), count(VALUE._col3), count(VALUE._col4) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select count(*), count(a), count(b), count(c), count(d) from stats_null_part +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*), count(a), count(b), count(c), count(d) from stats_null_part +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: stats_null_part + Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: double), b (type: int), c (type: string), d (type: smallint) + outputColumnNames: a, b, c, d + Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(a), count(b), count(c), count(d) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1), count(VALUE._col2), count(VALUE._col3), count(VALUE._col4) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_null +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_null +#### A masked pattern was here #### +PREHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_null_part +PREHOOK: Input: default@stats_null_part@dt=2010 +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_null_part +POSTHOOK: Input: default@stats_null_part@dt=2010 +#### A masked pattern was here #### +PREHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_null_part +PREHOOK: Input: default@stats_null_part@dt=2011 +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_null_part +POSTHOOK: Input: default@stats_null_part@dt=2011 +#### A masked pattern was here #### +PREHOOK: query: describe formatted stats_null_part partition (dt='2010') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_null_part +POSTHOOK: query: describe formatted stats_null_part partition (dt='2010') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_null_part +# col_name data_type comment + +a double +b int +c string +d smallint + +# Partition Information +# col_name data_type comment + +dt string + +# Detailed Partition Information +Partition Value: [2010] +Database: default +Table: stats_null_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 6 + rawDataSize 71 + totalSize 77 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted stats_null_part partition (dt='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_null_part +POSTHOOK: query: describe formatted stats_null_part partition (dt='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_null_part +# col_name data_type comment + +a double +b int +c string +d smallint + +# Partition Information +# col_name data_type comment + +dt string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: stats_null_part +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 4 + rawDataSize 49 + totalSize 53 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain +select count(*), count(a), count(b), count(c), count(d) from stats_null +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*), count(a), count(b), count(c), count(d) from stats_null +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select count(*), count(a), count(b), count(c), count(d) from stats_null_part +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*), count(a), count(b), count(c), count(d) from stats_null_part +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*), count(a), count(b), count(c), count(d) from stats_null +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_null +#### A masked pattern was here #### +POSTHOOK: query: select count(*), count(a), count(b), count(c), count(d) from stats_null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_null +#### A masked pattern was here #### +10 8 8 10 10 +PREHOOK: query: select count(*), count(a), count(b), count(c), count(d) from stats_null_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_null_part +#### A masked pattern was here #### +POSTHOOK: query: select count(*), count(a), count(b), count(c), count(d) from stats_null_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_null_part +#### A masked pattern was here #### +10 8 8 10 10 +PREHOOK: query: drop table stats_null_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_null_part +PREHOOK: Output: default@stats_null_part +POSTHOOK: query: drop table stats_null_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_null_part +POSTHOOK: Output: default@stats_null_part +PREHOOK: query: CREATE TABLE stats_null_part(a double, b int, c STRING, d smallint) partitioned by (dt int) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_null_part +POSTHOOK: query: CREATE TABLE stats_null_part(a double, b int, c STRING, d smallint) partitioned by (dt int) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_null_part +PREHOOK: query: insert into table stats_null_part partition(dt) select a,b,c,d,b from temps_null +PREHOOK: type: QUERY +PREHOOK: Input: default@temps_null +PREHOOK: Output: default@stats_null_part +POSTHOOK: query: insert into table stats_null_part partition(dt) select a,b,c,d,b from temps_null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@temps_null +POSTHOOK: Output: default@stats_null_part@dt=1 +POSTHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: stats_null_part PARTITION(dt=1).a SIMPLE [(temps_null)temps_null.FieldSchema(name:a, type:double, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=1).b SIMPLE [(temps_null)temps_null.FieldSchema(name:b, type:int, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=1).c SIMPLE [(temps_null)temps_null.FieldSchema(name:c, type:string, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=1).d SIMPLE [(temps_null)temps_null.FieldSchema(name:d, type:smallint, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=__HIVE_DEFAULT_PARTITION__).a SIMPLE [(temps_null)temps_null.FieldSchema(name:a, type:double, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(temps_null)temps_null.FieldSchema(name:b, type:int, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=__HIVE_DEFAULT_PARTITION__).c SIMPLE [(temps_null)temps_null.FieldSchema(name:c, type:string, comment:null), ] +POSTHOOK: Lineage: stats_null_part PARTITION(dt=__HIVE_DEFAULT_PARTITION__).d SIMPLE [(temps_null)temps_null.FieldSchema(name:d, type:smallint, comment:null), ] +PREHOOK: query: analyze table stats_null_part compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_null_part +PREHOOK: Input: default@stats_null_part@dt=1 +PREHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats_null_part compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_null_part +POSTHOOK: Input: default@stats_null_part@dt=1 +POSTHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +PREHOOK: query: describe formatted stats_null_part.a partition(dt = 1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats_null_part +POSTHOOK: query: describe formatted stats_null_part.a partition(dt = 1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats_null_part +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +a double 1.0 1.0 1 1 from deserializer +PREHOOK: query: drop table stats_null +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_null +PREHOOK: Output: default@stats_null +POSTHOOK: query: drop table stats_null +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_null +POSTHOOK: Output: default@stats_null +PREHOOK: query: drop table stats_null_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats_null_part +PREHOOK: Output: default@stats_null_part +POSTHOOK: query: drop table stats_null_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats_null_part +POSTHOOK: Output: default@stats_null_part +PREHOOK: query: drop table temps_null +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@temps_null +PREHOOK: Output: default@temps_null +POSTHOOK: query: drop table temps_null +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@temps_null +POSTHOOK: Output: default@temps_null diff --git ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out new file mode 100644 index 0000000..ccd90b2 --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out @@ -0,0 +1,222 @@ +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) +-- This test uses mapred.max.split.size/mapred.max.split.size for controlling +-- number of input splits. +-- stats_partscan_1.q is the same test with this but has different result. + +-- test analyze table ... compute statistics partialscan + +-- 1. prepare data +CREATE table analyze_srcpart_partial_scan (key STRING, value STRING) +partitioned by (ds string, hr string) +stored as rcfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) +-- This test uses mapred.max.split.size/mapred.max.split.size for controlling +-- number of input splits. +-- stats_partscan_1.q is the same test with this but has different result. + +-- test analyze table ... compute statistics partialscan + +-- 1. prepare data +CREATE table analyze_srcpart_partial_scan (key STRING, value STRING) +partitioned by (ds string, hr string) +stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart_partial_scan +PREHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: -- 2. partialscan +explain +analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +PREHOOK: type: QUERY +POSTHOOK: query: -- 2. partialscan +explain +analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-2 depends on stages: Stage-0, Stage-3 + +STAGE PLANS: + Stage: Stage-3 + Partial Scan Statistics + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart_partial_scan +PREHOOK: Input: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart_partial_scan +PREHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: Input: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +PREHOOK: query: -- 3. confirm result +describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: -- 3. confirm result +describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE false + numFiles 1 + numRows -1 + rawDataSize -1 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table analyze_srcpart_partial_scan +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: drop table analyze_srcpart_partial_scan +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: Output: default@analyze_srcpart_partial_scan diff --git ql/src/test/results/clientpositive/spark/statsfs.q.out ql/src/test/results/clientpositive/spark/statsfs.q.out new file mode 100644 index 0000000..b0bca41 --- /dev/null +++ ql/src/test/results/clientpositive/spark/statsfs.q.out @@ -0,0 +1,529 @@ +PREHOOK: query: -- stats computation on partitioned table with analyze command + +create table t1 (key string, value string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: -- stats computation on partitioned table with analyze command + +create table t1 (key string, value string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table t1 partition (ds = '2010') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table t1 partition (ds = '2010') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +POSTHOOK: Output: default@t1@ds=2010 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table t1 partition (ds = '2011') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table t1 partition (ds = '2011') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +POSTHOOK: Output: default@t1@ds=2011 +PREHOOK: query: analyze table t1 partition (ds) compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t1@ds=2010 +PREHOOK: Input: default@t1@ds=2011 +PREHOOK: Output: default@t1 +PREHOOK: Output: default@t1@ds=2010 +PREHOOK: Output: default@t1@ds=2011 +POSTHOOK: query: analyze table t1 partition (ds) compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t1@ds=2010 +POSTHOOK: Input: default@t1@ds=2011 +POSTHOOK: Output: default@t1 +POSTHOOK: Output: default@t1@ds=2010 +POSTHOOK: Output: default@t1@ds=2011 +PREHOOK: query: describe formatted t1 partition (ds='2010') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t1 +POSTHOOK: query: describe formatted t1 partition (ds='2010') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t1 +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2010] +Database: default +Table: t1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted t1 partition (ds='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t1 +POSTHOOK: query: describe formatted t1 partition (ds='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t1 +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: t1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: drop table t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: -- stats computation on partitioned table with autogather on insert query + +create table t1 (key string, value string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: -- stats computation on partitioned table with autogather on insert query + +create table t1 (key string, value string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: insert into table t1 partition (ds='2010') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1@ds=2010 +POSTHOOK: query: insert into table t1 partition (ds='2010') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1@ds=2010 +POSTHOOK: Lineage: t1 PARTITION(ds=2010).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2010).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table t1 partition (ds='2011') select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1@ds=2011 +POSTHOOK: query: insert into table t1 partition (ds='2011') select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1@ds=2011 +POSTHOOK: Lineage: t1 PARTITION(ds=2011).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted t1 partition (ds='2010') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t1 +POSTHOOK: query: describe formatted t1 partition (ds='2010') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t1 +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2010] +Database: default +Table: t1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted t1 partition (ds='2011') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t1 +POSTHOOK: query: describe formatted t1 partition (ds='2011') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t1 +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string + +# Detailed Partition Information +Partition Value: [2011] +Database: default +Table: t1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: drop table t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: -- analyze stmt on unpartitioned table + +create table t1 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: -- analyze stmt on unpartitioned table + +create table t1 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table t1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table t1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +PREHOOK: query: analyze table t1 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: analyze table t1 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: describe formatted t1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t1 +POSTHOOK: query: describe formatted t1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t1 +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: drop table t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: -- stats computation on unpartitioned table with autogather on insert query + +create table t1 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: -- stats computation on unpartitioned table with autogather on insert query + +create table t1 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: insert into table t1 select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1 +POSTHOOK: query: insert into table t1 select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1 +POSTHOOK: Lineage: t1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted t1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t1 +POSTHOOK: query: describe formatted t1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t1 +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: drop table t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: -- stats computation on partitioned table with autogather on insert query with dynamic partitioning + + +create table t1 (key string, value string) partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: -- stats computation on partitioned table with autogather on insert query with dynamic partitioning + + +create table t1 (key string, value string) partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: insert into table t1 partition (ds,hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@t1 +POSTHOOK: query: insert into table t1 partition (ds,hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@t1@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@t1@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@t1@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@t1@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: t1 PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1 PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted t1 partition (ds='2008-04-08',hr='11') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t1 +POSTHOOK: query: describe formatted t1 partition (ds='2008-04-08',hr='11') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t1 +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: t1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted t1 partition (ds='2008-04-09',hr='12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t1 +POSTHOOK: query: describe formatted t1 partition (ds='2008-04-09',hr='12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t1 +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 12] +Database: default +Table: t1 +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: drop table t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 -- 1.8.5.2 (Apple Git-48)