diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index fcedcfe..b71855f 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -312,6 +312,14 @@ minimr.query.negative.files=cluster_tasklog_retrieval.q,\ spark.query.files=spark_test.q \ avro_compression_enabled_native.q \ avro_decimal_native.q \ + sample1.q \ + sample2.q \ + sample3.q \ + sample4.q \ + sample5.q \ + sample6.q \ + sample7.q \ + sample9.q \ timestamp_1.q \ timestamp_2.q \ timestamp_3.q \ diff --git ql/src/test/results/clientpositive/spark/sample1.q.out ql/src/test/results/clientpositive/spark/sample1.q.out new file mode 100644 index 0000000..76138cb --- /dev/null +++ ql/src/test/results/clientpositive/spark/sample1.q.out @@ -0,0 +1,727 @@ +PREHOOK: query: CREATE TABLE dest1(key INT, value STRING, dt STRING, hr STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1 +POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING, dt STRING, hr STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1 +PREHOOK: query: -- no input pruning, no sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1 ON rand()) s +WHERE s.ds='2008-04-08' and s.hr='11' +PREHOOK: type: QUERY +POSTHOOK: query: -- no input pruning, no sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1 ON rand()) s +WHERE s.ds='2008-04-08' and s.hr='11' +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_TABLEBUCKETSAMPLE + 1 + 1 + TOK_FUNCTION + rand + s + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest1 + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_WHERE + and + = + . + TOK_TABLE_OR_COL + s + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + s + hr + '11' + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(rand()) & 2147483647) % 1) = 0) (type: boolean) + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,dt,hr + columns.comments + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string dt, string hr} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [s] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,dt,hr + columns.comments + columns.types int:string:string:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value, string dt, string hr} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1 ON rand()) s +WHERE s.ds='2008-04-08' and s.hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@dest1 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1 ON rand()) s +WHERE s.ds='2008-04-08' and s.hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@dest1 +POSTHOOK: Lineage: dest1.dt SIMPLE [(srcpart)s.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: dest1.hr SIMPLE [(srcpart)s.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)s.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.value SIMPLE [(srcpart)s.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +238 val_238 2008-04-08 11 +86 val_86 2008-04-08 11 +311 val_311 2008-04-08 11 +27 val_27 2008-04-08 11 +165 val_165 2008-04-08 11 +409 val_409 2008-04-08 11 +255 val_255 2008-04-08 11 +278 val_278 2008-04-08 11 +98 val_98 2008-04-08 11 +484 val_484 2008-04-08 11 +265 val_265 2008-04-08 11 +193 val_193 2008-04-08 11 +401 val_401 2008-04-08 11 +150 val_150 2008-04-08 11 +273 val_273 2008-04-08 11 +224 val_224 2008-04-08 11 +369 val_369 2008-04-08 11 +66 val_66 2008-04-08 11 +128 val_128 2008-04-08 11 +213 val_213 2008-04-08 11 +146 val_146 2008-04-08 11 +406 val_406 2008-04-08 11 +429 val_429 2008-04-08 11 +374 val_374 2008-04-08 11 +152 val_152 2008-04-08 11 +469 val_469 2008-04-08 11 +145 val_145 2008-04-08 11 +495 val_495 2008-04-08 11 +37 val_37 2008-04-08 11 +327 val_327 2008-04-08 11 +281 val_281 2008-04-08 11 +277 val_277 2008-04-08 11 +209 val_209 2008-04-08 11 +15 val_15 2008-04-08 11 +82 val_82 2008-04-08 11 +403 val_403 2008-04-08 11 +166 val_166 2008-04-08 11 +417 val_417 2008-04-08 11 +430 val_430 2008-04-08 11 +252 val_252 2008-04-08 11 +292 val_292 2008-04-08 11 +219 val_219 2008-04-08 11 +287 val_287 2008-04-08 11 +153 val_153 2008-04-08 11 +193 val_193 2008-04-08 11 +338 val_338 2008-04-08 11 +446 val_446 2008-04-08 11 +459 val_459 2008-04-08 11 +394 val_394 2008-04-08 11 +237 val_237 2008-04-08 11 +482 val_482 2008-04-08 11 +174 val_174 2008-04-08 11 +413 val_413 2008-04-08 11 +494 val_494 2008-04-08 11 +207 val_207 2008-04-08 11 +199 val_199 2008-04-08 11 +466 val_466 2008-04-08 11 +208 val_208 2008-04-08 11 +174 val_174 2008-04-08 11 +399 val_399 2008-04-08 11 +396 val_396 2008-04-08 11 +247 val_247 2008-04-08 11 +417 val_417 2008-04-08 11 +489 val_489 2008-04-08 11 +162 val_162 2008-04-08 11 +377 val_377 2008-04-08 11 +397 val_397 2008-04-08 11 +309 val_309 2008-04-08 11 +365 val_365 2008-04-08 11 +266 val_266 2008-04-08 11 +439 val_439 2008-04-08 11 +342 val_342 2008-04-08 11 +367 val_367 2008-04-08 11 +325 val_325 2008-04-08 11 +167 val_167 2008-04-08 11 +195 val_195 2008-04-08 11 +475 val_475 2008-04-08 11 +17 val_17 2008-04-08 11 +113 val_113 2008-04-08 11 +155 val_155 2008-04-08 11 +203 val_203 2008-04-08 11 +339 val_339 2008-04-08 11 +0 val_0 2008-04-08 11 +455 val_455 2008-04-08 11 +128 val_128 2008-04-08 11 +311 val_311 2008-04-08 11 +316 val_316 2008-04-08 11 +57 val_57 2008-04-08 11 +302 val_302 2008-04-08 11 +205 val_205 2008-04-08 11 +149 val_149 2008-04-08 11 +438 val_438 2008-04-08 11 +345 val_345 2008-04-08 11 +129 val_129 2008-04-08 11 +170 val_170 2008-04-08 11 +20 val_20 2008-04-08 11 +489 val_489 2008-04-08 11 +157 val_157 2008-04-08 11 +378 val_378 2008-04-08 11 +221 val_221 2008-04-08 11 +92 val_92 2008-04-08 11 +111 val_111 2008-04-08 11 +47 val_47 2008-04-08 11 +72 val_72 2008-04-08 11 +4 val_4 2008-04-08 11 +280 val_280 2008-04-08 11 +35 val_35 2008-04-08 11 +427 val_427 2008-04-08 11 +277 val_277 2008-04-08 11 +208 val_208 2008-04-08 11 +356 val_356 2008-04-08 11 +399 val_399 2008-04-08 11 +169 val_169 2008-04-08 11 +382 val_382 2008-04-08 11 +498 val_498 2008-04-08 11 +125 val_125 2008-04-08 11 +386 val_386 2008-04-08 11 +437 val_437 2008-04-08 11 +469 val_469 2008-04-08 11 +192 val_192 2008-04-08 11 +286 val_286 2008-04-08 11 +187 val_187 2008-04-08 11 +176 val_176 2008-04-08 11 +54 val_54 2008-04-08 11 +459 val_459 2008-04-08 11 +51 val_51 2008-04-08 11 +138 val_138 2008-04-08 11 +103 val_103 2008-04-08 11 +239 val_239 2008-04-08 11 +213 val_213 2008-04-08 11 +216 val_216 2008-04-08 11 +430 val_430 2008-04-08 11 +278 val_278 2008-04-08 11 +176 val_176 2008-04-08 11 +289 val_289 2008-04-08 11 +221 val_221 2008-04-08 11 +65 val_65 2008-04-08 11 +318 val_318 2008-04-08 11 +332 val_332 2008-04-08 11 +311 val_311 2008-04-08 11 +275 val_275 2008-04-08 11 +137 val_137 2008-04-08 11 +241 val_241 2008-04-08 11 +83 val_83 2008-04-08 11 +333 val_333 2008-04-08 11 +180 val_180 2008-04-08 11 +284 val_284 2008-04-08 11 +12 val_12 2008-04-08 11 +230 val_230 2008-04-08 11 +181 val_181 2008-04-08 11 +67 val_67 2008-04-08 11 +260 val_260 2008-04-08 11 +404 val_404 2008-04-08 11 +384 val_384 2008-04-08 11 +489 val_489 2008-04-08 11 +353 val_353 2008-04-08 11 +373 val_373 2008-04-08 11 +272 val_272 2008-04-08 11 +138 val_138 2008-04-08 11 +217 val_217 2008-04-08 11 +84 val_84 2008-04-08 11 +348 val_348 2008-04-08 11 +466 val_466 2008-04-08 11 +58 val_58 2008-04-08 11 +8 val_8 2008-04-08 11 +411 val_411 2008-04-08 11 +230 val_230 2008-04-08 11 +208 val_208 2008-04-08 11 +348 val_348 2008-04-08 11 +24 val_24 2008-04-08 11 +463 val_463 2008-04-08 11 +431 val_431 2008-04-08 11 +179 val_179 2008-04-08 11 +172 val_172 2008-04-08 11 +42 val_42 2008-04-08 11 +129 val_129 2008-04-08 11 +158 val_158 2008-04-08 11 +119 val_119 2008-04-08 11 +496 val_496 2008-04-08 11 +0 val_0 2008-04-08 11 +322 val_322 2008-04-08 11 +197 val_197 2008-04-08 11 +468 val_468 2008-04-08 11 +393 val_393 2008-04-08 11 +454 val_454 2008-04-08 11 +100 val_100 2008-04-08 11 +298 val_298 2008-04-08 11 +199 val_199 2008-04-08 11 +191 val_191 2008-04-08 11 +418 val_418 2008-04-08 11 +96 val_96 2008-04-08 11 +26 val_26 2008-04-08 11 +165 val_165 2008-04-08 11 +327 val_327 2008-04-08 11 +230 val_230 2008-04-08 11 +205 val_205 2008-04-08 11 +120 val_120 2008-04-08 11 +131 val_131 2008-04-08 11 +51 val_51 2008-04-08 11 +404 val_404 2008-04-08 11 +43 val_43 2008-04-08 11 +436 val_436 2008-04-08 11 +156 val_156 2008-04-08 11 +469 val_469 2008-04-08 11 +468 val_468 2008-04-08 11 +308 val_308 2008-04-08 11 +95 val_95 2008-04-08 11 +196 val_196 2008-04-08 11 +288 val_288 2008-04-08 11 +481 val_481 2008-04-08 11 +457 val_457 2008-04-08 11 +98 val_98 2008-04-08 11 +282 val_282 2008-04-08 11 +197 val_197 2008-04-08 11 +187 val_187 2008-04-08 11 +318 val_318 2008-04-08 11 +318 val_318 2008-04-08 11 +409 val_409 2008-04-08 11 +470 val_470 2008-04-08 11 +137 val_137 2008-04-08 11 +369 val_369 2008-04-08 11 +316 val_316 2008-04-08 11 +169 val_169 2008-04-08 11 +413 val_413 2008-04-08 11 +85 val_85 2008-04-08 11 +77 val_77 2008-04-08 11 +0 val_0 2008-04-08 11 +490 val_490 2008-04-08 11 +87 val_87 2008-04-08 11 +364 val_364 2008-04-08 11 +179 val_179 2008-04-08 11 +118 val_118 2008-04-08 11 +134 val_134 2008-04-08 11 +395 val_395 2008-04-08 11 +282 val_282 2008-04-08 11 +138 val_138 2008-04-08 11 +238 val_238 2008-04-08 11 +419 val_419 2008-04-08 11 +15 val_15 2008-04-08 11 +118 val_118 2008-04-08 11 +72 val_72 2008-04-08 11 +90 val_90 2008-04-08 11 +307 val_307 2008-04-08 11 +19 val_19 2008-04-08 11 +435 val_435 2008-04-08 11 +10 val_10 2008-04-08 11 +277 val_277 2008-04-08 11 +273 val_273 2008-04-08 11 +306 val_306 2008-04-08 11 +224 val_224 2008-04-08 11 +309 val_309 2008-04-08 11 +389 val_389 2008-04-08 11 +327 val_327 2008-04-08 11 +242 val_242 2008-04-08 11 +369 val_369 2008-04-08 11 +392 val_392 2008-04-08 11 +272 val_272 2008-04-08 11 +331 val_331 2008-04-08 11 +401 val_401 2008-04-08 11 +242 val_242 2008-04-08 11 +452 val_452 2008-04-08 11 +177 val_177 2008-04-08 11 +226 val_226 2008-04-08 11 +5 val_5 2008-04-08 11 +497 val_497 2008-04-08 11 +402 val_402 2008-04-08 11 +396 val_396 2008-04-08 11 +317 val_317 2008-04-08 11 +395 val_395 2008-04-08 11 +58 val_58 2008-04-08 11 +35 val_35 2008-04-08 11 +336 val_336 2008-04-08 11 +95 val_95 2008-04-08 11 +11 val_11 2008-04-08 11 +168 val_168 2008-04-08 11 +34 val_34 2008-04-08 11 +229 val_229 2008-04-08 11 +233 val_233 2008-04-08 11 +143 val_143 2008-04-08 11 +472 val_472 2008-04-08 11 +322 val_322 2008-04-08 11 +498 val_498 2008-04-08 11 +160 val_160 2008-04-08 11 +195 val_195 2008-04-08 11 +42 val_42 2008-04-08 11 +321 val_321 2008-04-08 11 +430 val_430 2008-04-08 11 +119 val_119 2008-04-08 11 +489 val_489 2008-04-08 11 +458 val_458 2008-04-08 11 +78 val_78 2008-04-08 11 +76 val_76 2008-04-08 11 +41 val_41 2008-04-08 11 +223 val_223 2008-04-08 11 +492 val_492 2008-04-08 11 +149 val_149 2008-04-08 11 +449 val_449 2008-04-08 11 +218 val_218 2008-04-08 11 +228 val_228 2008-04-08 11 +138 val_138 2008-04-08 11 +453 val_453 2008-04-08 11 +30 val_30 2008-04-08 11 +209 val_209 2008-04-08 11 +64 val_64 2008-04-08 11 +468 val_468 2008-04-08 11 +76 val_76 2008-04-08 11 +74 val_74 2008-04-08 11 +342 val_342 2008-04-08 11 +69 val_69 2008-04-08 11 +230 val_230 2008-04-08 11 +33 val_33 2008-04-08 11 +368 val_368 2008-04-08 11 +103 val_103 2008-04-08 11 +296 val_296 2008-04-08 11 +113 val_113 2008-04-08 11 +216 val_216 2008-04-08 11 +367 val_367 2008-04-08 11 +344 val_344 2008-04-08 11 +167 val_167 2008-04-08 11 +274 val_274 2008-04-08 11 +219 val_219 2008-04-08 11 +239 val_239 2008-04-08 11 +485 val_485 2008-04-08 11 +116 val_116 2008-04-08 11 +223 val_223 2008-04-08 11 +256 val_256 2008-04-08 11 +263 val_263 2008-04-08 11 +70 val_70 2008-04-08 11 +487 val_487 2008-04-08 11 +480 val_480 2008-04-08 11 +401 val_401 2008-04-08 11 +288 val_288 2008-04-08 11 +191 val_191 2008-04-08 11 +5 val_5 2008-04-08 11 +244 val_244 2008-04-08 11 +438 val_438 2008-04-08 11 +128 val_128 2008-04-08 11 +467 val_467 2008-04-08 11 +432 val_432 2008-04-08 11 +202 val_202 2008-04-08 11 +316 val_316 2008-04-08 11 +229 val_229 2008-04-08 11 +469 val_469 2008-04-08 11 +463 val_463 2008-04-08 11 +280 val_280 2008-04-08 11 +2 val_2 2008-04-08 11 +35 val_35 2008-04-08 11 +283 val_283 2008-04-08 11 +331 val_331 2008-04-08 11 +235 val_235 2008-04-08 11 +80 val_80 2008-04-08 11 +44 val_44 2008-04-08 11 +193 val_193 2008-04-08 11 +321 val_321 2008-04-08 11 +335 val_335 2008-04-08 11 +104 val_104 2008-04-08 11 +466 val_466 2008-04-08 11 +366 val_366 2008-04-08 11 +175 val_175 2008-04-08 11 +403 val_403 2008-04-08 11 +483 val_483 2008-04-08 11 +53 val_53 2008-04-08 11 +105 val_105 2008-04-08 11 +257 val_257 2008-04-08 11 +406 val_406 2008-04-08 11 +409 val_409 2008-04-08 11 +190 val_190 2008-04-08 11 +406 val_406 2008-04-08 11 +401 val_401 2008-04-08 11 +114 val_114 2008-04-08 11 +258 val_258 2008-04-08 11 +90 val_90 2008-04-08 11 +203 val_203 2008-04-08 11 +262 val_262 2008-04-08 11 +348 val_348 2008-04-08 11 +424 val_424 2008-04-08 11 +12 val_12 2008-04-08 11 +396 val_396 2008-04-08 11 +201 val_201 2008-04-08 11 +217 val_217 2008-04-08 11 +164 val_164 2008-04-08 11 +431 val_431 2008-04-08 11 +454 val_454 2008-04-08 11 +478 val_478 2008-04-08 11 +298 val_298 2008-04-08 11 +125 val_125 2008-04-08 11 +431 val_431 2008-04-08 11 +164 val_164 2008-04-08 11 +424 val_424 2008-04-08 11 +187 val_187 2008-04-08 11 +382 val_382 2008-04-08 11 +5 val_5 2008-04-08 11 +70 val_70 2008-04-08 11 +397 val_397 2008-04-08 11 +480 val_480 2008-04-08 11 +291 val_291 2008-04-08 11 +24 val_24 2008-04-08 11 +351 val_351 2008-04-08 11 +255 val_255 2008-04-08 11 +104 val_104 2008-04-08 11 +70 val_70 2008-04-08 11 +163 val_163 2008-04-08 11 +438 val_438 2008-04-08 11 +119 val_119 2008-04-08 11 +414 val_414 2008-04-08 11 +200 val_200 2008-04-08 11 +491 val_491 2008-04-08 11 +237 val_237 2008-04-08 11 +439 val_439 2008-04-08 11 +360 val_360 2008-04-08 11 +248 val_248 2008-04-08 11 +479 val_479 2008-04-08 11 +305 val_305 2008-04-08 11 +417 val_417 2008-04-08 11 +199 val_199 2008-04-08 11 +444 val_444 2008-04-08 11 +120 val_120 2008-04-08 11 +429 val_429 2008-04-08 11 +169 val_169 2008-04-08 11 +443 val_443 2008-04-08 11 +323 val_323 2008-04-08 11 +325 val_325 2008-04-08 11 +277 val_277 2008-04-08 11 +230 val_230 2008-04-08 11 +478 val_478 2008-04-08 11 +178 val_178 2008-04-08 11 +468 val_468 2008-04-08 11 +310 val_310 2008-04-08 11 +317 val_317 2008-04-08 11 +333 val_333 2008-04-08 11 +493 val_493 2008-04-08 11 +460 val_460 2008-04-08 11 +207 val_207 2008-04-08 11 +249 val_249 2008-04-08 11 +265 val_265 2008-04-08 11 +480 val_480 2008-04-08 11 +83 val_83 2008-04-08 11 +136 val_136 2008-04-08 11 +353 val_353 2008-04-08 11 +172 val_172 2008-04-08 11 +214 val_214 2008-04-08 11 +462 val_462 2008-04-08 11 +233 val_233 2008-04-08 11 +406 val_406 2008-04-08 11 +133 val_133 2008-04-08 11 +175 val_175 2008-04-08 11 +189 val_189 2008-04-08 11 +454 val_454 2008-04-08 11 +375 val_375 2008-04-08 11 +401 val_401 2008-04-08 11 +421 val_421 2008-04-08 11 +407 val_407 2008-04-08 11 +384 val_384 2008-04-08 11 +256 val_256 2008-04-08 11 +26 val_26 2008-04-08 11 +134 val_134 2008-04-08 11 +67 val_67 2008-04-08 11 +384 val_384 2008-04-08 11 +379 val_379 2008-04-08 11 +18 val_18 2008-04-08 11 +462 val_462 2008-04-08 11 +492 val_492 2008-04-08 11 +100 val_100 2008-04-08 11 +298 val_298 2008-04-08 11 +9 val_9 2008-04-08 11 +341 val_341 2008-04-08 11 +498 val_498 2008-04-08 11 +146 val_146 2008-04-08 11 +458 val_458 2008-04-08 11 +362 val_362 2008-04-08 11 +186 val_186 2008-04-08 11 +285 val_285 2008-04-08 11 +348 val_348 2008-04-08 11 +167 val_167 2008-04-08 11 +18 val_18 2008-04-08 11 +273 val_273 2008-04-08 11 +183 val_183 2008-04-08 11 +281 val_281 2008-04-08 11 +344 val_344 2008-04-08 11 +97 val_97 2008-04-08 11 +469 val_469 2008-04-08 11 +315 val_315 2008-04-08 11 +84 val_84 2008-04-08 11 +28 val_28 2008-04-08 11 +37 val_37 2008-04-08 11 +448 val_448 2008-04-08 11 +152 val_152 2008-04-08 11 +348 val_348 2008-04-08 11 +307 val_307 2008-04-08 11 +194 val_194 2008-04-08 11 +414 val_414 2008-04-08 11 +477 val_477 2008-04-08 11 +222 val_222 2008-04-08 11 +126 val_126 2008-04-08 11 +90 val_90 2008-04-08 11 +169 val_169 2008-04-08 11 +403 val_403 2008-04-08 11 +400 val_400 2008-04-08 11 +200 val_200 2008-04-08 11 +97 val_97 2008-04-08 11 +PREHOOK: query: select count(1) from srcbucket +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from srcbucket +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +#### A masked pattern was here #### +1000 diff --git ql/src/test/results/clientpositive/spark/sample2.q.out ql/src/test/results/clientpositive/spark/sample2.q.out new file mode 100644 index 0000000..85b47e3 --- /dev/null +++ ql/src/test/results/clientpositive/spark/sample2.q.out @@ -0,0 +1,689 @@ +PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1 +POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1 +PREHOOK: query: -- input pruning, no sample filter +-- default table sample columns +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2) s +PREHOOK: type: QUERY +POSTHOOK: query: -- input pruning, no sample filter +-- default table sample columns +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2) s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 1 + 2 + s + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest1 + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket0.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket/srcbucket0.txt [s] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2) s +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@dest1 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@dest1 +POSTHOOK: Lineage: dest1.key SIMPLE [(srcbucket)s.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: dest1.value SIMPLE [(srcbucket)s.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +474 val_475 +62 val_63 +468 val_469 +272 val_273 +448 val_449 +246 val_247 +440 val_441 +278 val_279 +296 val_297 +428 val_429 +126 val_127 +106 val_107 +356 val_357 +490 val_491 +402 val_403 +128 val_129 +10 val_11 +226 val_227 +110 val_111 +0 val_1 +240 val_241 +286 val_287 +408 val_409 +476 val_477 +482 val_483 +48 val_49 +424 val_425 +226 val_227 +494 val_495 +488 val_489 +94 val_95 +50 val_51 +402 val_403 +128 val_129 +468 val_469 +314 val_315 +224 val_225 +344 val_345 +4 val_5 +206 val_207 +114 val_115 +56 val_57 +114 val_115 +254 val_255 +390 val_391 +304 val_305 +264 val_265 +196 val_197 +238 val_239 +20 val_21 +492 val_493 +82 val_83 +58 val_59 +86 val_87 +438 val_439 +360 val_361 +222 val_223 +42 val_43 +338 val_339 +68 val_69 +16 val_17 +492 val_493 +376 val_377 +120 val_121 +306 val_307 +426 val_427 +132 val_133 +446 val_447 +386 val_387 +388 val_389 +184 val_185 +284 val_285 +246 val_247 +262 val_263 +122 val_123 +438 val_439 +390 val_391 +352 val_353 +226 val_227 +328 val_329 +382 val_383 +342 val_343 +480 val_481 +102 val_103 +480 val_481 +318 val_319 +392 val_393 +476 val_477 +258 val_259 +174 val_175 +252 val_253 +114 val_115 +264 val_265 +48 val_49 +336 val_337 +340 val_341 +390 val_391 +484 val_485 +6 val_7 +260 val_261 +2 val_3 +170 val_171 +164 val_165 +118 val_119 +310 val_311 +104 val_105 +80 val_81 +326 val_327 +450 val_451 +140 val_141 +212 val_213 +308 val_309 +30 val_31 +358 val_359 +416 val_417 +42 val_43 +386 val_387 +454 val_455 +364 val_365 +20 val_21 +52 val_53 +40 val_41 +8 val_9 +168 val_169 +384 val_385 +324 val_325 +310 val_311 +206 val_207 +404 val_405 +206 val_207 +226 val_227 +262 val_263 +260 val_261 +328 val_329 +322 val_323 +122 val_123 +404 val_405 +384 val_385 +76 val_77 +116 val_117 +42 val_43 +104 val_105 +406 val_407 +32 val_33 +132 val_133 +192 val_193 +58 val_59 +70 val_71 +356 val_357 +352 val_353 +52 val_53 +330 val_331 +138 val_139 +160 val_161 +454 val_455 +76 val_77 +174 val_175 +412 val_413 +16 val_17 +204 val_205 +126 val_127 +274 val_275 +374 val_375 +494 val_495 +216 val_217 +470 val_471 +196 val_197 +302 val_303 +450 val_451 +12 val_13 +398 val_399 +334 val_335 +384 val_385 +60 val_61 +442 val_443 +52 val_53 +404 val_405 +446 val_447 +300 val_301 +0 val_1 +268 val_269 +392 val_393 +104 val_105 +436 val_437 +156 val_157 +118 val_119 +172 val_173 +244 val_245 +6 val_7 +284 val_285 +164 val_165 +136 val_137 +462 val_463 +432 val_433 +496 val_497 +144 val_145 +408 val_409 +152 val_153 +382 val_383 +348 val_349 +122 val_123 +292 val_293 +182 val_183 +474 val_475 +310 val_311 +52 val_53 +486 val_487 +152 val_153 +378 val_379 +414 val_415 +256 val_257 +292 val_293 +412 val_413 +40 val_41 +478 val_479 +178 val_179 +100 val_101 +156 val_157 +228 val_229 +22 val_23 +248 val_249 +402 val_403 +62 val_63 +162 val_163 +244 val_245 +276 val_277 +46 val_47 +78 val_79 +134 val_135 +196 val_197 +410 val_411 +82 val_83 +440 val_441 +100 val_101 +308 val_309 +430 val_431 +468 val_469 +152 val_153 +138 val_139 +76 val_77 +300 val_301 +478 val_479 +118 val_119 +178 val_179 +242 val_243 +244 val_245 +238 val_238 +86 val_86 +278 val_278 +98 val_98 +484 val_484 +150 val_150 +224 val_224 +66 val_66 +128 val_128 +146 val_146 +406 val_406 +374 val_374 +152 val_152 +82 val_82 +166 val_166 +430 val_430 +252 val_252 +292 val_292 +338 val_338 +446 val_446 +394 val_394 +482 val_482 +174 val_174 +494 val_494 +466 val_466 +208 val_208 +174 val_174 +396 val_396 +162 val_162 +266 val_266 +342 val_342 +0 val_0 +128 val_128 +316 val_316 +302 val_302 +438 val_438 +170 val_170 +20 val_20 +378 val_378 +92 val_92 +72 val_72 +4 val_4 +280 val_280 +208 val_208 +356 val_356 +382 val_382 +498 val_498 +386 val_386 +192 val_192 +286 val_286 +176 val_176 +54 val_54 +138 val_138 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +318 val_318 +332 val_332 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +260 val_260 +404 val_404 +384 val_384 +272 val_272 +138 val_138 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +172 val_172 +42 val_42 +158 val_158 +496 val_496 +0 val_0 +322 val_322 +468 val_468 +454 val_454 +100 val_100 +298 val_298 +418 val_418 +96 val_96 +26 val_26 +230 val_230 +120 val_120 +404 val_404 +436 val_436 +156 val_156 +468 val_468 +308 val_308 +196 val_196 +288 val_288 +98 val_98 +282 val_282 +318 val_318 +318 val_318 +470 val_470 +316 val_316 +0 val_0 +490 val_490 +364 val_364 +118 val_118 +134 val_134 +282 val_282 +138 val_138 +238 val_238 +118 val_118 +72 val_72 +90 val_90 +10 val_10 +306 val_306 +224 val_224 +242 val_242 +392 val_392 +272 val_272 +242 val_242 +452 val_452 +226 val_226 +402 val_402 +396 val_396 +58 val_58 +336 val_336 +168 val_168 +34 val_34 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +42 val_42 +430 val_430 +458 val_458 +78 val_78 +76 val_76 +492 val_492 +218 val_218 +228 val_228 +138 val_138 +30 val_30 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +230 val_230 +368 val_368 +296 val_296 +216 val_216 +344 val_344 +274 val_274 +116 val_116 +256 val_256 +70 val_70 +480 val_480 +288 val_288 +244 val_244 +438 val_438 +128 val_128 +432 val_432 +202 val_202 +316 val_316 +280 val_280 +2 val_2 +80 val_80 +44 val_44 +104 val_104 +466 val_466 +366 val_366 +406 val_406 +190 val_190 +406 val_406 +114 val_114 +258 val_258 +90 val_90 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +164 val_164 +454 val_454 +478 val_478 +298 val_298 +164 val_164 +424 val_424 +382 val_382 +70 val_70 +480 val_480 +24 val_24 +104 val_104 +70 val_70 +438 val_438 +414 val_414 +200 val_200 +360 val_360 +248 val_248 +444 val_444 +120 val_120 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +460 val_460 +480 val_480 +136 val_136 +172 val_172 +214 val_214 +462 val_462 +406 val_406 +454 val_454 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +384 val_384 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +348 val_348 +18 val_18 +344 val_344 +84 val_84 +28 val_28 +448 val_448 +152 val_152 +348 val_348 +194 val_194 +414 val_414 +222 val_222 +126 val_126 +90 val_90 +400 val_400 +200 val_200 diff --git ql/src/test/results/clientpositive/spark/sample3.q.out ql/src/test/results/clientpositive/spark/sample3.q.out new file mode 100644 index 0000000..a95e234 --- /dev/null +++ ql/src/test/results/clientpositive/spark/sample3.q.out @@ -0,0 +1,229 @@ +PREHOOK: query: -- no input pruning, sample filter +EXPLAIN +SELECT s.key +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s +PREHOOK: type: QUERY +POSTHOOK: query: -- no input pruning, sample filter +EXPLAIN +SELECT s.key +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: s + Statistics: Num rows: 2900 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((hash(key) & 2147483647) % 5) = 0) (type: boolean) + Statistics: Num rows: 1450 Data size: 5801 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1450 Data size: 5801 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: SELECT s.key +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s SORT BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.key +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s SORT BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +#### A masked pattern was here #### +0 +0 +0 +0 +0 +5 +5 +5 +5 +10 +10 +15 +15 +15 +15 +20 +20 +20 +30 +30 +35 +35 +35 +35 +35 +35 +40 +40 +50 +60 +65 +65 +65 +70 +70 +70 +70 +75 +80 +80 +85 +85 +90 +90 +90 +95 +95 +100 +100 +100 +100 +105 +105 +105 +110 +120 +120 +120 +125 +125 +125 +135 +135 +135 +140 +145 +150 +155 +160 +160 +165 +165 +165 +170 +170 +175 +175 +175 +175 +180 +185 +190 +195 +195 +200 +200 +205 +205 +205 +215 +230 +230 +230 +230 +230 +235 +235 +240 +245 +245 +255 +255 +260 +260 +260 +265 +265 +265 +275 +275 +275 +280 +280 +285 +285 +295 +295 +300 +300 +305 +305 +310 +310 +310 +310 +315 +325 +325 +330 +335 +335 +335 +340 +345 +355 +355 +360 +360 +365 +375 +375 +375 +375 +375 +375 +385 +385 +390 +390 +390 +395 +395 +395 +395 +400 +405 +410 +415 +425 +430 +430 +430 +430 +435 +435 +440 +440 +450 +450 +455 +455 +455 +460 +470 +470 +475 +475 +480 +480 +480 +480 +480 +485 +485 +485 +490 +490 +495 +495 diff --git ql/src/test/results/clientpositive/spark/sample4.q.out ql/src/test/results/clientpositive/spark/sample4.q.out new file mode 100644 index 0000000..75281d7 --- /dev/null +++ ql/src/test/results/clientpositive/spark/sample4.q.out @@ -0,0 +1,691 @@ +PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1 +POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1 +PREHOOK: query: -- bucket column is the same as table sample +-- No need for sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +PREHOOK: type: QUERY +POSTHOOK: query: -- bucket column is the same as table sample +-- No need for sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 1 + 2 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest1 + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket0.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket/srcbucket0.txt [s] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@dest1 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@dest1 +POSTHOOK: Lineage: dest1.key SIMPLE [(srcbucket)s.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: dest1.value SIMPLE [(srcbucket)s.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +474 val_475 +62 val_63 +468 val_469 +272 val_273 +448 val_449 +246 val_247 +440 val_441 +278 val_279 +296 val_297 +428 val_429 +126 val_127 +106 val_107 +356 val_357 +490 val_491 +402 val_403 +128 val_129 +10 val_11 +226 val_227 +110 val_111 +0 val_1 +240 val_241 +286 val_287 +408 val_409 +476 val_477 +482 val_483 +48 val_49 +424 val_425 +226 val_227 +494 val_495 +488 val_489 +94 val_95 +50 val_51 +402 val_403 +128 val_129 +468 val_469 +314 val_315 +224 val_225 +344 val_345 +4 val_5 +206 val_207 +114 val_115 +56 val_57 +114 val_115 +254 val_255 +390 val_391 +304 val_305 +264 val_265 +196 val_197 +238 val_239 +20 val_21 +492 val_493 +82 val_83 +58 val_59 +86 val_87 +438 val_439 +360 val_361 +222 val_223 +42 val_43 +338 val_339 +68 val_69 +16 val_17 +492 val_493 +376 val_377 +120 val_121 +306 val_307 +426 val_427 +132 val_133 +446 val_447 +386 val_387 +388 val_389 +184 val_185 +284 val_285 +246 val_247 +262 val_263 +122 val_123 +438 val_439 +390 val_391 +352 val_353 +226 val_227 +328 val_329 +382 val_383 +342 val_343 +480 val_481 +102 val_103 +480 val_481 +318 val_319 +392 val_393 +476 val_477 +258 val_259 +174 val_175 +252 val_253 +114 val_115 +264 val_265 +48 val_49 +336 val_337 +340 val_341 +390 val_391 +484 val_485 +6 val_7 +260 val_261 +2 val_3 +170 val_171 +164 val_165 +118 val_119 +310 val_311 +104 val_105 +80 val_81 +326 val_327 +450 val_451 +140 val_141 +212 val_213 +308 val_309 +30 val_31 +358 val_359 +416 val_417 +42 val_43 +386 val_387 +454 val_455 +364 val_365 +20 val_21 +52 val_53 +40 val_41 +8 val_9 +168 val_169 +384 val_385 +324 val_325 +310 val_311 +206 val_207 +404 val_405 +206 val_207 +226 val_227 +262 val_263 +260 val_261 +328 val_329 +322 val_323 +122 val_123 +404 val_405 +384 val_385 +76 val_77 +116 val_117 +42 val_43 +104 val_105 +406 val_407 +32 val_33 +132 val_133 +192 val_193 +58 val_59 +70 val_71 +356 val_357 +352 val_353 +52 val_53 +330 val_331 +138 val_139 +160 val_161 +454 val_455 +76 val_77 +174 val_175 +412 val_413 +16 val_17 +204 val_205 +126 val_127 +274 val_275 +374 val_375 +494 val_495 +216 val_217 +470 val_471 +196 val_197 +302 val_303 +450 val_451 +12 val_13 +398 val_399 +334 val_335 +384 val_385 +60 val_61 +442 val_443 +52 val_53 +404 val_405 +446 val_447 +300 val_301 +0 val_1 +268 val_269 +392 val_393 +104 val_105 +436 val_437 +156 val_157 +118 val_119 +172 val_173 +244 val_245 +6 val_7 +284 val_285 +164 val_165 +136 val_137 +462 val_463 +432 val_433 +496 val_497 +144 val_145 +408 val_409 +152 val_153 +382 val_383 +348 val_349 +122 val_123 +292 val_293 +182 val_183 +474 val_475 +310 val_311 +52 val_53 +486 val_487 +152 val_153 +378 val_379 +414 val_415 +256 val_257 +292 val_293 +412 val_413 +40 val_41 +478 val_479 +178 val_179 +100 val_101 +156 val_157 +228 val_229 +22 val_23 +248 val_249 +402 val_403 +62 val_63 +162 val_163 +244 val_245 +276 val_277 +46 val_47 +78 val_79 +134 val_135 +196 val_197 +410 val_411 +82 val_83 +440 val_441 +100 val_101 +308 val_309 +430 val_431 +468 val_469 +152 val_153 +138 val_139 +76 val_77 +300 val_301 +478 val_479 +118 val_119 +178 val_179 +242 val_243 +244 val_245 +238 val_238 +86 val_86 +278 val_278 +98 val_98 +484 val_484 +150 val_150 +224 val_224 +66 val_66 +128 val_128 +146 val_146 +406 val_406 +374 val_374 +152 val_152 +82 val_82 +166 val_166 +430 val_430 +252 val_252 +292 val_292 +338 val_338 +446 val_446 +394 val_394 +482 val_482 +174 val_174 +494 val_494 +466 val_466 +208 val_208 +174 val_174 +396 val_396 +162 val_162 +266 val_266 +342 val_342 +0 val_0 +128 val_128 +316 val_316 +302 val_302 +438 val_438 +170 val_170 +20 val_20 +378 val_378 +92 val_92 +72 val_72 +4 val_4 +280 val_280 +208 val_208 +356 val_356 +382 val_382 +498 val_498 +386 val_386 +192 val_192 +286 val_286 +176 val_176 +54 val_54 +138 val_138 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +318 val_318 +332 val_332 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +260 val_260 +404 val_404 +384 val_384 +272 val_272 +138 val_138 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +172 val_172 +42 val_42 +158 val_158 +496 val_496 +0 val_0 +322 val_322 +468 val_468 +454 val_454 +100 val_100 +298 val_298 +418 val_418 +96 val_96 +26 val_26 +230 val_230 +120 val_120 +404 val_404 +436 val_436 +156 val_156 +468 val_468 +308 val_308 +196 val_196 +288 val_288 +98 val_98 +282 val_282 +318 val_318 +318 val_318 +470 val_470 +316 val_316 +0 val_0 +490 val_490 +364 val_364 +118 val_118 +134 val_134 +282 val_282 +138 val_138 +238 val_238 +118 val_118 +72 val_72 +90 val_90 +10 val_10 +306 val_306 +224 val_224 +242 val_242 +392 val_392 +272 val_272 +242 val_242 +452 val_452 +226 val_226 +402 val_402 +396 val_396 +58 val_58 +336 val_336 +168 val_168 +34 val_34 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +42 val_42 +430 val_430 +458 val_458 +78 val_78 +76 val_76 +492 val_492 +218 val_218 +228 val_228 +138 val_138 +30 val_30 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +230 val_230 +368 val_368 +296 val_296 +216 val_216 +344 val_344 +274 val_274 +116 val_116 +256 val_256 +70 val_70 +480 val_480 +288 val_288 +244 val_244 +438 val_438 +128 val_128 +432 val_432 +202 val_202 +316 val_316 +280 val_280 +2 val_2 +80 val_80 +44 val_44 +104 val_104 +466 val_466 +366 val_366 +406 val_406 +190 val_190 +406 val_406 +114 val_114 +258 val_258 +90 val_90 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +164 val_164 +454 val_454 +478 val_478 +298 val_298 +164 val_164 +424 val_424 +382 val_382 +70 val_70 +480 val_480 +24 val_24 +104 val_104 +70 val_70 +438 val_438 +414 val_414 +200 val_200 +360 val_360 +248 val_248 +444 val_444 +120 val_120 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +460 val_460 +480 val_480 +136 val_136 +172 val_172 +214 val_214 +462 val_462 +406 val_406 +454 val_454 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +384 val_384 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +348 val_348 +18 val_18 +344 val_344 +84 val_84 +28 val_28 +448 val_448 +152 val_152 +348 val_348 +194 val_194 +414 val_414 +222 val_222 +126 val_126 +90 val_90 +400 val_400 +200 val_200 diff --git ql/src/test/results/clientpositive/spark/sample5.q.out ql/src/test/results/clientpositive/spark/sample5.q.out new file mode 100644 index 0000000..3618314 --- /dev/null +++ ql/src/test/results/clientpositive/spark/sample5.q.out @@ -0,0 +1,385 @@ +PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1 +POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1 +PREHOOK: query: -- no input pruning, sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s +PREHOOK: type: QUERY +POSTHOOK: query: -- no input pruning, sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 1 + 5 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest1 + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 5) = 0) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket [s] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@dest1 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* -- here's another test +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@dest1 +POSTHOOK: Lineage: dest1.key SIMPLE [(srcbucket)s.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: dest1.value SIMPLE [(srcbucket)s.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT dest1.* FROM dest1 SORT BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1.* FROM dest1 SORT BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +5 val_5 +5 val_5 +5 val_5 +5 val_6 +10 val_10 +10 val_11 +15 val_15 +15 val_15 +15 val_16 +15 val_16 +20 val_20 +20 val_21 +20 val_21 +30 val_30 +30 val_31 +35 val_35 +35 val_35 +35 val_35 +35 val_36 +35 val_36 +35 val_36 +40 val_41 +40 val_41 +50 val_51 +60 val_61 +65 val_65 +65 val_66 +65 val_66 +70 val_70 +70 val_70 +70 val_70 +70 val_71 +75 val_76 +80 val_80 +80 val_81 +85 val_85 +85 val_86 +90 val_90 +90 val_90 +90 val_90 +95 val_95 +95 val_95 +100 val_100 +100 val_100 +100 val_101 +100 val_101 +105 val_105 +105 val_106 +105 val_106 +110 val_111 +120 val_120 +120 val_120 +120 val_121 +125 val_125 +125 val_125 +125 val_126 +135 val_136 +135 val_136 +135 val_136 +140 val_141 +145 val_145 +150 val_150 +155 val_155 +160 val_160 +160 val_161 +165 val_165 +165 val_165 +165 val_166 +170 val_170 +170 val_171 +175 val_175 +175 val_175 +175 val_176 +175 val_176 +180 val_180 +185 val_186 +190 val_190 +195 val_195 +195 val_195 +200 val_200 +200 val_200 +205 val_205 +205 val_205 +205 val_206 +215 val_216 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +235 val_235 +235 val_236 +240 val_241 +245 val_246 +245 val_246 +255 val_255 +255 val_255 +260 val_260 +260 val_261 +260 val_261 +265 val_265 +265 val_265 +265 val_266 +275 val_275 +275 val_276 +275 val_276 +280 val_280 +280 val_280 +285 val_285 +285 val_286 +295 val_296 +295 val_296 +300 val_301 +300 val_301 +305 val_305 +305 val_306 +310 val_310 +310 val_311 +310 val_311 +310 val_311 +315 val_315 +325 val_325 +325 val_325 +330 val_331 +335 val_335 +335 val_336 +335 val_336 +340 val_341 +345 val_345 +355 val_356 +355 val_356 +360 val_360 +360 val_361 +365 val_365 +375 val_375 +375 val_376 +375 val_376 +375 val_376 +375 val_376 +375 val_376 +385 val_386 +385 val_386 +390 val_391 +390 val_391 +390 val_391 +395 val_395 +395 val_395 +395 val_396 +395 val_396 +400 val_400 +405 val_406 +410 val_411 +415 val_416 +425 val_426 +430 val_430 +430 val_430 +430 val_430 +430 val_431 +435 val_435 +435 val_436 +440 val_441 +440 val_441 +450 val_451 +450 val_451 +455 val_455 +455 val_456 +455 val_456 +460 val_460 +470 val_470 +470 val_471 +475 val_475 +475 val_476 +480 val_480 +480 val_480 +480 val_480 +480 val_481 +480 val_481 +485 val_485 +485 val_486 +485 val_486 +490 val_490 +490 val_491 +495 val_495 +495 val_496 diff --git ql/src/test/results/clientpositive/spark/sample6.q.out ql/src/test/results/clientpositive/spark/sample6.q.out new file mode 100644 index 0000000..96ccc2c --- /dev/null +++ ql/src/test/results/clientpositive/spark/sample6.q.out @@ -0,0 +1,3128 @@ +PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1 +POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1 +PREHOOK: query: -- both input pruning and sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s +PREHOOK: type: QUERY +POSTHOOK: query: -- both input pruning and sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 1 + 4 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest1 + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 4) = 0) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket0.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket/srcbucket0.txt [s] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@dest1 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@dest1 +POSTHOOK: Lineage: dest1.key SIMPLE [(srcbucket)s.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: dest1.value SIMPLE [(srcbucket)s.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +468 val_469 +272 val_273 +448 val_449 +440 val_441 +296 val_297 +428 val_429 +356 val_357 +128 val_129 +0 val_1 +240 val_241 +408 val_409 +476 val_477 +48 val_49 +424 val_425 +488 val_489 +128 val_129 +468 val_469 +224 val_225 +344 val_345 +4 val_5 +56 val_57 +304 val_305 +264 val_265 +196 val_197 +20 val_21 +492 val_493 +360 val_361 +68 val_69 +16 val_17 +492 val_493 +376 val_377 +120 val_121 +132 val_133 +388 val_389 +184 val_185 +284 val_285 +352 val_353 +328 val_329 +480 val_481 +480 val_481 +392 val_393 +476 val_477 +252 val_253 +264 val_265 +48 val_49 +336 val_337 +340 val_341 +484 val_485 +260 val_261 +164 val_165 +104 val_105 +80 val_81 +140 val_141 +212 val_213 +308 val_309 +416 val_417 +364 val_365 +20 val_21 +52 val_53 +40 val_41 +8 val_9 +168 val_169 +384 val_385 +324 val_325 +404 val_405 +260 val_261 +328 val_329 +404 val_405 +384 val_385 +76 val_77 +116 val_117 +104 val_105 +32 val_33 +132 val_133 +192 val_193 +356 val_357 +352 val_353 +52 val_53 +160 val_161 +76 val_77 +412 val_413 +16 val_17 +204 val_205 +216 val_217 +196 val_197 +12 val_13 +384 val_385 +60 val_61 +52 val_53 +404 val_405 +300 val_301 +0 val_1 +268 val_269 +392 val_393 +104 val_105 +436 val_437 +156 val_157 +172 val_173 +244 val_245 +284 val_285 +164 val_165 +136 val_137 +432 val_433 +496 val_497 +144 val_145 +408 val_409 +152 val_153 +348 val_349 +292 val_293 +52 val_53 +152 val_153 +256 val_257 +292 val_293 +412 val_413 +40 val_41 +100 val_101 +156 val_157 +228 val_229 +248 val_249 +244 val_245 +276 val_277 +196 val_197 +440 val_441 +100 val_101 +308 val_309 +468 val_469 +152 val_153 +76 val_77 +300 val_301 +244 val_245 +484 val_484 +224 val_224 +128 val_128 +152 val_152 +252 val_252 +292 val_292 +208 val_208 +396 val_396 +0 val_0 +128 val_128 +316 val_316 +20 val_20 +92 val_92 +72 val_72 +4 val_4 +280 val_280 +208 val_208 +356 val_356 +192 val_192 +176 val_176 +216 val_216 +176 val_176 +332 val_332 +180 val_180 +284 val_284 +12 val_12 +260 val_260 +404 val_404 +384 val_384 +272 val_272 +84 val_84 +348 val_348 +8 val_8 +208 val_208 +348 val_348 +24 val_24 +172 val_172 +496 val_496 +0 val_0 +468 val_468 +100 val_100 +96 val_96 +120 val_120 +404 val_404 +436 val_436 +156 val_156 +468 val_468 +308 val_308 +196 val_196 +288 val_288 +316 val_316 +0 val_0 +364 val_364 +72 val_72 +224 val_224 +392 val_392 +272 val_272 +452 val_452 +396 val_396 +336 val_336 +168 val_168 +472 val_472 +160 val_160 +76 val_76 +492 val_492 +228 val_228 +64 val_64 +468 val_468 +76 val_76 +368 val_368 +296 val_296 +216 val_216 +344 val_344 +116 val_116 +256 val_256 +480 val_480 +288 val_288 +244 val_244 +128 val_128 +432 val_432 +316 val_316 +280 val_280 +80 val_80 +44 val_44 +104 val_104 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +164 val_164 +164 val_164 +424 val_424 +480 val_480 +24 val_24 +104 val_104 +200 val_200 +360 val_360 +248 val_248 +444 val_444 +120 val_120 +468 val_468 +460 val_460 +480 val_480 +136 val_136 +172 val_172 +384 val_384 +256 val_256 +384 val_384 +492 val_492 +100 val_100 +348 val_348 +344 val_344 +84 val_84 +28 val_28 +448 val_448 +152 val_152 +348 val_348 +400 val_400 +200 val_200 +PREHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 4 OUT OF 4 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 4 OUT OF 4 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 4 + 4 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 4) = 3) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket1.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket/srcbucket1.txt [s] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 4 OUT OF 4 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 4 OUT OF 4 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +#### A masked pattern was here #### +3 val_4 +11 val_11 +11 val_12 +11 val_12 +11 val_12 +15 val_15 +15 val_15 +15 val_16 +15 val_16 +19 val_19 +19 val_20 +23 val_24 +27 val_27 +31 val_32 +35 val_35 +35 val_35 +35 val_35 +35 val_36 +35 val_36 +35 val_36 +43 val_43 +47 val_47 +47 val_48 +51 val_51 +51 val_51 +51 val_52 +59 val_60 +63 val_64 +67 val_67 +67 val_67 +71 val_72 +75 val_76 +83 val_83 +83 val_83 +87 val_87 +87 val_88 +87 val_88 +91 val_92 +95 val_95 +95 val_95 +99 val_100 +103 val_103 +103 val_103 +111 val_111 +119 val_119 +119 val_119 +119 val_119 +119 val_120 +119 val_120 +119 val_120 +123 val_124 +123 val_124 +131 val_131 +135 val_136 +135 val_136 +135 val_136 +143 val_143 +143 val_144 +147 val_148 +147 val_148 +151 val_152 +151 val_152 +155 val_155 +163 val_163 +167 val_167 +167 val_167 +167 val_167 +167 val_168 +175 val_175 +175 val_175 +175 val_176 +175 val_176 +179 val_179 +179 val_179 +179 val_180 +183 val_183 +183 val_184 +187 val_187 +187 val_187 +187 val_187 +191 val_191 +191 val_191 +191 val_192 +195 val_195 +195 val_195 +199 val_199 +199 val_199 +199 val_199 +199 val_200 +203 val_203 +203 val_203 +207 val_207 +207 val_207 +215 val_216 +219 val_219 +219 val_219 +223 val_223 +223 val_223 +227 val_228 +231 val_232 +235 val_235 +235 val_236 +239 val_239 +239 val_239 +239 val_240 +239 val_240 +243 val_244 +243 val_244 +247 val_247 +255 val_255 +255 val_255 +259 val_260 +259 val_260 +263 val_263 +267 val_268 +271 val_272 +275 val_275 +275 val_276 +275 val_276 +283 val_283 +287 val_287 +287 val_288 +287 val_288 +291 val_291 +291 val_292 +291 val_292 +295 val_296 +295 val_296 +303 val_304 +303 val_304 +307 val_307 +307 val_307 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +323 val_323 +323 val_324 +327 val_327 +327 val_327 +327 val_327 +331 val_331 +331 val_331 +331 val_332 +331 val_332 +335 val_335 +335 val_336 +335 val_336 +339 val_339 +343 val_344 +347 val_348 +347 val_348 +351 val_351 +351 val_352 +351 val_352 +355 val_356 +355 val_356 +363 val_364 +363 val_364 +363 val_364 +367 val_367 +367 val_367 +367 val_368 +367 val_368 +371 val_372 +371 val_372 +371 val_372 +371 val_372 +375 val_375 +375 val_376 +375 val_376 +375 val_376 +375 val_376 +375 val_376 +379 val_379 +379 val_380 +391 val_392 +391 val_392 +395 val_395 +395 val_395 +395 val_396 +395 val_396 +399 val_399 +399 val_399 +399 val_400 +399 val_400 +403 val_403 +403 val_403 +403 val_403 +407 val_407 +407 val_408 +407 val_408 +407 val_408 +411 val_411 +411 val_412 +415 val_416 +419 val_419 +423 val_424 +427 val_427 +427 val_428 +427 val_428 +431 val_431 +431 val_431 +431 val_431 +431 val_432 +435 val_435 +435 val_436 +439 val_439 +439 val_439 +439 val_440 +439 val_440 +443 val_443 +443 val_444 +443 val_444 +443 val_444 +447 val_448 +451 val_452 +455 val_455 +455 val_456 +455 val_456 +459 val_459 +459 val_459 +459 val_460 +463 val_463 +463 val_463 +463 val_464 +467 val_467 +467 val_468 +475 val_475 +475 val_476 +479 val_479 +483 val_483 +487 val_487 +487 val_488 +491 val_491 +491 val_492 +491 val_492 +495 val_495 +495 val_496 +PREHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 1 + 2 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket0.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket/srcbucket0.txt [s] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +2 val_2 +2 val_3 +4 val_4 +4 val_5 +6 val_7 +6 val_7 +8 val_8 +8 val_9 +10 val_10 +10 val_11 +12 val_12 +12 val_12 +12 val_13 +16 val_17 +16 val_17 +18 val_18 +18 val_18 +20 val_20 +20 val_21 +20 val_21 +22 val_23 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +28 val_28 +30 val_30 +30 val_31 +32 val_33 +34 val_34 +40 val_41 +40 val_41 +42 val_42 +42 val_42 +42 val_43 +42 val_43 +42 val_43 +44 val_44 +46 val_47 +48 val_49 +48 val_49 +50 val_51 +52 val_53 +52 val_53 +52 val_53 +52 val_53 +54 val_54 +56 val_57 +58 val_58 +58 val_58 +58 val_59 +58 val_59 +60 val_61 +62 val_63 +62 val_63 +64 val_64 +66 val_66 +68 val_69 +70 val_70 +70 val_70 +70 val_70 +70 val_71 +72 val_72 +72 val_72 +74 val_74 +76 val_76 +76 val_76 +76 val_77 +76 val_77 +76 val_77 +78 val_78 +78 val_79 +80 val_80 +80 val_81 +82 val_82 +82 val_83 +82 val_83 +84 val_84 +84 val_84 +86 val_86 +86 val_87 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +94 val_95 +96 val_96 +98 val_98 +98 val_98 +100 val_100 +100 val_100 +100 val_101 +100 val_101 +102 val_103 +104 val_104 +104 val_104 +104 val_105 +104 val_105 +104 val_105 +106 val_107 +110 val_111 +114 val_114 +114 val_115 +114 val_115 +114 val_115 +116 val_116 +116 val_117 +118 val_118 +118 val_118 +118 val_119 +118 val_119 +118 val_119 +120 val_120 +120 val_120 +120 val_121 +122 val_123 +122 val_123 +122 val_123 +126 val_126 +126 val_127 +126 val_127 +128 val_128 +128 val_128 +128 val_128 +128 val_129 +128 val_129 +132 val_133 +132 val_133 +134 val_134 +134 val_134 +134 val_135 +136 val_136 +136 val_137 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_139 +138 val_139 +140 val_141 +144 val_145 +146 val_146 +146 val_146 +150 val_150 +152 val_152 +152 val_152 +152 val_153 +152 val_153 +152 val_153 +156 val_156 +156 val_157 +156 val_157 +158 val_158 +160 val_160 +160 val_161 +162 val_162 +162 val_163 +164 val_164 +164 val_164 +164 val_165 +164 val_165 +166 val_166 +168 val_168 +168 val_169 +170 val_170 +170 val_171 +172 val_172 +172 val_172 +172 val_173 +174 val_174 +174 val_174 +174 val_175 +174 val_175 +176 val_176 +176 val_176 +178 val_178 +178 val_179 +178 val_179 +180 val_180 +182 val_183 +184 val_185 +186 val_186 +190 val_190 +192 val_192 +192 val_193 +194 val_194 +196 val_196 +196 val_197 +196 val_197 +196 val_197 +200 val_200 +200 val_200 +202 val_202 +204 val_205 +206 val_207 +206 val_207 +206 val_207 +208 val_208 +208 val_208 +208 val_208 +212 val_213 +214 val_214 +216 val_216 +216 val_216 +216 val_217 +218 val_218 +222 val_222 +222 val_223 +224 val_224 +224 val_224 +224 val_225 +226 val_226 +226 val_227 +226 val_227 +226 val_227 +226 val_227 +228 val_228 +228 val_229 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +238 val_238 +238 val_238 +238 val_239 +240 val_241 +242 val_242 +242 val_242 +242 val_243 +244 val_244 +244 val_245 +244 val_245 +244 val_245 +246 val_247 +246 val_247 +248 val_248 +248 val_249 +252 val_252 +252 val_253 +254 val_255 +256 val_256 +256 val_256 +256 val_257 +258 val_258 +258 val_259 +260 val_260 +260 val_261 +260 val_261 +262 val_262 +262 val_263 +262 val_263 +264 val_265 +264 val_265 +266 val_266 +268 val_269 +272 val_272 +272 val_272 +272 val_273 +274 val_274 +274 val_275 +276 val_277 +278 val_278 +278 val_278 +278 val_279 +280 val_280 +280 val_280 +282 val_282 +282 val_282 +284 val_284 +284 val_285 +284 val_285 +286 val_286 +286 val_287 +288 val_288 +288 val_288 +292 val_292 +292 val_293 +292 val_293 +296 val_296 +296 val_297 +298 val_298 +298 val_298 +298 val_298 +300 val_301 +300 val_301 +302 val_302 +302 val_303 +304 val_305 +306 val_306 +306 val_307 +308 val_308 +308 val_309 +308 val_309 +310 val_310 +310 val_311 +310 val_311 +310 val_311 +314 val_315 +316 val_316 +316 val_316 +316 val_316 +318 val_318 +318 val_318 +318 val_318 +318 val_319 +322 val_322 +322 val_322 +322 val_323 +324 val_325 +326 val_327 +328 val_329 +328 val_329 +330 val_331 +332 val_332 +334 val_335 +336 val_336 +336 val_337 +338 val_338 +338 val_339 +340 val_341 +342 val_342 +342 val_342 +342 val_343 +344 val_344 +344 val_344 +344 val_345 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_349 +352 val_353 +352 val_353 +356 val_356 +356 val_357 +356 val_357 +358 val_359 +360 val_360 +360 val_361 +362 val_362 +364 val_364 +364 val_365 +366 val_366 +368 val_368 +374 val_374 +374 val_375 +376 val_377 +378 val_378 +378 val_379 +382 val_382 +382 val_382 +382 val_383 +382 val_383 +384 val_384 +384 val_384 +384 val_384 +384 val_385 +384 val_385 +384 val_385 +386 val_386 +386 val_387 +386 val_387 +388 val_389 +390 val_391 +390 val_391 +390 val_391 +392 val_392 +392 val_393 +392 val_393 +394 val_394 +396 val_396 +396 val_396 +396 val_396 +398 val_399 +400 val_400 +402 val_402 +402 val_403 +402 val_403 +402 val_403 +404 val_404 +404 val_404 +404 val_405 +404 val_405 +404 val_405 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_407 +408 val_409 +408 val_409 +410 val_411 +412 val_413 +412 val_413 +414 val_414 +414 val_414 +414 val_415 +416 val_417 +418 val_418 +424 val_424 +424 val_424 +424 val_425 +426 val_427 +428 val_429 +430 val_430 +430 val_430 +430 val_430 +430 val_431 +432 val_432 +432 val_433 +436 val_436 +436 val_437 +438 val_438 +438 val_438 +438 val_438 +438 val_439 +438 val_439 +440 val_441 +440 val_441 +442 val_443 +444 val_444 +446 val_446 +446 val_447 +446 val_447 +448 val_448 +448 val_449 +450 val_451 +450 val_451 +452 val_452 +454 val_454 +454 val_454 +454 val_454 +454 val_455 +454 val_455 +458 val_458 +458 val_458 +460 val_460 +462 val_462 +462 val_462 +462 val_463 +466 val_466 +466 val_466 +466 val_466 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_469 +468 val_469 +468 val_469 +470 val_470 +470 val_471 +472 val_472 +474 val_475 +474 val_475 +476 val_477 +476 val_477 +478 val_478 +478 val_478 +478 val_479 +478 val_479 +480 val_480 +480 val_480 +480 val_480 +480 val_481 +480 val_481 +482 val_482 +482 val_483 +484 val_484 +484 val_485 +486 val_487 +488 val_489 +490 val_490 +490 val_491 +492 val_492 +492 val_492 +492 val_493 +492 val_493 +494 val_494 +494 val_495 +494 val_495 +496 val_496 +496 val_497 +498 val_498 +498 val_498 +498 val_498 +PREHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 3 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 3 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 1 + 3 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 3) = 0) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket [s] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 3 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 3 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +0 val_1 +0 val_1 +3 val_4 +6 val_7 +6 val_7 +9 val_9 +12 val_12 +12 val_12 +12 val_13 +15 val_15 +15 val_15 +15 val_16 +15 val_16 +18 val_18 +18 val_18 +21 val_22 +21 val_22 +21 val_22 +21 val_22 +24 val_24 +24 val_24 +27 val_27 +30 val_30 +30 val_31 +33 val_33 +33 val_34 +42 val_42 +42 val_42 +42 val_43 +42 val_43 +42 val_43 +48 val_49 +48 val_49 +51 val_51 +51 val_51 +51 val_52 +54 val_54 +57 val_57 +60 val_61 +63 val_64 +66 val_66 +69 val_69 +69 val_70 +72 val_72 +72 val_72 +75 val_76 +78 val_78 +78 val_79 +84 val_84 +84 val_84 +87 val_87 +87 val_88 +87 val_88 +90 val_90 +90 val_90 +90 val_90 +93 val_94 +93 val_94 +93 val_94 +96 val_96 +99 val_100 +102 val_103 +105 val_105 +105 val_106 +105 val_106 +111 val_111 +114 val_114 +114 val_115 +114 val_115 +114 val_115 +117 val_118 +117 val_118 +120 val_120 +120 val_120 +120 val_121 +123 val_124 +123 val_124 +126 val_126 +126 val_127 +126 val_127 +129 val_129 +129 val_129 +129 val_130 +129 val_130 +132 val_133 +132 val_133 +135 val_136 +135 val_136 +135 val_136 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_139 +138 val_139 +144 val_145 +147 val_148 +147 val_148 +150 val_150 +153 val_153 +153 val_154 +153 val_154 +156 val_156 +156 val_157 +156 val_157 +162 val_162 +162 val_163 +165 val_165 +165 val_165 +165 val_166 +168 val_168 +168 val_169 +174 val_174 +174 val_174 +174 val_175 +174 val_175 +177 val_177 +177 val_178 +177 val_178 +180 val_180 +183 val_183 +183 val_184 +186 val_186 +189 val_189 +189 val_190 +192 val_192 +192 val_193 +195 val_195 +195 val_195 +201 val_201 +204 val_205 +207 val_207 +207 val_207 +213 val_213 +213 val_213 +213 val_214 +216 val_216 +216 val_216 +216 val_217 +219 val_219 +219 val_219 +222 val_222 +222 val_223 +228 val_228 +228 val_229 +231 val_232 +237 val_237 +237 val_237 +240 val_241 +243 val_244 +243 val_244 +246 val_247 +246 val_247 +249 val_249 +249 val_250 +249 val_250 +252 val_252 +252 val_253 +255 val_255 +255 val_255 +258 val_258 +258 val_259 +261 val_262 +264 val_265 +264 val_265 +267 val_268 +273 val_273 +273 val_273 +273 val_273 +273 val_274 +276 val_277 +282 val_282 +282 val_282 +285 val_285 +285 val_286 +288 val_288 +288 val_288 +291 val_291 +291 val_292 +291 val_292 +300 val_301 +300 val_301 +303 val_304 +303 val_304 +306 val_306 +306 val_307 +309 val_309 +309 val_309 +309 val_310 +315 val_315 +318 val_318 +318 val_318 +318 val_318 +318 val_319 +321 val_321 +321 val_321 +324 val_325 +327 val_327 +327 val_327 +327 val_327 +330 val_331 +333 val_333 +333 val_333 +333 val_334 +336 val_336 +336 val_337 +339 val_339 +342 val_342 +342 val_342 +342 val_343 +345 val_345 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_349 +351 val_351 +351 val_352 +351 val_352 +360 val_360 +360 val_361 +363 val_364 +363 val_364 +363 val_364 +366 val_366 +369 val_369 +369 val_369 +369 val_369 +369 val_370 +375 val_375 +375 val_376 +375 val_376 +375 val_376 +375 val_376 +375 val_376 +378 val_378 +378 val_379 +381 val_382 +384 val_384 +384 val_384 +384 val_384 +384 val_385 +384 val_385 +384 val_385 +390 val_391 +390 val_391 +390 val_391 +393 val_393 +393 val_394 +393 val_394 +396 val_396 +396 val_396 +396 val_396 +399 val_399 +399 val_399 +399 val_400 +399 val_400 +402 val_402 +402 val_403 +402 val_403 +402 val_403 +405 val_406 +408 val_409 +408 val_409 +411 val_411 +411 val_412 +414 val_414 +414 val_414 +414 val_415 +417 val_417 +417 val_417 +417 val_417 +423 val_424 +426 val_427 +429 val_429 +429 val_429 +429 val_430 +429 val_430 +432 val_432 +432 val_433 +435 val_435 +435 val_436 +438 val_438 +438 val_438 +438 val_438 +438 val_439 +438 val_439 +441 val_442 +444 val_444 +447 val_448 +450 val_451 +450 val_451 +453 val_453 +453 val_454 +459 val_459 +459 val_459 +459 val_460 +462 val_462 +462 val_462 +462 val_463 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_469 +468 val_469 +468 val_469 +474 val_475 +474 val_475 +477 val_477 +477 val_478 +480 val_480 +480 val_480 +480 val_480 +480 val_481 +480 val_481 +483 val_483 +486 val_487 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_490 +492 val_492 +492 val_492 +492 val_493 +492 val_493 +495 val_495 +495 val_496 +498 val_498 +498 val_498 +498 val_498 +PREHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 2 OUT OF 3 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 2 OUT OF 3 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 2 + 3 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 3) = 1) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket [s] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 2 OUT OF 3 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 2 OUT OF 3 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +#### A masked pattern was here #### +1 val_2 +4 val_4 +4 val_5 +10 val_10 +10 val_11 +16 val_17 +16 val_17 +19 val_19 +19 val_20 +22 val_23 +28 val_28 +31 val_32 +34 val_34 +37 val_37 +37 val_37 +40 val_41 +40 val_41 +43 val_43 +46 val_47 +49 val_50 +49 val_50 +52 val_53 +52 val_53 +52 val_53 +52 val_53 +58 val_58 +58 val_58 +58 val_59 +58 val_59 +61 val_62 +64 val_64 +67 val_67 +67 val_67 +70 val_70 +70 val_70 +70 val_70 +70 val_71 +76 val_76 +76 val_76 +76 val_77 +76 val_77 +76 val_77 +82 val_82 +82 val_83 +82 val_83 +85 val_85 +85 val_86 +91 val_92 +94 val_95 +97 val_97 +97 val_97 +97 val_98 +97 val_98 +100 val_100 +100 val_100 +100 val_101 +100 val_101 +103 val_103 +103 val_103 +106 val_107 +118 val_118 +118 val_118 +118 val_119 +118 val_119 +118 val_119 +121 val_122 +121 val_122 +133 val_133 +133 val_134 +136 val_136 +136 val_137 +145 val_145 +151 val_152 +151 val_152 +157 val_157 +157 val_158 +157 val_158 +160 val_160 +160 val_161 +163 val_163 +166 val_166 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +172 val_172 +172 val_172 +172 val_173 +175 val_175 +175 val_175 +175 val_176 +175 val_176 +178 val_178 +178 val_179 +178 val_179 +181 val_181 +184 val_185 +187 val_187 +187 val_187 +187 val_187 +190 val_190 +193 val_193 +193 val_193 +193 val_193 +196 val_196 +196 val_197 +196 val_197 +196 val_197 +199 val_199 +199 val_199 +199 val_199 +199 val_200 +202 val_202 +205 val_205 +205 val_205 +205 val_206 +208 val_208 +208 val_208 +208 val_208 +214 val_214 +217 val_217 +217 val_217 +217 val_218 +217 val_218 +223 val_223 +223 val_223 +226 val_226 +226 val_227 +226 val_227 +226 val_227 +226 val_227 +229 val_229 +229 val_229 +235 val_235 +235 val_236 +238 val_238 +238 val_238 +238 val_239 +241 val_241 +241 val_242 +241 val_242 +241 val_242 +241 val_242 +244 val_244 +244 val_245 +244 val_245 +244 val_245 +247 val_247 +256 val_256 +256 val_256 +256 val_257 +259 val_260 +259 val_260 +262 val_262 +262 val_263 +262 val_263 +265 val_265 +265 val_265 +265 val_266 +268 val_269 +271 val_272 +274 val_274 +274 val_275 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_278 +277 val_278 +280 val_280 +280 val_280 +283 val_283 +286 val_286 +286 val_287 +289 val_289 +289 val_290 +292 val_292 +292 val_293 +292 val_293 +295 val_296 +295 val_296 +298 val_298 +298 val_298 +298 val_298 +304 val_305 +307 val_307 +307 val_307 +310 val_310 +310 val_311 +310 val_311 +310 val_311 +313 val_314 +316 val_316 +316 val_316 +316 val_316 +322 val_322 +322 val_322 +322 val_323 +325 val_325 +325 val_325 +328 val_329 +328 val_329 +331 val_331 +331 val_331 +331 val_332 +331 val_332 +334 val_335 +337 val_338 +340 val_341 +343 val_344 +349 val_350 +349 val_350 +349 val_350 +349 val_350 +352 val_353 +352 val_353 +355 val_356 +355 val_356 +358 val_359 +364 val_364 +364 val_365 +367 val_367 +367 val_367 +367 val_368 +367 val_368 +373 val_373 +373 val_374 +376 val_377 +379 val_379 +379 val_380 +382 val_382 +382 val_382 +382 val_383 +382 val_383 +385 val_386 +385 val_386 +388 val_389 +391 val_392 +391 val_392 +394 val_394 +397 val_397 +397 val_397 +400 val_400 +403 val_403 +403 val_403 +403 val_403 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_407 +409 val_409 +409 val_409 +409 val_409 +409 val_410 +409 val_410 +412 val_413 +412 val_413 +415 val_416 +418 val_418 +421 val_421 +421 val_422 +421 val_422 +424 val_424 +424 val_424 +424 val_425 +427 val_427 +427 val_428 +427 val_428 +430 val_430 +430 val_430 +430 val_430 +430 val_431 +436 val_436 +436 val_437 +439 val_439 +439 val_439 +439 val_440 +439 val_440 +442 val_443 +448 val_448 +448 val_449 +451 val_452 +454 val_454 +454 val_454 +454 val_454 +454 val_455 +454 val_455 +457 val_457 +457 val_458 +457 val_458 +460 val_460 +463 val_463 +463 val_463 +463 val_464 +466 val_466 +466 val_466 +466 val_466 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_470 +472 val_472 +475 val_475 +475 val_476 +478 val_478 +478 val_478 +478 val_479 +478 val_479 +481 val_481 +481 val_482 +484 val_484 +484 val_485 +487 val_487 +487 val_488 +490 val_490 +490 val_491 +493 val_493 +496 val_496 +496 val_497 +PREHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket2 TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket2 TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket2 + TOK_TABLEBUCKETSAMPLE + 1 + 2 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket20.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket2 + numFiles 4 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket2 + numFiles 4 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket2 + name: default.srcbucket2 +#### A masked pattern was here #### + Partition + base file name: srcbucket22.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket2 + numFiles 4 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket2 + numFiles 4 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket2 + name: default.srcbucket2 + Truncated Path -> Alias: + /srcbucket2/srcbucket20.txt [s] + /srcbucket2/srcbucket22.txt [s] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s.* FROM srcbucket2 TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.* FROM srcbucket2 TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket2 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +2 val_2 +4 val_4 +8 val_8 +20 val_20 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +28 val_28 +42 val_42 +42 val_42 +44 val_44 +64 val_64 +66 val_66 +80 val_80 +82 val_82 +84 val_84 +84 val_84 +86 val_86 +114 val_114 +116 val_116 +118 val_118 +118 val_118 +134 val_134 +134 val_134 +136 val_136 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +150 val_150 +152 val_152 +152 val_152 +156 val_156 +158 val_158 +170 val_170 +172 val_172 +172 val_172 +174 val_174 +174 val_174 +176 val_176 +176 val_176 +178 val_178 +190 val_190 +192 val_192 +194 val_194 +196 val_196 +200 val_200 +200 val_200 +202 val_202 +208 val_208 +208 val_208 +208 val_208 +222 val_222 +224 val_224 +224 val_224 +226 val_226 +228 val_228 +242 val_242 +242 val_242 +244 val_244 +248 val_248 +260 val_260 +262 val_262 +266 val_266 +280 val_280 +280 val_280 +282 val_282 +282 val_282 +284 val_284 +286 val_286 +288 val_288 +288 val_288 +310 val_310 +316 val_316 +316 val_316 +316 val_316 +318 val_318 +318 val_318 +318 val_318 +332 val_332 +336 val_336 +338 val_338 +356 val_356 +374 val_374 +378 val_378 +392 val_392 +394 val_394 +396 val_396 +396 val_396 +396 val_396 +400 val_400 +402 val_402 +404 val_404 +404 val_404 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +424 val_424 +424 val_424 +444 val_444 +446 val_446 +448 val_448 +460 val_460 +462 val_462 +462 val_462 +466 val_466 +466 val_466 +466 val_466 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +480 val_480 +480 val_480 +480 val_480 +482 val_482 +484 val_484 +PREHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket2 TABLESAMPLE (BUCKET 2 OUT OF 4 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM srcbucket2 TABLESAMPLE (BUCKET 2 OUT OF 4 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket2 + TOK_TABLEBUCKETSAMPLE + 2 + 4 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 4) = 1) (type: boolean) + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE + tag: -1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket21.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket2 + numFiles 4 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket2 + numFiles 4 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket2 + name: default.srcbucket2 + Truncated Path -> Alias: + /srcbucket2/srcbucket21.txt [s] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s.* FROM srcbucket2 TABLESAMPLE (BUCKET 2 OUT OF 4 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.* FROM srcbucket2 TABLESAMPLE (BUCKET 2 OUT OF 4 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket2 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +9 val_9 +41 val_41 +85 val_85 +133 val_133 +137 val_137 +137 val_137 +177 val_177 +221 val_221 +221 val_221 +229 val_229 +229 val_229 +265 val_265 +265 val_265 +317 val_317 +317 val_317 +353 val_353 +353 val_353 +393 val_393 +397 val_397 +397 val_397 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +409 val_409 +409 val_409 +409 val_409 +449 val_449 +481 val_481 +485 val_485 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +PREHOOK: query: CREATE TABLE empty_bucket (key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@empty_bucket +POSTHOOK: query: CREATE TABLE empty_bucket (key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@empty_bucket +PREHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM empty_bucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED SELECT s.* FROM empty_bucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + empty_bucket + TOK_TABLEBUCKETSAMPLE + 1 + 2 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s.* FROM empty_bucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +PREHOOK: type: QUERY +PREHOOK: Input: default@empty_bucket +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.* FROM empty_bucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) s +ORDER BY key, value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@empty_bucket +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/sample7.q.out ql/src/test/results/clientpositive/spark/sample7.q.out new file mode 100644 index 0000000..08f6148 --- /dev/null +++ ql/src/test/results/clientpositive/spark/sample7.q.out @@ -0,0 +1,405 @@ +PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1 +POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1 +PREHOOK: query: -- both input pruning and sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s +WHERE s.key > 100 +PREHOOK: type: QUERY +POSTHOOK: query: -- both input pruning and sample filter +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s +WHERE s.key > 100 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 1 + 4 + TOK_TABLE_OR_COL + key + s + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest1 + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + TOK_WHERE + > + . + TOK_TABLE_OR_COL + s + key + 100 + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((((hash(key) & 2147483647) % 4) = 0) and (key > 100)) (type: boolean) + Statistics: Num rows: 18 Data size: 1881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 1881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 18 Data size: 1881 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket0.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket/srcbucket0.txt [s] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s +WHERE s.key > 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +PREHOOK: Output: default@dest1 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 4 on key) s +WHERE s.key > 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +POSTHOOK: Output: default@dest1 +POSTHOOK: Lineage: dest1.key SIMPLE [(srcbucket)s.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: dest1.value SIMPLE [(srcbucket)s.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +468 val_469 +272 val_273 +448 val_449 +440 val_441 +296 val_297 +428 val_429 +356 val_357 +128 val_129 +240 val_241 +408 val_409 +476 val_477 +424 val_425 +488 val_489 +128 val_129 +468 val_469 +224 val_225 +344 val_345 +304 val_305 +264 val_265 +196 val_197 +492 val_493 +360 val_361 +492 val_493 +376 val_377 +120 val_121 +132 val_133 +388 val_389 +184 val_185 +284 val_285 +352 val_353 +328 val_329 +480 val_481 +480 val_481 +392 val_393 +476 val_477 +252 val_253 +264 val_265 +336 val_337 +340 val_341 +484 val_485 +260 val_261 +164 val_165 +104 val_105 +140 val_141 +212 val_213 +308 val_309 +416 val_417 +364 val_365 +168 val_169 +384 val_385 +324 val_325 +404 val_405 +260 val_261 +328 val_329 +404 val_405 +384 val_385 +116 val_117 +104 val_105 +132 val_133 +192 val_193 +356 val_357 +352 val_353 +160 val_161 +412 val_413 +204 val_205 +216 val_217 +196 val_197 +384 val_385 +404 val_405 +300 val_301 +268 val_269 +392 val_393 +104 val_105 +436 val_437 +156 val_157 +172 val_173 +244 val_245 +284 val_285 +164 val_165 +136 val_137 +432 val_433 +496 val_497 +144 val_145 +408 val_409 +152 val_153 +348 val_349 +292 val_293 +152 val_153 +256 val_257 +292 val_293 +412 val_413 +156 val_157 +228 val_229 +248 val_249 +244 val_245 +276 val_277 +196 val_197 +440 val_441 +308 val_309 +468 val_469 +152 val_153 +300 val_301 +244 val_245 +484 val_484 +224 val_224 +128 val_128 +152 val_152 +252 val_252 +292 val_292 +208 val_208 +396 val_396 +128 val_128 +316 val_316 +280 val_280 +208 val_208 +356 val_356 +192 val_192 +176 val_176 +216 val_216 +176 val_176 +332 val_332 +180 val_180 +284 val_284 +260 val_260 +404 val_404 +384 val_384 +272 val_272 +348 val_348 +208 val_208 +348 val_348 +172 val_172 +496 val_496 +468 val_468 +120 val_120 +404 val_404 +436 val_436 +156 val_156 +468 val_468 +308 val_308 +196 val_196 +288 val_288 +316 val_316 +364 val_364 +224 val_224 +392 val_392 +272 val_272 +452 val_452 +396 val_396 +336 val_336 +168 val_168 +472 val_472 +160 val_160 +492 val_492 +228 val_228 +468 val_468 +368 val_368 +296 val_296 +216 val_216 +344 val_344 +116 val_116 +256 val_256 +480 val_480 +288 val_288 +244 val_244 +128 val_128 +432 val_432 +316 val_316 +280 val_280 +104 val_104 +348 val_348 +424 val_424 +396 val_396 +164 val_164 +164 val_164 +424 val_424 +480 val_480 +104 val_104 +200 val_200 +360 val_360 +248 val_248 +444 val_444 +120 val_120 +468 val_468 +460 val_460 +480 val_480 +136 val_136 +172 val_172 +384 val_384 +256 val_256 +384 val_384 +492 val_492 +348 val_348 +344 val_344 +448 val_448 +152 val_152 +348 val_348 +400 val_400 +200 val_200 diff --git ql/src/test/results/clientpositive/spark/sample9.q.out ql/src/test/results/clientpositive/spark/sample9.q.out new file mode 100644 index 0000000..6e8912e --- /dev/null +++ ql/src/test/results/clientpositive/spark/sample9.q.out @@ -0,0 +1,650 @@ +PREHOOK: query: EXPLAIN EXTENDED +SELECT s.* +FROM (SELECT a.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) a) s +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +SELECT s.* +FROM (SELECT a.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) a) s +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcbucket + TOK_TABLEBUCKETSAMPLE + 1 + 2 + TOK_TABLE_OR_COL + key + a + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + a + s + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_TABNAME + s + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 111 Data size: 11603 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: true + predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 55 Data size: 5749 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: srcbucket0.txt + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket + numFiles 2 + numRows 0 + rawDataSize 0 + serialization.ddl struct srcbucket { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11603 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket + name: default.srcbucket + Truncated Path -> Alias: + /srcbucket/srcbucket0.txt [a] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s.* +FROM (SELECT a.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) a) s +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket +#### A masked pattern was here #### +POSTHOOK: query: SELECT s.* +FROM (SELECT a.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) a) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket +#### A masked pattern was here #### +474 val_475 +62 val_63 +468 val_469 +272 val_273 +448 val_449 +246 val_247 +440 val_441 +278 val_279 +296 val_297 +428 val_429 +126 val_127 +106 val_107 +356 val_357 +490 val_491 +402 val_403 +128 val_129 +10 val_11 +226 val_227 +110 val_111 +0 val_1 +240 val_241 +286 val_287 +408 val_409 +476 val_477 +482 val_483 +48 val_49 +424 val_425 +226 val_227 +494 val_495 +488 val_489 +94 val_95 +50 val_51 +402 val_403 +128 val_129 +468 val_469 +314 val_315 +224 val_225 +344 val_345 +4 val_5 +206 val_207 +114 val_115 +56 val_57 +114 val_115 +254 val_255 +390 val_391 +304 val_305 +264 val_265 +196 val_197 +238 val_239 +20 val_21 +492 val_493 +82 val_83 +58 val_59 +86 val_87 +438 val_439 +360 val_361 +222 val_223 +42 val_43 +338 val_339 +68 val_69 +16 val_17 +492 val_493 +376 val_377 +120 val_121 +306 val_307 +426 val_427 +132 val_133 +446 val_447 +386 val_387 +388 val_389 +184 val_185 +284 val_285 +246 val_247 +262 val_263 +122 val_123 +438 val_439 +390 val_391 +352 val_353 +226 val_227 +328 val_329 +382 val_383 +342 val_343 +480 val_481 +102 val_103 +480 val_481 +318 val_319 +392 val_393 +476 val_477 +258 val_259 +174 val_175 +252 val_253 +114 val_115 +264 val_265 +48 val_49 +336 val_337 +340 val_341 +390 val_391 +484 val_485 +6 val_7 +260 val_261 +2 val_3 +170 val_171 +164 val_165 +118 val_119 +310 val_311 +104 val_105 +80 val_81 +326 val_327 +450 val_451 +140 val_141 +212 val_213 +308 val_309 +30 val_31 +358 val_359 +416 val_417 +42 val_43 +386 val_387 +454 val_455 +364 val_365 +20 val_21 +52 val_53 +40 val_41 +8 val_9 +168 val_169 +384 val_385 +324 val_325 +310 val_311 +206 val_207 +404 val_405 +206 val_207 +226 val_227 +262 val_263 +260 val_261 +328 val_329 +322 val_323 +122 val_123 +404 val_405 +384 val_385 +76 val_77 +116 val_117 +42 val_43 +104 val_105 +406 val_407 +32 val_33 +132 val_133 +192 val_193 +58 val_59 +70 val_71 +356 val_357 +352 val_353 +52 val_53 +330 val_331 +138 val_139 +160 val_161 +454 val_455 +76 val_77 +174 val_175 +412 val_413 +16 val_17 +204 val_205 +126 val_127 +274 val_275 +374 val_375 +494 val_495 +216 val_217 +470 val_471 +196 val_197 +302 val_303 +450 val_451 +12 val_13 +398 val_399 +334 val_335 +384 val_385 +60 val_61 +442 val_443 +52 val_53 +404 val_405 +446 val_447 +300 val_301 +0 val_1 +268 val_269 +392 val_393 +104 val_105 +436 val_437 +156 val_157 +118 val_119 +172 val_173 +244 val_245 +6 val_7 +284 val_285 +164 val_165 +136 val_137 +462 val_463 +432 val_433 +496 val_497 +144 val_145 +408 val_409 +152 val_153 +382 val_383 +348 val_349 +122 val_123 +292 val_293 +182 val_183 +474 val_475 +310 val_311 +52 val_53 +486 val_487 +152 val_153 +378 val_379 +414 val_415 +256 val_257 +292 val_293 +412 val_413 +40 val_41 +478 val_479 +178 val_179 +100 val_101 +156 val_157 +228 val_229 +22 val_23 +248 val_249 +402 val_403 +62 val_63 +162 val_163 +244 val_245 +276 val_277 +46 val_47 +78 val_79 +134 val_135 +196 val_197 +410 val_411 +82 val_83 +440 val_441 +100 val_101 +308 val_309 +430 val_431 +468 val_469 +152 val_153 +138 val_139 +76 val_77 +300 val_301 +478 val_479 +118 val_119 +178 val_179 +242 val_243 +244 val_245 +238 val_238 +86 val_86 +278 val_278 +98 val_98 +484 val_484 +150 val_150 +224 val_224 +66 val_66 +128 val_128 +146 val_146 +406 val_406 +374 val_374 +152 val_152 +82 val_82 +166 val_166 +430 val_430 +252 val_252 +292 val_292 +338 val_338 +446 val_446 +394 val_394 +482 val_482 +174 val_174 +494 val_494 +466 val_466 +208 val_208 +174 val_174 +396 val_396 +162 val_162 +266 val_266 +342 val_342 +0 val_0 +128 val_128 +316 val_316 +302 val_302 +438 val_438 +170 val_170 +20 val_20 +378 val_378 +92 val_92 +72 val_72 +4 val_4 +280 val_280 +208 val_208 +356 val_356 +382 val_382 +498 val_498 +386 val_386 +192 val_192 +286 val_286 +176 val_176 +54 val_54 +138 val_138 +216 val_216 +430 val_430 +278 val_278 +176 val_176 +318 val_318 +332 val_332 +180 val_180 +284 val_284 +12 val_12 +230 val_230 +260 val_260 +404 val_404 +384 val_384 +272 val_272 +138 val_138 +84 val_84 +348 val_348 +466 val_466 +58 val_58 +8 val_8 +230 val_230 +208 val_208 +348 val_348 +24 val_24 +172 val_172 +42 val_42 +158 val_158 +496 val_496 +0 val_0 +322 val_322 +468 val_468 +454 val_454 +100 val_100 +298 val_298 +418 val_418 +96 val_96 +26 val_26 +230 val_230 +120 val_120 +404 val_404 +436 val_436 +156 val_156 +468 val_468 +308 val_308 +196 val_196 +288 val_288 +98 val_98 +282 val_282 +318 val_318 +318 val_318 +470 val_470 +316 val_316 +0 val_0 +490 val_490 +364 val_364 +118 val_118 +134 val_134 +282 val_282 +138 val_138 +238 val_238 +118 val_118 +72 val_72 +90 val_90 +10 val_10 +306 val_306 +224 val_224 +242 val_242 +392 val_392 +272 val_272 +242 val_242 +452 val_452 +226 val_226 +402 val_402 +396 val_396 +58 val_58 +336 val_336 +168 val_168 +34 val_34 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +42 val_42 +430 val_430 +458 val_458 +78 val_78 +76 val_76 +492 val_492 +218 val_218 +228 val_228 +138 val_138 +30 val_30 +64 val_64 +468 val_468 +76 val_76 +74 val_74 +342 val_342 +230 val_230 +368 val_368 +296 val_296 +216 val_216 +344 val_344 +274 val_274 +116 val_116 +256 val_256 +70 val_70 +480 val_480 +288 val_288 +244 val_244 +438 val_438 +128 val_128 +432 val_432 +202 val_202 +316 val_316 +280 val_280 +2 val_2 +80 val_80 +44 val_44 +104 val_104 +466 val_466 +366 val_366 +406 val_406 +190 val_190 +406 val_406 +114 val_114 +258 val_258 +90 val_90 +262 val_262 +348 val_348 +424 val_424 +12 val_12 +396 val_396 +164 val_164 +454 val_454 +478 val_478 +298 val_298 +164 val_164 +424 val_424 +382 val_382 +70 val_70 +480 val_480 +24 val_24 +104 val_104 +70 val_70 +438 val_438 +414 val_414 +200 val_200 +360 val_360 +248 val_248 +444 val_444 +120 val_120 +230 val_230 +478 val_478 +178 val_178 +468 val_468 +310 val_310 +460 val_460 +480 val_480 +136 val_136 +172 val_172 +214 val_214 +462 val_462 +406 val_406 +454 val_454 +384 val_384 +256 val_256 +26 val_26 +134 val_134 +384 val_384 +18 val_18 +462 val_462 +492 val_492 +100 val_100 +298 val_298 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +348 val_348 +18 val_18 +344 val_344 +84 val_84 +28 val_28 +448 val_448 +152 val_152 +348 val_348 +194 val_194 +414 val_414 +222 val_222 +126 val_126 +90 val_90 +400 val_400 +200 val_200