Index: ql/src/test/results/clientpositive/combine3.q.out =================================================================== --- ql/src/test/results/clientpositive/combine3.q.out (revision 0) +++ ql/src/test/results/clientpositive/combine3.q.out (revision 0) @@ -0,0 +1,1090 @@ +PREHOOK: query: drop table combine_3_srcpart_seq_rc +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table combine_3_srcpart_seq_rc +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table combine_3_srcpart_seq_rc (key int , value string) partitioned by (ds string, hr string) stored as sequencefile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table combine_3_srcpart_seq_rc (key int , value string) partitioned by (ds string, hr string) stored as sequencefile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@combine_3_srcpart_seq_rc +PREHOOK: query: insert overwrite table combine_3_srcpart_seq_rc partition (ds="2010-08-03", hr="00") select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@combine_3_srcpart_seq_rc@ds=2010-08-03/hr=00 +POSTHOOK: query: insert overwrite table combine_3_srcpart_seq_rc partition (ds="2010-08-03", hr="00") select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@combine_3_srcpart_seq_rc@ds=2010-08-03/hr=00 +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: alter table combine_3_srcpart_seq_rc set fileformat rcfile +PREHOOK: type: null +POSTHOOK: query: alter table combine_3_srcpart_seq_rc set fileformat rcfile +POSTHOOK: type: null +POSTHOOK: Input: default@combine_3_srcpart_seq_rc +POSTHOOK: Output: default@combine_3_srcpart_seq_rc +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table combine_3_srcpart_seq_rc partition (ds="2010-08-03", hr="001") select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@combine_3_srcpart_seq_rc@ds=2010-08-03/hr=001 +POSTHOOK: query: insert overwrite table combine_3_srcpart_seq_rc partition (ds="2010-08-03", hr="001") select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@combine_3_srcpart_seq_rc@ds=2010-08-03/hr=001 +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc extended combine_3_srcpart_seq_rc partition(ds="2010-08-03", hr="00") +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc extended combine_3_srcpart_seq_rc partition(ds="2010-08-03", hr="00") +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +key int +value string +ds string +hr string + +Detailed Partition Information Partition(values:[2010-08-03, 00], dbName:default, tableName:combine_3_srcpart_seq_rc, createTime:1280883808, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:pfile:/Users/heyongqiang/Documents/workspace/Hive-2/build/ql/test/data/warehouse/combine_3_srcpart_seq_rc/ds=2010-08-03/hr=00, inputFormat:org.apache.hadoop.mapred.SequenceFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{transient_lastDdlTime=1280883808}) +PREHOOK: query: desc extended combine_3_srcpart_seq_rc partition(ds="2010-08-03", hr="001") +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc extended combine_3_srcpart_seq_rc partition(ds="2010-08-03", hr="001") +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +key int +value string +ds string +hr string + +Detailed Partition Information Partition(values:[2010-08-03, 001], dbName:default, tableName:combine_3_srcpart_seq_rc, createTime:1280883813, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:pfile:/Users/heyongqiang/Documents/workspace/Hive-2/build/ql/test/data/warehouse/combine_3_srcpart_seq_rc/ds=2010-08-03/hr=001, inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{transient_lastDdlTime=1280883813}) +PREHOOK: query: select key, value from combine_3_srcpart_seq_rc where ds="2010-08-03" order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@combine_3_srcpart_seq_rc@ds=2010-08-03/hr=00 +PREHOOK: Input: default@combine_3_srcpart_seq_rc@ds=2010-08-03/hr=001 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-08-03_18-03-33_949_4148217782176646366/-mr-10000 +POSTHOOK: query: select key, value from combine_3_srcpart_seq_rc where ds="2010-08-03" order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@combine_3_srcpart_seq_rc@ds=2010-08-03/hr=00 +POSTHOOK: Input: default@combine_3_srcpart_seq_rc@ds=2010-08-03/hr=001 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-08-03_18-03-33_949_4148217782176646366/-mr-10000 +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +0 val_0 +2 val_2 +2 val_2 +4 val_4 +4 val_4 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +5 val_5 +8 val_8 +8 val_8 +9 val_9 +9 val_9 +10 val_10 +10 val_10 +11 val_11 +11 val_11 +12 val_12 +12 val_12 +12 val_12 +12 val_12 +15 val_15 +15 val_15 +15 val_15 +15 val_15 +17 val_17 +17 val_17 +18 val_18 +18 val_18 +18 val_18 +18 val_18 +19 val_19 +19 val_19 +20 val_20 +20 val_20 +24 val_24 +24 val_24 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +26 val_26 +26 val_26 +27 val_27 +27 val_27 +28 val_28 +28 val_28 +30 val_30 +30 val_30 +33 val_33 +33 val_33 +34 val_34 +34 val_34 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +35 val_35 +37 val_37 +37 val_37 +37 val_37 +37 val_37 +41 val_41 +41 val_41 +42 val_42 +42 val_42 +42 val_42 +42 val_42 +43 val_43 +43 val_43 +44 val_44 +44 val_44 +47 val_47 +47 val_47 +51 val_51 +51 val_51 +51 val_51 +51 val_51 +53 val_53 +53 val_53 +54 val_54 +54 val_54 +57 val_57 +57 val_57 +58 val_58 +58 val_58 +58 val_58 +58 val_58 +64 val_64 +64 val_64 +65 val_65 +65 val_65 +66 val_66 +66 val_66 +67 val_67 +67 val_67 +67 val_67 +67 val_67 +69 val_69 +69 val_69 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +70 val_70 +72 val_72 +72 val_72 +72 val_72 +72 val_72 +74 val_74 +74 val_74 +76 val_76 +76 val_76 +76 val_76 +76 val_76 +77 val_77 +77 val_77 +78 val_78 +78 val_78 +80 val_80 +80 val_80 +82 val_82 +82 val_82 +83 val_83 +83 val_83 +83 val_83 +83 val_83 +84 val_84 +84 val_84 +84 val_84 +84 val_84 +85 val_85 +85 val_85 +86 val_86 +86 val_86 +87 val_87 +87 val_87 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +90 val_90 +92 val_92 +92 val_92 +95 val_95 +95 val_95 +95 val_95 +95 val_95 +96 val_96 +96 val_96 +97 val_97 +97 val_97 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +98 val_98 +98 val_98 +100 val_100 +100 val_100 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +104 val_104 +104 val_104 +105 val_105 +105 val_105 +111 val_111 +111 val_111 +113 val_113 +113 val_113 +113 val_113 +113 val_113 +114 val_114 +114 val_114 +116 val_116 +116 val_116 +118 val_118 +118 val_118 +118 val_118 +118 val_118 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +120 val_120 +120 val_120 +120 val_120 +120 val_120 +125 val_125 +125 val_125 +125 val_125 +125 val_125 +126 val_126 +126 val_126 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +128 val_128 +129 val_129 +129 val_129 +129 val_129 +129 val_129 +131 val_131 +131 val_131 +133 val_133 +133 val_133 +134 val_134 +134 val_134 +134 val_134 +134 val_134 +136 val_136 +136 val_136 +137 val_137 +137 val_137 +137 val_137 +137 val_137 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +138 val_138 +143 val_143 +143 val_143 +145 val_145 +145 val_145 +146 val_146 +146 val_146 +146 val_146 +146 val_146 +149 val_149 +149 val_149 +149 val_149 +149 val_149 +150 val_150 +150 val_150 +152 val_152 +152 val_152 +152 val_152 +152 val_152 +153 val_153 +153 val_153 +155 val_155 +155 val_155 +156 val_156 +156 val_156 +157 val_157 +157 val_157 +158 val_158 +158 val_158 +160 val_160 +160 val_160 +162 val_162 +162 val_162 +163 val_163 +163 val_163 +164 val_164 +164 val_164 +164 val_164 +164 val_164 +165 val_165 +165 val_165 +165 val_165 +165 val_165 +166 val_166 +166 val_166 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +167 val_167 +168 val_168 +168 val_168 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +170 val_170 +170 val_170 +172 val_172 +172 val_172 +172 val_172 +172 val_172 +174 val_174 +174 val_174 +174 val_174 +174 val_174 +175 val_175 +175 val_175 +175 val_175 +175 val_175 +176 val_176 +176 val_176 +176 val_176 +176 val_176 +177 val_177 +177 val_177 +178 val_178 +178 val_178 +179 val_179 +179 val_179 +179 val_179 +179 val_179 +180 val_180 +180 val_180 +181 val_181 +181 val_181 +183 val_183 +183 val_183 +186 val_186 +186 val_186 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +187 val_187 +189 val_189 +189 val_189 +190 val_190 +190 val_190 +191 val_191 +191 val_191 +191 val_191 +191 val_191 +192 val_192 +192 val_192 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +193 val_193 +194 val_194 +194 val_194 +195 val_195 +195 val_195 +195 val_195 +195 val_195 +196 val_196 +196 val_196 +197 val_197 +197 val_197 +197 val_197 +197 val_197 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +199 val_199 +200 val_200 +200 val_200 +200 val_200 +200 val_200 +201 val_201 +201 val_201 +202 val_202 +202 val_202 +203 val_203 +203 val_203 +203 val_203 +203 val_203 +205 val_205 +205 val_205 +205 val_205 +205 val_205 +207 val_207 +207 val_207 +207 val_207 +207 val_207 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +208 val_208 +209 val_209 +209 val_209 +209 val_209 +209 val_209 +213 val_213 +213 val_213 +213 val_213 +213 val_213 +214 val_214 +214 val_214 +216 val_216 +216 val_216 +216 val_216 +216 val_216 +217 val_217 +217 val_217 +217 val_217 +217 val_217 +218 val_218 +218 val_218 +219 val_219 +219 val_219 +219 val_219 +219 val_219 +221 val_221 +221 val_221 +221 val_221 +221 val_221 +222 val_222 +222 val_222 +223 val_223 +223 val_223 +223 val_223 +223 val_223 +224 val_224 +224 val_224 +224 val_224 +224 val_224 +226 val_226 +226 val_226 +228 val_228 +228 val_228 +229 val_229 +229 val_229 +229 val_229 +229 val_229 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +230 val_230 +233 val_233 +233 val_233 +233 val_233 +233 val_233 +235 val_235 +235 val_235 +237 val_237 +237 val_237 +237 val_237 +237 val_237 +238 val_238 +238 val_238 +238 val_238 +238 val_238 +239 val_239 +239 val_239 +239 val_239 +239 val_239 +241 val_241 +241 val_241 +242 val_242 +242 val_242 +242 val_242 +242 val_242 +244 val_244 +244 val_244 +247 val_247 +247 val_247 +248 val_248 +248 val_248 +249 val_249 +249 val_249 +252 val_252 +252 val_252 +255 val_255 +255 val_255 +255 val_255 +255 val_255 +256 val_256 +256 val_256 +256 val_256 +256 val_256 +257 val_257 +257 val_257 +258 val_258 +258 val_258 +260 val_260 +260 val_260 +262 val_262 +262 val_262 +263 val_263 +263 val_263 +265 val_265 +265 val_265 +265 val_265 +265 val_265 +266 val_266 +266 val_266 +272 val_272 +272 val_272 +272 val_272 +272 val_272 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +273 val_273 +274 val_274 +274 val_274 +275 val_275 +275 val_275 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +277 val_277 +278 val_278 +278 val_278 +278 val_278 +278 val_278 +280 val_280 +280 val_280 +280 val_280 +280 val_280 +281 val_281 +281 val_281 +281 val_281 +281 val_281 +282 val_282 +282 val_282 +282 val_282 +282 val_282 +283 val_283 +283 val_283 +284 val_284 +284 val_284 +285 val_285 +285 val_285 +286 val_286 +286 val_286 +287 val_287 +287 val_287 +288 val_288 +288 val_288 +288 val_288 +288 val_288 +289 val_289 +289 val_289 +291 val_291 +291 val_291 +292 val_292 +292 val_292 +296 val_296 +296 val_296 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +298 val_298 +302 val_302 +302 val_302 +305 val_305 +305 val_305 +306 val_306 +306 val_306 +307 val_307 +307 val_307 +307 val_307 +307 val_307 +308 val_308 +308 val_308 +309 val_309 +309 val_309 +309 val_309 +309 val_309 +310 val_310 +310 val_310 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +311 val_311 +315 val_315 +315 val_315 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +316 val_316 +317 val_317 +317 val_317 +317 val_317 +317 val_317 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +318 val_318 +321 val_321 +321 val_321 +321 val_321 +321 val_321 +322 val_322 +322 val_322 +322 val_322 +322 val_322 +323 val_323 +323 val_323 +325 val_325 +325 val_325 +325 val_325 +325 val_325 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +327 val_327 +331 val_331 +331 val_331 +331 val_331 +331 val_331 +332 val_332 +332 val_332 +333 val_333 +333 val_333 +333 val_333 +333 val_333 +335 val_335 +335 val_335 +336 val_336 +336 val_336 +338 val_338 +338 val_338 +339 val_339 +339 val_339 +341 val_341 +341 val_341 +342 val_342 +342 val_342 +342 val_342 +342 val_342 +344 val_344 +344 val_344 +344 val_344 +344 val_344 +345 val_345 +345 val_345 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +348 val_348 +351 val_351 +351 val_351 +353 val_353 +353 val_353 +353 val_353 +353 val_353 +356 val_356 +356 val_356 +360 val_360 +360 val_360 +362 val_362 +362 val_362 +364 val_364 +364 val_364 +365 val_365 +365 val_365 +366 val_366 +366 val_366 +367 val_367 +367 val_367 +367 val_367 +367 val_367 +368 val_368 +368 val_368 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +369 val_369 +373 val_373 +373 val_373 +374 val_374 +374 val_374 +375 val_375 +375 val_375 +377 val_377 +377 val_377 +378 val_378 +378 val_378 +379 val_379 +379 val_379 +382 val_382 +382 val_382 +382 val_382 +382 val_382 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +384 val_384 +386 val_386 +386 val_386 +389 val_389 +389 val_389 +392 val_392 +392 val_392 +393 val_393 +393 val_393 +394 val_394 +394 val_394 +395 val_395 +395 val_395 +395 val_395 +395 val_395 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +396 val_396 +397 val_397 +397 val_397 +397 val_397 +397 val_397 +399 val_399 +399 val_399 +399 val_399 +399 val_399 +400 val_400 +400 val_400 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +401 val_401 +402 val_402 +402 val_402 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +403 val_403 +404 val_404 +404 val_404 +404 val_404 +404 val_404 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +406 val_406 +407 val_407 +407 val_407 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +409 val_409 +411 val_411 +411 val_411 +413 val_413 +413 val_413 +413 val_413 +413 val_413 +414 val_414 +414 val_414 +414 val_414 +414 val_414 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +417 val_417 +418 val_418 +418 val_418 +419 val_419 +419 val_419 +421 val_421 +421 val_421 +424 val_424 +424 val_424 +424 val_424 +424 val_424 +427 val_427 +427 val_427 +429 val_429 +429 val_429 +429 val_429 +429 val_429 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +430 val_430 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +431 val_431 +432 val_432 +432 val_432 +435 val_435 +435 val_435 +436 val_436 +436 val_436 +437 val_437 +437 val_437 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +438 val_438 +439 val_439 +439 val_439 +439 val_439 +439 val_439 +443 val_443 +443 val_443 +444 val_444 +444 val_444 +446 val_446 +446 val_446 +448 val_448 +448 val_448 +449 val_449 +449 val_449 +452 val_452 +452 val_452 +453 val_453 +453 val_453 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +454 val_454 +455 val_455 +455 val_455 +457 val_457 +457 val_457 +458 val_458 +458 val_458 +458 val_458 +458 val_458 +459 val_459 +459 val_459 +459 val_459 +459 val_459 +460 val_460 +460 val_460 +462 val_462 +462 val_462 +462 val_462 +462 val_462 +463 val_463 +463 val_463 +463 val_463 +463 val_463 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +466 val_466 +467 val_467 +467 val_467 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +468 val_468 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +469 val_469 +470 val_470 +470 val_470 +472 val_472 +472 val_472 +475 val_475 +475 val_475 +477 val_477 +477 val_477 +478 val_478 +478 val_478 +478 val_478 +478 val_478 +479 val_479 +479 val_479 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +480 val_480 +481 val_481 +481 val_481 +482 val_482 +482 val_482 +483 val_483 +483 val_483 +484 val_484 +484 val_484 +485 val_485 +485 val_485 +487 val_487 +487 val_487 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +489 val_489 +490 val_490 +490 val_490 +491 val_491 +491 val_491 +492 val_492 +492 val_492 +492 val_492 +492 val_492 +493 val_493 +493 val_493 +494 val_494 +494 val_494 +495 val_495 +495 val_495 +496 val_496 +496 val_496 +497 val_497 +497 val_497 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +498 val_498 +PREHOOK: query: drop table combine_3_srcpart_seq_rc +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table combine_3_srcpart_seq_rc +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@combine_3_srcpart_seq_rc +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=001).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: combine_3_srcpart_seq_rc PARTITION(ds=2010-08-03,hr=00).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Index: ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveFileFormatUtils.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveFileFormatUtils.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveFileFormatUtils.java (revision 0) @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.plan.PartitionDesc; + +public class TestHiveFileFormatUtils extends TestCase { + + public void testGetPartitionDescFromPathRecursively() throws IOException { + + PartitionDesc partDesc_1 = new PartitionDesc(); + Map pathToPartitionInfo = new HashMap(); + + pathToPartitionInfo.put( + new Path("file:///tbl/par1/part2/part3").toString(), partDesc_1); + pathToPartitionInfo.put(new Path("/tbl/par1/part2/part4").toString(), + partDesc_1); + pathToPartitionInfo.put(new Path("/tbl/par1/part2/part5/").toString(), + partDesc_1); + pathToPartitionInfo.put(new Path("hdfs:///tbl/par1/part2/part6/") + .toString(), partDesc_1); + + // first group + PartitionDesc ret = HiveFileFormatUtils + .getPartitionDescFromPathRecursively(pathToPartitionInfo, new Path( + "file:///tbl/par1/part2/part3")); + assertEquals("file:///tbl/par1/part2/part3 not found.", partDesc_1, ret); + + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("/tbl/par1/part2/part3")); + assertEquals("/tbl/par1/part2/part3 not found.", partDesc_1, ret); + + boolean exception = false; + try { + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("hdfs:///tbl/par1/part2/part3")); + } catch (IOException e) { + exception = true; + } + assertEquals("hdfs:///tbl/par1/part2/part3 should return null", true, + exception); + exception = false; + + // second group + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("file:///tbl/par1/part2/part4")); + assertEquals("file:///tbl/par1/part2/part4 not found.", partDesc_1, ret); + + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("/tbl/par1/part2/part4")); + assertEquals("/tbl/par1/part2/part4 not found.", partDesc_1, ret); + + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("hdfs:///tbl/par1/part2/part4")); + + assertEquals("hdfs:///tbl/par1/part2/part4 should not found", partDesc_1, + ret); + + // third group + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("file:///tbl/par1/part2/part5")); + assertEquals("file:///tbl/par1/part2/part5 not found.", partDesc_1, ret); + + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("/tbl/par1/part2/part5")); + assertEquals("/tbl/par1/part2/part5 not found.", partDesc_1, ret); + + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("hdfs:///tbl/par1/part2/part5")); + assertEquals("hdfs:///tbl/par1/part2/part5 not found", partDesc_1, ret); + + // forth group + try { + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("file:///tbl/par1/part2/part6")); + } catch (IOException e) { + exception = true; + } + assertEquals("file:///tbl/par1/part2/part6 should return null", true, + exception); + exception = false; + + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("/tbl/par1/part2/part6")); + assertEquals("/tbl/par1/part2/part6 not found.", partDesc_1, ret); + + ret = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + pathToPartitionInfo, new Path("hdfs:///tbl/par1/part2/part6")); + assertEquals("hdfs:///tbl/par1/part2/part6 not found.", partDesc_1, ret); + + } + +} Index: ql/src/test/queries/clientpositive/combine3.q =================================================================== --- ql/src/test/queries/clientpositive/combine3.q (revision 0) +++ ql/src/test/queries/clientpositive/combine3.q (revision 0) @@ -0,0 +1,17 @@ +set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; + +drop table combine_3_srcpart_seq_rc; + +create table combine_3_srcpart_seq_rc (key int , value string) partitioned by (ds string, hr string) stored as sequencefile; + +insert overwrite table combine_3_srcpart_seq_rc partition (ds="2010-08-03", hr="00") select * from src; + +alter table combine_3_srcpart_seq_rc set fileformat rcfile; +insert overwrite table combine_3_srcpart_seq_rc partition (ds="2010-08-03", hr="001") select * from src; + +desc extended combine_3_srcpart_seq_rc partition(ds="2010-08-03", hr="00"); +desc extended combine_3_srcpart_seq_rc partition(ds="2010-08-03", hr="001"); + +select key, value from combine_3_srcpart_seq_rc where ds="2010-08-03" order by key; + +drop table combine_3_srcpart_seq_rc; Index: ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (revision 982003) +++ ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (working copy) @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.Map; @@ -97,7 +98,7 @@ // CombinedSplit. Path[] ipaths = inputSplitShim.getPaths(); if (ipaths.length > 0) { - PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, ipaths[0]); + PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively(pathToPartitionInfo, ipaths[0]); inputFormatClassName = part.getInputFileFormatClass().getName(); } } @@ -198,7 +199,7 @@ // extract all the inputFormatClass names for each chunk in the // CombinedSplit. - PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, + PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively(pathToPartitionInfo, inputSplitShim.getPath(0)); // create a new InputFormat instance if this is the first time to see @@ -235,7 +236,7 @@ Set poolSet = new HashSet(); for (Path path : paths) { - PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, path); + PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively(pathToPartitionInfo, path); TableDesc tableDesc = part.getTableDesc(); if ((tableDesc != null) && tableDesc.isNonNative()) { return super.getSplits(job, numSplits); @@ -346,64 +347,6 @@ CombineHiveRecordReader.class); } - protected static PartitionDesc getPartitionDescFromPath( - Map pathToPartitionInfo, Path dir) throws IOException { - - // We first do exact match, and then do prefix matching. The latter is due to input dir - // could be /dir/ds='2001-02-21'/part-03 where part-03 is not part of partition - String dirPath = dir.toUri().getPath(); - PartitionDesc part = pathToPartitionInfo.get(dir.toString()); - if (part == null) { - // LOG.warn("exact match not found, try ripping input path's theme and authority"); - part = pathToPartitionInfo.get(dirPath); - } - if (part == null) { - - // LOG.warn("still does not found just the path part: " + dirPath + " in pathToPartitionInfo." - // + " Will try prefix matching"); - for (Map.Entry entry : pathToPartitionInfo.entrySet()) { - String keyPath = entry.getKey(); - String dirStr = dir.toString(); - // keyPath could start with hdfs:// or not, so we need to match both cases. - if (dirStr.startsWith(keyPath)) { - part = entry.getValue(); - break; - } else { - Path p = new Path(keyPath); - String newP = p.toUri().getPath().toString(); - if (dirStr.startsWith(newP)) { - part = entry.getValue(); - break; - } - // This case handles the situation where dir is a fully qualified - // subdirectory of a path in pathToPartitionInfo. e.g. - // dir = hdfs://host:9000/user/warehouse/tableName/abc - // pathToPartitionInfo = {/user/warehouse/tableName : myPart} - // In such a case, just compare the path components. - - // This could result in aliasing if we have a case where - // two entries in pathToPartitionInfo differ only by scheme - // or authority, but this problem exists anyway in the above checks. - - // This check was precipitated by changes that allow recursive dirs - // in the input path, and an upcoming change to CombineFileInputFormat - // where the paths in splits no longer have the scheme and authority - // stripped out. - if (dirPath.startsWith(newP)) { - part = entry.getValue(); - break; - } - } - } - } - if (part != null) { - return part; - } else { - throw new IOException("cannot find dir = " + dir.toString() - + " in partToPartitionInfo: " + pathToPartitionInfo.keySet()); - } - } - static class CombineFilter implements PathFilter { private final String pString; Index: ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (revision 982003) +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (working copy) @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.io; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; +import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.Writable; @@ -67,6 +69,9 @@ private static Map, Class> outputFormatSubstituteMap; + private static Map, Map> cacheMap = + new HashMap, Map>(); + /** * register a substitute. * @@ -242,7 +247,71 @@ } return null; } + + public static PartitionDesc getPartitionDescFromPathRecursively( + Map pathToPartitionInfo, Path dir) throws IOException { + PartitionDesc part = doGetPartitionDescFromPath(pathToPartitionInfo, dir); + if (part == null + && (dir.toUri().getScheme() == null || dir.toUri().getScheme().trim() + .equals(""))) { + Map newPathToPartitionInfo = cacheMap.get(pathToPartitionInfo); + if(newPathToPartitionInfo == null) { + newPathToPartitionInfo = new HashMap(); + for (Map.Entry entry: pathToPartitionInfo.entrySet()) { + String entryKey = entry.getKey(); + PartitionDesc partDesc = entry.getValue(); + Path newP = new Path(entryKey); + String pathOnly = newP.toUri().getPath(); + newPathToPartitionInfo.put(pathOnly, partDesc); + } + cacheMap.put(pathToPartitionInfo, newPathToPartitionInfo); + } + part = doGetPartitionDescFromPath(newPathToPartitionInfo, dir); + } + + if (part != null) { + return part; + } else { + throw new IOException("cannot find dir = " + dir.toString() + + " in partToPartitionInfo: " + pathToPartitionInfo.keySet()); + } + } + + private static PartitionDesc doGetPartitionDescFromPath( + Map pathToPartitionInfo, Path dir) { + // We first do exact match, and then do prefix matching. The latter is due to input dir + // could be /dir/ds='2001-02-21'/part-03 where part-03 is not part of partition + String dirPath = dir.toUri().getPath(); + PartitionDesc part = pathToPartitionInfo.get(dir.toString()); + if (part == null) { + // LOG.warn("exact match not found, try ripping input path's theme and authority"); + part = pathToPartitionInfo.get(dirPath); + } + + if (part == null) { + String dirStr = dir.toString(); + int dirPathIndex = dirPath.lastIndexOf(File.separator); + int dirStrIndex = dirStr.lastIndexOf(File.separator); + while (dirPathIndex >= 0 && dirStrIndex >= 0) { + dirStr = dirStr.substring(0, dirStrIndex); + dirPath = dirPath.substring(0, dirPathIndex); + //first try full match + part = pathToPartitionInfo.get(dirStr); + if (part == null) { + // LOG.warn("exact match not found, try ripping input path's theme and authority"); + part = pathToPartitionInfo.get(dirPath); + } + if (part != null) { + break; + } + dirPathIndex = dirPath.lastIndexOf(File.separator); + dirStrIndex = dirStr.lastIndexOf(File.separator); + } + } + return part; + } + private HiveFileFormatUtils() { // prevent instantiation }