Index: ql/src/test/results/clientpositive/create_idx_immediate.q.out =================================================================== --- ql/src/test/results/clientpositive/create_idx_immediate.q.out (revision 0) +++ ql/src/test/results/clientpositive/create_idx_immediate.q.out (revision 0) @@ -0,0 +1,1881 @@ +PREHOOK: query: drop index src_index_2 on src +PREHOOK: type: DROPINDEX +POSTHOOK: query: drop index src_index_2 on src +POSTHOOK: type: DROPINDEX +PREHOOK: query: create index src_index_2 on table src(key) as 'compact' +PREHOOK: type: CREATEINDEX +PREHOOK: Input: default@src +POSTHOOK: query: create index src_index_2 on table src(key) as 'compact' +POSTHOOK: type: CREATEINDEX +POSTHOOK: Input: default@src +POSTHOOK: Output: default@default__src_src_index_2__ +PREHOOK: query: desc extended default__src_src_index_2__ +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc extended default__src_src_index_2__ +POSTHOOK: type: DESCTABLE +key string default +_bucketname string +_offsets array + +#### A masked pattern was here #### +PREHOOK: query: select key,`_offsets` from default__src_src_index_2__ +PREHOOK: type: QUERY +PREHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +POSTHOOK: query: select key,`_offsets` from default__src_src_index_2__ +POSTHOOK: type: QUERY +POSTHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +0 [968,2632,2088] +10 [2846] +100 [5374,2156] +103 [3614,1484] +104 [4628,4114] +105 [4196] +11 [3170] +111 [1186] +113 [920,3638] +114 [4280] +116 [3746] +118 [2780,2686] +119 [2064,4674,3332] +12 [4362,1720] +120 [2284,4830] +125 [4468,1344] +126 [5732] +128 [3896,988,208] +129 [2040,1094] +131 [2296] +133 [5164] +134 [2698,5294] +136 [5080] +137 [2552,1650] +138 [2734,1848,1472,3470] +143 [3226] +145 [304] +146 [232,5430] +149 [3422,1058] +15 [386,2770] +150 [150] +152 [280,5648] +153 [502] +155 [932] +156 [2352] +157 [1140] +158 [2052] +160 [3274] +162 [754] +163 [4650] +164 [4492,4408] +165 [2236,44] +166 [418] +167 [5502,874,3686] +168 [3180] +169 [5754,1308,2588,4854] +17 [910] +170 [1106] +172 [2018,5104] +174 [682,598] +175 [5176,4150] +176 [1428,1556] +177 [3036] +178 [4938] +179 [2674,2006] +18 [5514,5340] +180 [1696] +181 [1742] +183 [5536] +186 [5466] +187 [2492,1416,4516] +189 [5188] +19 [2824] +190 [4244] +191 [3852,2192] +192 [1392] +193 [514,126,4078] +194 [5684] +195 [3286,886] +196 [2410] +197 [2480,2108] +199 [646,2180,4806] +2 [4004] +20 [1118] +200 [4698,5790] +201 [4384] +202 [3932] +203 [4314,944] +205 [1046,2272] +207 [5022,634] +208 [670,1948,1272] +209 [3504,374] +213 [220,1508] +214 [5116] +216 [1520,3650] +217 [4396,1860] +218 [3446] +219 [478,3710] +221 [1580,1164] +222 [5720] +223 [3398,3758] +224 [174,2892] +226 [3048] +228 [3458] +229 [3202,3956] +230 [1730,1936,4914,2260,3580] +233 [5140,3214] +235 [4046] +237 [4722,574] +238 [0,2746] +239 [1496,3722] +24 [1972,4594] +241 [1662] +242 [2940,3012] +244 [3872] +247 [718] +248 [4758] +249 [5034] +252 [454] +255 [68,4616] +256 [5272,3770] +257 [4208] +258 [4292] +26 [5284,2226] +260 [1764] +262 [4326] +263 [3782] +265 [114,5046] +266 [814] +27 [34] +272 [1836,2976] +273 [2868,5524,162] +274 [3698] +275 [1638] +277 [4902,1260,2856,362] +278 [1544,80] +28 [5616] +280 [3992,1226] +281 [350,5548] +282 [2468,2722] +283 [4022] +284 [1708] +285 [5478] +286 [1404] +287 [490] +288 [2422,3840] +289 [1568] +291 [4582] +292 [466] +296 [3626] +298 [5386,4456,2168] +30 [3494] +302 [1034] +305 [4782] +306 [2880] +307 [2812,5672] +308 [2388] +309 [790,2904] +310 [4962] +311 [1000,1626,22] +315 [5594] +316 [3944,2576,1012] +317 [3104,4974] +318 [2504,1602,2516] +321 [4090,3308] +322 [3250,2096] +323 [4878] +325 [4890,862] +327 [2928,338,2248] +33 [3592] +331 [2988,4034] +332 [1614] +333 [1684,4986] +335 [4102] +336 [3148] +338 [526] +339 [956] +34 [3192] +341 [5406] +342 [3558,838] +344 [3674,5560] +345 [1082] +348 [5660,1882,1960,4338,5490] +35 [1238,4012,3138] +351 [4604] +353 [1812,5092] +356 [1284] +360 [4746] +362 [5454] +364 [2662] +365 [802] +366 [4138] +367 [850,3662] +368 [3602] +369 [186,2564,2952] +37 [328,5626] +373 [1824] +374 [268] +375 [5212] +377 [766] +378 [1152] +379 [5328] +382 [1320,4528] +384 [5316,5260,1788] +386 [1356] +389 [2916] +392 [2964] +393 [2132] +394 [562] +395 [3116,2710] +396 [4372,706,3092] +397 [4558,778] +399 [694,1296] +4 [1218] +400 [5778] +401 [138,3000,3828,4268,5224] +402 [3080] +403 [5766,4162,406] +404 [1776,2318] +406 [244,4220,4256,5152] +407 [5248] +409 [4232,56,2528] +41 [3388] +411 [1924] +413 [610,2600] +414 [5696,4686] +417 [730,4794,430] +418 [2204] +419 [2758] +42 [3298,2030] +421 [5236] +424 [4350,4504] +427 [1248] +429 [4842,256] +43 [2330] +430 [442,1532,3320] +431 [4420,1994,4480] +432 [3920] +435 [2834] +436 [2340] +437 [1368] +438 [3884,4662,1070] +439 [4734,826] +44 [4068] +443 [4866] +444 [4818] +446 [538] +448 [5636] +449 [3434] +452 [3024] +453 [3482] +454 [2144,4432,5200] +455 [976] +457 [2446] +458 [3356,5442] +459 [550,1450] +460 [5010] +462 [5350,5128] +463 [1982,3980] +466 [658,1894,4126] +467 [3908] +468 [3526,4950,2120,2376] +469 [292,3968,1380,5582,2364] +47 [1198] +470 [2540] +472 [3238] +475 [898] +477 [5708] +478 [4444,4926] +479 [4770] +480 [4570,5058,3816] +481 [2434] +482 [586] +483 [4174] +484 [102] +485 [3734] +487 [3804] +489 [1128,1800,3344,742] +490 [2640] +491 [4710] +492 [5362,3410] +493 [4998] +494 [622] +495 [316] +496 [2076] +497 [3068] +498 [5418,3262,1332] +5 [4540,3864,3060] +51 [1462,2308] +53 [4186] +54 [1440] +57 [1024] +58 [3128,1906] +64 [3516] +65 [1592] +66 [198] +67 [5306,1754] +69 [3570] +70 [3794,4640,4548] +72 [2792,1208] +74 [3548] +76 [3538,3378] +77 [2622] +78 [3368] +8 [1916] +80 [4058] +82 [396] +83 [5070,1674] +84 [5606,1872] +85 [2612] +86 [12] +87 [2652] +9 [5398] +90 [2802,5744,4304] +92 [1176] +95 [2400,3160] +96 [2216] +97 [5572,5802] +98 [2458,92] +PREHOOK: query: drop index src_index_2 on src +PREHOOK: type: DROPINDEX +POSTHOOK: query: drop index src_index_2 on src +POSTHOOK: type: DROPINDEX +PREHOOK: query: create index src_index_2 on table src(key) as 'compact' ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\\' +PREHOOK: type: CREATEINDEX +PREHOOK: Input: default@src +POSTHOOK: query: create index src_index_2 on table src(key) as 'compact' ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\\' +POSTHOOK: type: CREATEINDEX +POSTHOOK: Input: default@src +POSTHOOK: Output: default@default__src_src_index_2__ +PREHOOK: query: desc extended default__src_src_index_2__ +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc extended default__src_src_index_2__ +POSTHOOK: type: DESCTABLE +key string default +_bucketname string +_offsets array + +#### A masked pattern was here #### +PREHOOK: query: select key,`_offsets` from default__src_src_index_2__ +PREHOOK: type: QUERY +PREHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +POSTHOOK: query: select key,`_offsets` from default__src_src_index_2__ +POSTHOOK: type: QUERY +POSTHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +0 [968,2632,2088] +10 [2846] +100 [5374,2156] +103 [3614,1484] +104 [4628,4114] +105 [4196] +11 [3170] +111 [1186] +113 [920,3638] +114 [4280] +116 [3746] +118 [2780,2686] +119 [2064,4674,3332] +12 [4362,1720] +120 [2284,4830] +125 [4468,1344] +126 [5732] +128 [3896,988,208] +129 [2040,1094] +131 [2296] +133 [5164] +134 [2698,5294] +136 [5080] +137 [2552,1650] +138 [2734,1848,1472,3470] +143 [3226] +145 [304] +146 [232,5430] +149 [3422,1058] +15 [386,2770] +150 [150] +152 [280,5648] +153 [502] +155 [932] +156 [2352] +157 [1140] +158 [2052] +160 [3274] +162 [754] +163 [4650] +164 [4492,4408] +165 [2236,44] +166 [418] +167 [5502,874,3686] +168 [3180] +169 [5754,1308,2588,4854] +17 [910] +170 [1106] +172 [2018,5104] +174 [682,598] +175 [5176,4150] +176 [1428,1556] +177 [3036] +178 [4938] +179 [2674,2006] +18 [5514,5340] +180 [1696] +181 [1742] +183 [5536] +186 [5466] +187 [2492,1416,4516] +189 [5188] +19 [2824] +190 [4244] +191 [3852,2192] +192 [1392] +193 [514,126,4078] +194 [5684] +195 [3286,886] +196 [2410] +197 [2480,2108] +199 [646,2180,4806] +2 [4004] +20 [1118] +200 [4698,5790] +201 [4384] +202 [3932] +203 [4314,944] +205 [1046,2272] +207 [5022,634] +208 [670,1948,1272] +209 [3504,374] +213 [220,1508] +214 [5116] +216 [1520,3650] +217 [4396,1860] +218 [3446] +219 [478,3710] +221 [1580,1164] +222 [5720] +223 [3398,3758] +224 [174,2892] +226 [3048] +228 [3458] +229 [3202,3956] +230 [1730,1936,4914,2260,3580] +233 [5140,3214] +235 [4046] +237 [4722,574] +238 [0,2746] +239 [1496,3722] +24 [1972,4594] +241 [1662] +242 [2940,3012] +244 [3872] +247 [718] +248 [4758] +249 [5034] +252 [454] +255 [68,4616] +256 [5272,3770] +257 [4208] +258 [4292] +26 [5284,2226] +260 [1764] +262 [4326] +263 [3782] +265 [114,5046] +266 [814] +27 [34] +272 [1836,2976] +273 [2868,5524,162] +274 [3698] +275 [1638] +277 [4902,1260,2856,362] +278 [1544,80] +28 [5616] +280 [3992,1226] +281 [350,5548] +282 [2468,2722] +283 [4022] +284 [1708] +285 [5478] +286 [1404] +287 [490] +288 [2422,3840] +289 [1568] +291 [4582] +292 [466] +296 [3626] +298 [5386,4456,2168] +30 [3494] +302 [1034] +305 [4782] +306 [2880] +307 [2812,5672] +308 [2388] +309 [790,2904] +310 [4962] +311 [1000,1626,22] +315 [5594] +316 [3944,2576,1012] +317 [3104,4974] +318 [2504,1602,2516] +321 [4090,3308] +322 [3250,2096] +323 [4878] +325 [4890,862] +327 [2928,338,2248] +33 [3592] +331 [2988,4034] +332 [1614] +333 [1684,4986] +335 [4102] +336 [3148] +338 [526] +339 [956] +34 [3192] +341 [5406] +342 [3558,838] +344 [3674,5560] +345 [1082] +348 [5660,1882,1960,4338,5490] +35 [1238,4012,3138] +351 [4604] +353 [1812,5092] +356 [1284] +360 [4746] +362 [5454] +364 [2662] +365 [802] +366 [4138] +367 [850,3662] +368 [3602] +369 [186,2564,2952] +37 [328,5626] +373 [1824] +374 [268] +375 [5212] +377 [766] +378 [1152] +379 [5328] +382 [1320,4528] +384 [5316,5260,1788] +386 [1356] +389 [2916] +392 [2964] +393 [2132] +394 [562] +395 [3116,2710] +396 [4372,706,3092] +397 [4558,778] +399 [694,1296] +4 [1218] +400 [5778] +401 [138,3000,3828,4268,5224] +402 [3080] +403 [5766,4162,406] +404 [1776,2318] +406 [244,4220,4256,5152] +407 [5248] +409 [4232,56,2528] +41 [3388] +411 [1924] +413 [610,2600] +414 [5696,4686] +417 [730,4794,430] +418 [2204] +419 [2758] +42 [3298,2030] +421 [5236] +424 [4350,4504] +427 [1248] +429 [4842,256] +43 [2330] +430 [442,1532,3320] +431 [4420,1994,4480] +432 [3920] +435 [2834] +436 [2340] +437 [1368] +438 [3884,4662,1070] +439 [4734,826] +44 [4068] +443 [4866] +444 [4818] +446 [538] +448 [5636] +449 [3434] +452 [3024] +453 [3482] +454 [2144,4432,5200] +455 [976] +457 [2446] +458 [3356,5442] +459 [550,1450] +460 [5010] +462 [5350,5128] +463 [1982,3980] +466 [658,1894,4126] +467 [3908] +468 [3526,4950,2120,2376] +469 [292,3968,1380,5582,2364] +47 [1198] +470 [2540] +472 [3238] +475 [898] +477 [5708] +478 [4444,4926] +479 [4770] +480 [4570,5058,3816] +481 [2434] +482 [586] +483 [4174] +484 [102] +485 [3734] +487 [3804] +489 [1128,1800,3344,742] +490 [2640] +491 [4710] +492 [5362,3410] +493 [4998] +494 [622] +495 [316] +496 [2076] +497 [3068] +498 [5418,3262,1332] +5 [4540,3864,3060] +51 [1462,2308] +53 [4186] +54 [1440] +57 [1024] +58 [3128,1906] +64 [3516] +65 [1592] +66 [198] +67 [5306,1754] +69 [3570] +70 [3794,4640,4548] +72 [2792,1208] +74 [3548] +76 [3538,3378] +77 [2622] +78 [3368] +8 [1916] +80 [4058] +82 [396] +83 [5070,1674] +84 [5606,1872] +85 [2612] +86 [12] +87 [2652] +9 [5398] +90 [2802,5744,4304] +92 [1176] +95 [2400,3160] +96 [2216] +97 [5572,5802] +98 [2458,92] +PREHOOK: query: drop index src_index_2 on src +PREHOOK: type: DROPINDEX +POSTHOOK: query: drop index src_index_2 on src +POSTHOOK: type: DROPINDEX +PREHOOK: query: create index src_index_2 on table src(key) as 'compact' STORED AS TEXTFILE +PREHOOK: type: CREATEINDEX +PREHOOK: Input: default@src +POSTHOOK: query: create index src_index_2 on table src(key) as 'compact' STORED AS TEXTFILE +POSTHOOK: type: CREATEINDEX +POSTHOOK: Input: default@src +POSTHOOK: Output: default@default__src_src_index_2__ +PREHOOK: query: desc extended default__src_src_index_2__ +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc extended default__src_src_index_2__ +POSTHOOK: type: DESCTABLE +key string default +_bucketname string +_offsets array + +#### A masked pattern was here #### +PREHOOK: query: select key,`_offsets` from default__src_src_index_2__ +PREHOOK: type: QUERY +PREHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +POSTHOOK: query: select key,`_offsets` from default__src_src_index_2__ +POSTHOOK: type: QUERY +POSTHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +0 [968,2632,2088] +10 [2846] +100 [5374,2156] +103 [3614,1484] +104 [4628,4114] +105 [4196] +11 [3170] +111 [1186] +113 [920,3638] +114 [4280] +116 [3746] +118 [2780,2686] +119 [2064,4674,3332] +12 [4362,1720] +120 [2284,4830] +125 [4468,1344] +126 [5732] +128 [3896,988,208] +129 [2040,1094] +131 [2296] +133 [5164] +134 [2698,5294] +136 [5080] +137 [2552,1650] +138 [2734,1848,1472,3470] +143 [3226] +145 [304] +146 [232,5430] +149 [3422,1058] +15 [386,2770] +150 [150] +152 [280,5648] +153 [502] +155 [932] +156 [2352] +157 [1140] +158 [2052] +160 [3274] +162 [754] +163 [4650] +164 [4492,4408] +165 [2236,44] +166 [418] +167 [5502,874,3686] +168 [3180] +169 [5754,1308,2588,4854] +17 [910] +170 [1106] +172 [2018,5104] +174 [682,598] +175 [5176,4150] +176 [1428,1556] +177 [3036] +178 [4938] +179 [2674,2006] +18 [5514,5340] +180 [1696] +181 [1742] +183 [5536] +186 [5466] +187 [2492,1416,4516] +189 [5188] +19 [2824] +190 [4244] +191 [3852,2192] +192 [1392] +193 [514,126,4078] +194 [5684] +195 [3286,886] +196 [2410] +197 [2480,2108] +199 [646,2180,4806] +2 [4004] +20 [1118] +200 [4698,5790] +201 [4384] +202 [3932] +203 [4314,944] +205 [1046,2272] +207 [5022,634] +208 [670,1948,1272] +209 [3504,374] +213 [220,1508] +214 [5116] +216 [1520,3650] +217 [4396,1860] +218 [3446] +219 [478,3710] +221 [1580,1164] +222 [5720] +223 [3398,3758] +224 [174,2892] +226 [3048] +228 [3458] +229 [3202,3956] +230 [1730,1936,4914,2260,3580] +233 [5140,3214] +235 [4046] +237 [4722,574] +238 [0,2746] +239 [1496,3722] +24 [1972,4594] +241 [1662] +242 [2940,3012] +244 [3872] +247 [718] +248 [4758] +249 [5034] +252 [454] +255 [68,4616] +256 [5272,3770] +257 [4208] +258 [4292] +26 [5284,2226] +260 [1764] +262 [4326] +263 [3782] +265 [114,5046] +266 [814] +27 [34] +272 [1836,2976] +273 [2868,5524,162] +274 [3698] +275 [1638] +277 [4902,1260,2856,362] +278 [1544,80] +28 [5616] +280 [3992,1226] +281 [350,5548] +282 [2468,2722] +283 [4022] +284 [1708] +285 [5478] +286 [1404] +287 [490] +288 [2422,3840] +289 [1568] +291 [4582] +292 [466] +296 [3626] +298 [5386,4456,2168] +30 [3494] +302 [1034] +305 [4782] +306 [2880] +307 [2812,5672] +308 [2388] +309 [790,2904] +310 [4962] +311 [1000,1626,22] +315 [5594] +316 [3944,2576,1012] +317 [3104,4974] +318 [2504,1602,2516] +321 [4090,3308] +322 [3250,2096] +323 [4878] +325 [4890,862] +327 [2928,338,2248] +33 [3592] +331 [2988,4034] +332 [1614] +333 [1684,4986] +335 [4102] +336 [3148] +338 [526] +339 [956] +34 [3192] +341 [5406] +342 [3558,838] +344 [3674,5560] +345 [1082] +348 [5660,1882,1960,4338,5490] +35 [1238,4012,3138] +351 [4604] +353 [1812,5092] +356 [1284] +360 [4746] +362 [5454] +364 [2662] +365 [802] +366 [4138] +367 [850,3662] +368 [3602] +369 [186,2564,2952] +37 [328,5626] +373 [1824] +374 [268] +375 [5212] +377 [766] +378 [1152] +379 [5328] +382 [1320,4528] +384 [5316,5260,1788] +386 [1356] +389 [2916] +392 [2964] +393 [2132] +394 [562] +395 [3116,2710] +396 [4372,706,3092] +397 [4558,778] +399 [694,1296] +4 [1218] +400 [5778] +401 [138,3000,3828,4268,5224] +402 [3080] +403 [5766,4162,406] +404 [1776,2318] +406 [244,4220,4256,5152] +407 [5248] +409 [4232,56,2528] +41 [3388] +411 [1924] +413 [610,2600] +414 [5696,4686] +417 [730,4794,430] +418 [2204] +419 [2758] +42 [3298,2030] +421 [5236] +424 [4350,4504] +427 [1248] +429 [4842,256] +43 [2330] +430 [442,1532,3320] +431 [4420,1994,4480] +432 [3920] +435 [2834] +436 [2340] +437 [1368] +438 [3884,4662,1070] +439 [4734,826] +44 [4068] +443 [4866] +444 [4818] +446 [538] +448 [5636] +449 [3434] +452 [3024] +453 [3482] +454 [2144,4432,5200] +455 [976] +457 [2446] +458 [3356,5442] +459 [550,1450] +460 [5010] +462 [5350,5128] +463 [1982,3980] +466 [658,1894,4126] +467 [3908] +468 [3526,4950,2120,2376] +469 [292,3968,1380,5582,2364] +47 [1198] +470 [2540] +472 [3238] +475 [898] +477 [5708] +478 [4444,4926] +479 [4770] +480 [4570,5058,3816] +481 [2434] +482 [586] +483 [4174] +484 [102] +485 [3734] +487 [3804] +489 [1128,1800,3344,742] +490 [2640] +491 [4710] +492 [5362,3410] +493 [4998] +494 [622] +495 [316] +496 [2076] +497 [3068] +498 [5418,3262,1332] +5 [4540,3864,3060] +51 [1462,2308] +53 [4186] +54 [1440] +57 [1024] +58 [3128,1906] +64 [3516] +65 [1592] +66 [198] +67 [5306,1754] +69 [3570] +70 [3794,4640,4548] +72 [2792,1208] +74 [3548] +76 [3538,3378] +77 [2622] +78 [3368] +8 [1916] +80 [4058] +82 [396] +83 [5070,1674] +84 [5606,1872] +85 [2612] +86 [12] +87 [2652] +9 [5398] +90 [2802,5744,4304] +92 [1176] +95 [2400,3160] +96 [2216] +97 [5572,5802] +98 [2458,92] +PREHOOK: query: drop index src_index_2 on src +PREHOOK: type: DROPINDEX +POSTHOOK: query: drop index src_index_2 on src +POSTHOOK: type: DROPINDEX +PREHOOK: query: create index src_index_2 on table src(key) as 'compact' location '/user/hive/warehouse' +PREHOOK: type: CREATEINDEX +PREHOOK: Input: default@src +POSTHOOK: query: create index src_index_2 on table src(key) as 'compact' location '/user/hive/warehouse' +POSTHOOK: type: CREATEINDEX +POSTHOOK: Input: default@src +POSTHOOK: Output: default@default__src_src_index_2__ +PREHOOK: query: desc extended default__src_src_index_2__ +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc extended default__src_src_index_2__ +POSTHOOK: type: DESCTABLE +key string default +_bucketname string +_offsets array + +#### A masked pattern was here #### +PREHOOK: query: select key,`_offsets` from default__src_src_index_2__ +PREHOOK: type: QUERY +PREHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +POSTHOOK: query: select key,`_offsets` from default__src_src_index_2__ +POSTHOOK: type: QUERY +POSTHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +0 [968,2632,2088] +10 [2846] +100 [5374,2156] +103 [3614,1484] +104 [4628,4114] +105 [4196] +11 [3170] +111 [1186] +113 [920,3638] +114 [4280] +116 [3746] +118 [2780,2686] +119 [2064,4674,3332] +12 [4362,1720] +120 [2284,4830] +125 [4468,1344] +126 [5732] +128 [3896,988,208] +129 [2040,1094] +131 [2296] +133 [5164] +134 [2698,5294] +136 [5080] +137 [2552,1650] +138 [2734,1848,1472,3470] +143 [3226] +145 [304] +146 [232,5430] +149 [3422,1058] +15 [386,2770] +150 [150] +152 [280,5648] +153 [502] +155 [932] +156 [2352] +157 [1140] +158 [2052] +160 [3274] +162 [754] +163 [4650] +164 [4492,4408] +165 [2236,44] +166 [418] +167 [5502,874,3686] +168 [3180] +169 [5754,1308,2588,4854] +17 [910] +170 [1106] +172 [2018,5104] +174 [682,598] +175 [5176,4150] +176 [1428,1556] +177 [3036] +178 [4938] +179 [2674,2006] +18 [5514,5340] +180 [1696] +181 [1742] +183 [5536] +186 [5466] +187 [2492,1416,4516] +189 [5188] +19 [2824] +190 [4244] +191 [3852,2192] +192 [1392] +193 [514,126,4078] +194 [5684] +195 [3286,886] +196 [2410] +197 [2480,2108] +199 [646,2180,4806] +2 [4004] +20 [1118] +200 [4698,5790] +201 [4384] +202 [3932] +203 [4314,944] +205 [1046,2272] +207 [5022,634] +208 [670,1948,1272] +209 [3504,374] +213 [220,1508] +214 [5116] +216 [1520,3650] +217 [4396,1860] +218 [3446] +219 [478,3710] +221 [1580,1164] +222 [5720] +223 [3398,3758] +224 [174,2892] +226 [3048] +228 [3458] +229 [3202,3956] +230 [1730,1936,4914,2260,3580] +233 [5140,3214] +235 [4046] +237 [4722,574] +238 [0,2746] +239 [1496,3722] +24 [1972,4594] +241 [1662] +242 [2940,3012] +244 [3872] +247 [718] +248 [4758] +249 [5034] +252 [454] +255 [68,4616] +256 [5272,3770] +257 [4208] +258 [4292] +26 [5284,2226] +260 [1764] +262 [4326] +263 [3782] +265 [114,5046] +266 [814] +27 [34] +272 [1836,2976] +273 [2868,5524,162] +274 [3698] +275 [1638] +277 [4902,1260,2856,362] +278 [1544,80] +28 [5616] +280 [3992,1226] +281 [350,5548] +282 [2468,2722] +283 [4022] +284 [1708] +285 [5478] +286 [1404] +287 [490] +288 [2422,3840] +289 [1568] +291 [4582] +292 [466] +296 [3626] +298 [5386,4456,2168] +30 [3494] +302 [1034] +305 [4782] +306 [2880] +307 [2812,5672] +308 [2388] +309 [790,2904] +310 [4962] +311 [1000,1626,22] +315 [5594] +316 [3944,2576,1012] +317 [3104,4974] +318 [2504,1602,2516] +321 [4090,3308] +322 [3250,2096] +323 [4878] +325 [4890,862] +327 [2928,338,2248] +33 [3592] +331 [2988,4034] +332 [1614] +333 [1684,4986] +335 [4102] +336 [3148] +338 [526] +339 [956] +34 [3192] +341 [5406] +342 [3558,838] +344 [3674,5560] +345 [1082] +348 [5660,1882,1960,4338,5490] +35 [1238,4012,3138] +351 [4604] +353 [1812,5092] +356 [1284] +360 [4746] +362 [5454] +364 [2662] +365 [802] +366 [4138] +367 [850,3662] +368 [3602] +369 [186,2564,2952] +37 [328,5626] +373 [1824] +374 [268] +375 [5212] +377 [766] +378 [1152] +379 [5328] +382 [1320,4528] +384 [5316,5260,1788] +386 [1356] +389 [2916] +392 [2964] +393 [2132] +394 [562] +395 [3116,2710] +396 [4372,706,3092] +397 [4558,778] +399 [694,1296] +4 [1218] +400 [5778] +401 [138,3000,3828,4268,5224] +402 [3080] +403 [5766,4162,406] +404 [1776,2318] +406 [244,4220,4256,5152] +407 [5248] +409 [4232,56,2528] +41 [3388] +411 [1924] +413 [610,2600] +414 [5696,4686] +417 [730,4794,430] +418 [2204] +419 [2758] +42 [3298,2030] +421 [5236] +424 [4350,4504] +427 [1248] +429 [4842,256] +43 [2330] +430 [442,1532,3320] +431 [4420,1994,4480] +432 [3920] +435 [2834] +436 [2340] +437 [1368] +438 [3884,4662,1070] +439 [4734,826] +44 [4068] +443 [4866] +444 [4818] +446 [538] +448 [5636] +449 [3434] +452 [3024] +453 [3482] +454 [2144,4432,5200] +455 [976] +457 [2446] +458 [3356,5442] +459 [550,1450] +460 [5010] +462 [5350,5128] +463 [1982,3980] +466 [658,1894,4126] +467 [3908] +468 [3526,4950,2120,2376] +469 [292,3968,1380,5582,2364] +47 [1198] +470 [2540] +472 [3238] +475 [898] +477 [5708] +478 [4444,4926] +479 [4770] +480 [4570,5058,3816] +481 [2434] +482 [586] +483 [4174] +484 [102] +485 [3734] +487 [3804] +489 [1128,1800,3344,742] +490 [2640] +491 [4710] +492 [5362,3410] +493 [4998] +494 [622] +495 [316] +496 [2076] +497 [3068] +498 [5418,3262,1332] +5 [4540,3864,3060] +51 [1462,2308] +53 [4186] +54 [1440] +57 [1024] +58 [3128,1906] +64 [3516] +65 [1592] +66 [198] +67 [5306,1754] +69 [3570] +70 [3794,4640,4548] +72 [2792,1208] +74 [3548] +76 [3538,3378] +77 [2622] +78 [3368] +8 [1916] +80 [4058] +82 [396] +83 [5070,1674] +84 [5606,1872] +85 [2612] +86 [12] +87 [2652] +9 [5398] +90 [2802,5744,4304] +92 [1176] +95 [2400,3160] +96 [2216] +97 [5572,5802] +98 [2458,92] +PREHOOK: query: drop index src_index_2 on src +PREHOOK: type: DROPINDEX +POSTHOOK: query: drop index src_index_2 on src +POSTHOOK: type: DROPINDEX +PREHOOK: query: create index src_index_2 on table src(key) as 'bitmap' +PREHOOK: type: CREATEINDEX +PREHOOK: Input: default@src +POSTHOOK: query: create index src_index_2 on table src(key) as 'bitmap' +POSTHOOK: type: CREATEINDEX +POSTHOOK: Input: default@src +POSTHOOK: Output: default@default__src_src_index_2__ +PREHOOK: query: desc extended default__src_src_index_2__ +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc extended default__src_src_index_2__ +POSTHOOK: type: DESCTABLE +key string default +_bucketname string +_offset bigint +_bitmaps array + +#### A masked pattern was here #### +PREHOOK: query: select key,`_offset` from default__src_src_index_2__ +PREHOOK: type: QUERY +PREHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +POSTHOOK: query: select key,`_offset` from default__src_src_index_2__ +POSTHOOK: type: QUERY +POSTHOOK: Input: default@default__src_src_index_2__ +#### A masked pattern was here #### +238 0 +86 12 +311 22 +27 34 +165 44 +409 56 +255 68 +278 80 +98 92 +484 102 +265 114 +193 126 +401 138 +150 150 +273 162 +224 174 +369 186 +66 198 +128 208 +213 220 +146 232 +406 244 +429 256 +374 268 +152 280 +469 292 +145 304 +495 316 +37 328 +327 338 +281 350 +277 362 +209 374 +15 386 +82 396 +403 406 +166 418 +417 430 +430 442 +252 454 +292 466 +219 478 +287 490 +153 502 +193 514 +338 526 +446 538 +459 550 +394 562 +237 574 +482 586 +174 598 +413 610 +494 622 +207 634 +199 646 +466 658 +208 670 +174 682 +399 694 +396 706 +247 718 +417 730 +489 742 +162 754 +377 766 +397 778 +309 790 +365 802 +266 814 +439 826 +342 838 +367 850 +325 862 +167 874 +195 886 +475 898 +17 910 +113 920 +155 932 +203 944 +339 956 +0 968 +455 976 +128 988 +311 1000 +316 1012 +57 1024 +302 1034 +205 1046 +149 1058 +438 1070 +345 1082 +129 1094 +170 1106 +20 1118 +489 1128 +157 1140 +378 1152 +221 1164 +92 1176 +111 1186 +47 1198 +72 1208 +4 1218 +280 1226 +35 1238 +427 1248 +277 1260 +208 1272 +356 1284 +399 1296 +169 1308 +382 1320 +498 1332 +125 1344 +386 1356 +437 1368 +469 1380 +192 1392 +286 1404 +187 1416 +176 1428 +54 1440 +459 1450 +51 1462 +138 1472 +103 1484 +239 1496 +213 1508 +216 1520 +430 1532 +278 1544 +176 1556 +289 1568 +221 1580 +65 1592 +318 1602 +332 1614 +311 1626 +275 1638 +137 1650 +241 1662 +83 1674 +333 1684 +180 1696 +284 1708 +12 1720 +230 1730 +181 1742 +67 1754 +260 1764 +404 1776 +384 1788 +489 1800 +353 1812 +373 1824 +272 1836 +138 1848 +217 1860 +84 1872 +348 1882 +466 1894 +58 1906 +8 1916 +411 1924 +230 1936 +208 1948 +348 1960 +24 1972 +463 1982 +431 1994 +179 2006 +172 2018 +42 2030 +129 2040 +158 2052 +119 2064 +496 2076 +0 2088 +322 2096 +197 2108 +468 2120 +393 2132 +454 2144 +100 2156 +298 2168 +199 2180 +191 2192 +418 2204 +96 2216 +26 2226 +165 2236 +327 2248 +230 2260 +205 2272 +120 2284 +131 2296 +51 2308 +404 2318 +43 2330 +436 2340 +156 2352 +469 2364 +468 2376 +308 2388 +95 2400 +196 2410 +288 2422 +481 2434 +457 2446 +98 2458 +282 2468 +197 2480 +187 2492 +318 2504 +318 2516 +409 2528 +470 2540 +137 2552 +369 2564 +316 2576 +169 2588 +413 2600 +85 2612 +77 2622 +0 2632 +490 2640 +87 2652 +364 2662 +179 2674 +118 2686 +134 2698 +395 2710 +282 2722 +138 2734 +238 2746 +419 2758 +15 2770 +118 2780 +72 2792 +90 2802 +307 2812 +19 2824 +435 2834 +10 2846 +277 2856 +273 2868 +306 2880 +224 2892 +309 2904 +389 2916 +327 2928 +242 2940 +369 2952 +392 2964 +272 2976 +331 2988 +401 3000 +242 3012 +452 3024 +177 3036 +226 3048 +5 3060 +497 3068 +402 3080 +396 3092 +317 3104 +395 3116 +58 3128 +35 3138 +336 3148 +95 3160 +11 3170 +168 3180 +34 3192 +229 3202 +233 3214 +143 3226 +472 3238 +322 3250 +498 3262 +160 3274 +195 3286 +42 3298 +321 3308 +430 3320 +119 3332 +489 3344 +458 3356 +78 3368 +76 3378 +41 3388 +223 3398 +492 3410 +149 3422 +449 3434 +218 3446 +228 3458 +138 3470 +453 3482 +30 3494 +209 3504 +64 3516 +468 3526 +76 3538 +74 3548 +342 3558 +69 3570 +230 3580 +33 3592 +368 3602 +103 3614 +296 3626 +113 3638 +216 3650 +367 3662 +344 3674 +167 3686 +274 3698 +219 3710 +239 3722 +485 3734 +116 3746 +223 3758 +256 3770 +263 3782 +70 3794 +487 3804 +480 3816 +401 3828 +288 3840 +191 3852 +5 3864 +244 3872 +438 3884 +128 3896 +467 3908 +432 3920 +202 3932 +316 3944 +229 3956 +469 3968 +463 3980 +280 3992 +2 4004 +35 4012 +283 4022 +331 4034 +235 4046 +80 4058 +44 4068 +193 4078 +321 4090 +335 4102 +104 4114 +466 4126 +366 4138 +175 4150 +403 4162 +483 4174 +53 4186 +105 4196 +257 4208 +406 4220 +409 4232 +190 4244 +406 4256 +401 4268 +114 4280 +258 4292 +90 4304 +203 4314 +262 4326 +348 4338 +424 4350 +12 4362 +396 4372 +201 4384 +217 4396 +164 4408 +431 4420 +454 4432 +478 4444 +298 4456 +125 4468 +431 4480 +164 4492 +424 4504 +187 4516 +382 4528 +5 4540 +70 4548 +397 4558 +480 4570 +291 4582 +24 4594 +351 4604 +255 4616 +104 4628 +70 4640 +163 4650 +438 4662 +119 4674 +414 4686 +200 4698 +491 4710 +237 4722 +439 4734 +360 4746 +248 4758 +479 4770 +305 4782 +417 4794 +199 4806 +444 4818 +120 4830 +429 4842 +169 4854 +443 4866 +323 4878 +325 4890 +277 4902 +230 4914 +478 4926 +178 4938 +468 4950 +310 4962 +317 4974 +333 4986 +493 4998 +460 5010 +207 5022 +249 5034 +265 5046 +480 5058 +83 5070 +136 5080 +353 5092 +172 5104 +214 5116 +462 5128 +233 5140 +406 5152 +133 5164 +175 5176 +189 5188 +454 5200 +375 5212 +401 5224 +421 5236 +407 5248 +384 5260 +256 5272 +26 5284 +134 5294 +67 5306 +384 5316 +379 5328 +18 5340 +462 5350 +492 5362 +100 5374 +298 5386 +9 5398 +341 5406 +498 5418 +146 5430 +458 5442 +362 5454 +186 5466 +285 5478 +348 5490 +167 5502 +18 5514 +273 5524 +183 5536 +281 5548 +344 5560 +97 5572 +469 5582 +315 5594 +84 5606 +28 5616 +37 5626 +448 5636 +152 5648 +348 5660 +307 5672 +194 5684 +414 5696 +477 5708 +222 5720 +126 5732 +90 5744 +169 5754 +403 5766 +400 5778 +200 5790 +97 5802 +PREHOOK: query: drop index src_index_2 on src +PREHOOK: type: DROPINDEX +POSTHOOK: query: drop index src_index_2 on src +POSTHOOK: type: DROPINDEX Index: ql/src/test/queries/clientpositive/create_idx_immediate.q =================================================================== --- ql/src/test/queries/clientpositive/create_idx_immediate.q (revision 0) +++ ql/src/test/queries/clientpositive/create_idx_immediate.q (revision 0) @@ -0,0 +1,31 @@ +drop index src_index_2 on src; + +create index src_index_2 on table src(key) as 'compact'; +desc extended default__src_src_index_2__; +select key,`_offsets` from default__src_src_index_2__; + +drop index src_index_2 on src; + +create index src_index_2 on table src(key) as 'compact' ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\\'; +desc extended default__src_src_index_2__; +select key,`_offsets` from default__src_src_index_2__; + +drop index src_index_2 on src; + +create index src_index_2 on table src(key) as 'compact' STORED AS TEXTFILE; +desc extended default__src_src_index_2__; +select key,`_offsets` from default__src_src_index_2__; + +drop index src_index_2 on src; + +create index src_index_2 on table src(key) as 'compact' location '/user/hive/warehouse'; +desc extended default__src_src_index_2__; +select key,`_offsets` from default__src_src_index_2__; + +drop index src_index_2 on src; + +create index src_index_2 on table src(key) as 'bitmap'; +desc extended default__src_src_index_2__; +select key,`_offset` from default__src_src_index_2__; + +drop index src_index_2 on src; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java (revision 1338947) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java (working copy) @@ -274,5 +274,21 @@ return rootTask; } + public static Task createIndexRootTask(HiveConf builderConf, Set inputs, + Set outputs, StringBuilder command, + String indexTableName, String dbName){ + // Don't try to index optimize the query to build the index + HiveConf.setBoolVar(builderConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER, false); + Driver driver = new Driver(builderConf); + driver.compile(command.toString()); + + Task rootTask = driver.getPlan().getRootTasks().get(0); + inputs.addAll(driver.getPlan().getInputs()); + outputs.addAll(driver.getPlan().getOutputs()); + + return rootTask; + } + + } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1338947) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Index; @@ -789,16 +790,47 @@ CreateIndexDesc crtIndexDesc = new CreateIndexDesc(tableName, indexName, - indexedCols, indexTableName, deferredRebuild, storageFormat.inputFormat, storageFormat.outputFormat, - storageFormat.storageHandler, typeName, location, idxProps, tblProps, - shared.serde, shared.serdeProps, rowFormatParams.collItemDelim, - rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, - rowFormatParams.lineDelim, rowFormatParams.mapKeyDelim, indexComment); + indexedCols, indexTableName, deferredRebuild, storageFormat.inputFormat, + storageFormat.outputFormat, storageFormat.storageHandler, typeName, + location, idxProps, tblProps, shared.serde, shared.serdeProps, + rowFormatParams.collItemDelim,rowFormatParams.fieldDelim, + rowFormatParams.fieldEscape, rowFormatParams.lineDelim, + rowFormatParams.mapKeyDelim, indexComment); + Task createIndex = TaskFactory.get(new DDLWork(getInputs(), getOutputs(), crtIndexDesc), conf); - rootTasks.add(createIndex); + + if(deferredRebuild) { + rootTasks.add(createIndex); + } else { + addCreateIndexTasks(tableName, indexName, createIndex, crtIndexDesc); + } } + private void addCreateIndexTasks(String baseTableName, String indexName, + Task createIndexDDLTask, CreateIndexDesc crtIndexDesc) throws SemanticException { + try { + String dbName = db.getCurrentDatabase(); + Table baseTbl = db.getTable(dbName, baseTableName); + + String handlerCls = crtIndexDesc.getIndexTypeHandlerClass(); + HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); + + String indexTableName = crtIndexDesc.getIndexTableName() != null ? + crtIndexDesc.getIndexTableName() : + MetaStoreUtils.getIndexTableName(dbName, crtIndexDesc.getTableName(), indexName); + + Task ret = handler.generateCreateIndexTask(baseTbl, getInputs(), + getOutputs(), createIndexDDLTask, crtIndexDesc, indexTableName); + + rootTasks.add(ret); + + } catch (Exception e) { + throw new SemanticException(e); + } + } + + private void analyzeDropIndex(ASTNode ast) throws SemanticException { String indexName = unescapeIdentifier(ast.getChild(0).getText()); String tableName = getUnescapedName((ASTNode)ast.getChild(1)); Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java (revision 1338947) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java (working copy) @@ -199,6 +199,12 @@ return "`" + identifier + "`"; } + /** + * Regenerate an identifier as part of unparsing it back to SQL text. + */ + public static String unparseIdentifierToString(String identifier) { + return "'" + identifier + "'"; + } public static HiveStorageHandler getStorageHandler( Configuration conf, String className) throws HiveException { @@ -306,4 +312,19 @@ } return sb.toString(); } + + /** + * Generate comma separated fields from a list + */ + public static String getUnparsedColumnNames( + List columnNames) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < columnNames.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(HiveUtils.unparseIdentifier(columnNames.get(i))); + } + return sb.toString(); + } } Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 1338947) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -31,13 +31,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.HashSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -63,7 +63,6 @@ import org.apache.hadoop.hive.metastore.api.HiveObjectRef; import org.apache.hadoop.hive.metastore.api.HiveObjectType; import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -703,10 +702,6 @@ } } - if(!deferredRebuild) { - throw new RuntimeException("Please specify deferred rebuild using \" WITH DEFERRED REBUILD \"."); - } - Index indexDesc = new Index(indexName, indexHandlerClass, dbName, tableName, time, time, indexTblName, storageDescriptor, params, deferredRebuild); indexDesc.getParameters().put("comment", indexComment); Index: ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java (revision 1338947) +++ ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java (working copy) @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; /** @@ -112,6 +113,42 @@ throws HiveException; /** + * Requests that the handler generate a plan for creating the index; It + * takes task which creates the index metadata as input and adds map red + * task which builds the index data + * + * @param baseTbl + * the definition of the table being indexed + * + * @param inputs + * inputs for hooks, supplemental outputs going + * along with the return value + * + * @param outputs + * outputs for hooks, supplemental outputs going + * along with the return value + * + * @param createIndexDDLTask + * task to create index metadata + * + * @param crtIndexDesc + * Index descriptor + * + * @param indexTableName + * index table name + * + * @return task which when executed creates the index + * + * @throws HiveException if plan generation fails + */ + Task generateCreateIndexTask( + org.apache.hadoop.hive.ql.metadata.Table baseTbl, + Set inputs, Set outputs, + Task createIndexDDLTask, CreateIndexDesc crtIndexDesc, + String indexTableName) + throws HiveException; + + /** * Generate the list of tasks required to run an index optimized sub-query for the * given predicate, using the given indexes. If multiple indexes are * provided, it is up to the handler whether to use none, one, some or all of Index: ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (revision 1338947) +++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (working copy) @@ -18,12 +18,19 @@ package org.apache.hadoop.hive.ql.index.compact; +import static org.apache.hadoop.hive.serde.Constants.COLLECTION_DELIM; +import static org.apache.hadoop.hive.serde.Constants.ESCAPE_CHAR; +import static org.apache.hadoop.hive.serde.Constants.FIELD_DELIM; +import static org.apache.hadoop.hive.serde.Constants.LINE_DELIM; +import static org.apache.hadoop.hive.serde.Constants.MAPKEY_DELIM; + import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.logging.Log; @@ -53,6 +60,7 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.IndexUtils; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; @@ -63,8 +71,6 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; public class CompactIndexHandler extends TableBasedIndexHandler { @@ -92,6 +98,110 @@ } @Override + protected Task getCreateIndexMapRedTask(Set inputs, Set outputs, + CreateIndexDesc crtIndexDesc, String indexTableName, + org.apache.hadoop.hive.ql.metadata.Table baseTbl, Task createIndexDDLTask) throws HiveException { + + String indexCols = HiveUtils.getUnparsedColumnNames(crtIndexDesc.getIndexedCols()); + StringBuilder command= new StringBuilder(); + StringBuilder serdePropsConcat = new StringBuilder(); + + String collItemDelim = crtIndexDesc.getCollItemDelim() == null? baseTbl.getSerdeParam(COLLECTION_DELIM) : crtIndexDesc.getCollItemDelim(); + String fieldDelim = crtIndexDesc.getFieldDelim() == null? baseTbl.getSerdeParam(FIELD_DELIM) : crtIndexDesc.getFieldDelim(); + String fieldEscape = crtIndexDesc.getFieldEscape() == null? baseTbl.getSerdeParam(ESCAPE_CHAR) : crtIndexDesc.getFieldEscape(); + String lineDelim = crtIndexDesc.getLineDelim() == null? baseTbl.getSerdeParam(LINE_DELIM) : crtIndexDesc.getLineDelim(); + String mapKeys = crtIndexDesc.getMapKeyDelim() == null? baseTbl.getSerdeParam(MAPKEY_DELIM) : crtIndexDesc.getMapKeyDelim(); + + if(crtIndexDesc.getSerdeProps() != null && crtIndexDesc.getSerdeProps().size()!=0) { + serdePropsConcat.append(" WITH SERDEPROPERTIES ("); + + String commaAppend = ""; + + for(Map.Entry serdeProp : crtIndexDesc.getSerdeProps().entrySet()) { + serdePropsConcat.append(commaAppend); + commaAppend = ","; + serdePropsConcat.append(HiveUtils.unparseIdentifier(serdeProp.getKey())); + serdePropsConcat.append(commaAppend); + serdePropsConcat.append(HiveUtils.unparseIdentifier(serdeProp.getValue())); + } + serdePropsConcat.append(")"); + } + + command.append("CREATE TABLE " + HiveUtils.unparseIdentifier(indexTableName)); + if(crtIndexDesc.getStorageHandler() != null) { + command.append(" STORED BY " + HiveUtils.unparseIdentifier(crtIndexDesc.getStorageHandler())); + command.append(serdePropsConcat); + } + else { + if(collItemDelim != null || fieldDelim != null || lineDelim != null || mapKeys != null || fieldEscape != null || crtIndexDesc.getSerde() != null) { + command.append(" ROW FORMAT"); + + if(crtIndexDesc.getSerde() == null) { + command.append(" DELIMITED"); + + if(fieldDelim != null) { + command.append(" FIELDS TERMINATED BY " + HiveUtils.unparseIdentifierToString(fieldDelim)); + } + if(fieldEscape != null) { + command.append(" ESCAPED BY '" + fieldEscape.replaceAll("\\\\", "\\\\\\\\") + "'"); + } + if(collItemDelim != null) { + command.append(" COLLECTION ITEMS TERMINATED BY " + HiveUtils.unparseIdentifierToString(collItemDelim)); + } + if(mapKeys != null) { + command.append(" MAP KEYS TERMINATED BY " + HiveUtils.unparseIdentifierToString(mapKeys)); + } + if(lineDelim != null) { + command.append(" LINES TERMINATED BY " + HiveUtils.unparseIdentifierToString(lineDelim)); + } + } + else { + command.append(" SERDE " + HiveUtils.unparseIdentifierToString(crtIndexDesc.getSerde())); + command.append(serdePropsConcat); + } + } + + if(crtIndexDesc.getInputFormat() != null) { + command.append(" STORED AS"); + command.append(" INPUTFORMAT " + HiveUtils.unparseIdentifierToString(crtIndexDesc.getInputFormat())); + command.append(" OUTPUTFORMAT " + HiveUtils.unparseIdentifierToString(crtIndexDesc.getOutputFormat())); + } + } + if(crtIndexDesc.getLocation()!=null) { + command.append(" LOCATION " + HiveUtils.unparseIdentifierToString(crtIndexDesc.getLocation())); + } + + command.append(" AS SELECT "); + command.append(indexCols); + command.append(","); + command.append(VirtualColumn.FILENAME.getName()); + command.append(","); + command.append(" collect_set ("); + command.append(VirtualColumn.BLOCKOFFSET.getName()); + command.append(") "); + command.append(" FROM " + HiveUtils.unparseIdentifier(baseTbl.getTableName()) ); + + command.append(" GROUP BY "); + command.append(indexCols + ", " + VirtualColumn.FILENAME.getName()); + + HiveConf builderConf = new HiveConf(getConf(), CompactIndexHandler.class); + builderConf.setBoolVar(HiveConf.ConfVars.HIVEMERGEMAPFILES, false); + builderConf.setBoolVar(HiveConf.ConfVars.HIVEMERGEMAPREDFILES, false); + Task rootTask = IndexUtils.createIndexRootTask(builderConf, inputs, outputs, + command, indexTableName, baseTbl.getDbName()); + + List> indexBuilder = rootTask.getChildTasks(); + for(Task inx : indexBuilder) + { + inx.getChildTasks().removeAll(inx.getChildTasks()); + } + + rootTask.addDependentTask(createIndexDDLTask); + + return rootTask; + } + + @Override protected Task getIndexBuilderMapRedTask(Set inputs, Set outputs, List indexField, boolean partitioned, PartitionDesc indexTblPartDesc, String indexTableName, Index: ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java (revision 1338947) +++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java (working copy) @@ -18,6 +18,12 @@ package org.apache.hadoop.hive.ql.index.bitmap; +import static org.apache.hadoop.hive.serde.Constants.COLLECTION_DELIM; +import static org.apache.hadoop.hive.serde.Constants.ESCAPE_CHAR; +import static org.apache.hadoop.hive.serde.Constants.FIELD_DELIM; +import static org.apache.hadoop.hive.serde.Constants.LINE_DELIM; +import static org.apache.hadoop.hive.serde.Constants.MAPKEY_DELIM; + import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -48,6 +54,7 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.IndexUtils; import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; @@ -291,6 +298,136 @@ } @Override + protected Task getCreateIndexMapRedTask(Set inputs, Set outputs, + CreateIndexDesc crtIndexDesc, String indexTableName, + org.apache.hadoop.hive.ql.metadata.Table baseTbl, + Task createIndexDDLTask) throws HiveException { + + HiveConf builderConf = new HiveConf(getConf(), BitmapIndexHandler.class); + HiveConf.setBoolVar(builderConf, HiveConf.ConfVars.HIVEROWOFFSET, true); + + String indexCols = HiveUtils.getUnparsedColumnNames(crtIndexDesc.getIndexedCols()); + StringBuilder command= new StringBuilder(); + StringBuilder serdePropsConcat = new StringBuilder(); + + String collItemDelim = crtIndexDesc.getCollItemDelim() == null? + baseTbl.getSerdeParam(COLLECTION_DELIM) : crtIndexDesc.getCollItemDelim(); + String fieldDelim = crtIndexDesc.getFieldDelim() == null? + baseTbl.getSerdeParam(FIELD_DELIM) : crtIndexDesc.getFieldDelim(); + String fieldEscape = crtIndexDesc.getFieldEscape() == null? + baseTbl.getSerdeParam(ESCAPE_CHAR) : crtIndexDesc.getFieldEscape(); + String lineDelim = crtIndexDesc.getLineDelim() == null? + baseTbl.getSerdeParam(LINE_DELIM) : crtIndexDesc.getLineDelim(); + String mapKeys = crtIndexDesc.getMapKeyDelim() == null? + baseTbl.getSerdeParam(MAPKEY_DELIM) : crtIndexDesc.getMapKeyDelim(); + + if(crtIndexDesc.getSerdeProps() != null && crtIndexDesc.getSerdeProps().size()!=0) { + serdePropsConcat.append(" WITH SERDEPROPERTIES ("); + + String commaAppend = ""; + + for(Map.Entry serdeProp : crtIndexDesc.getSerdeProps().entrySet()) { + serdePropsConcat.append(commaAppend); + commaAppend = ","; + serdePropsConcat.append(HiveUtils.unparseIdentifier(serdeProp.getKey())); + serdePropsConcat.append(commaAppend); + serdePropsConcat.append(HiveUtils.unparseIdentifier(serdeProp.getValue())); + } + serdePropsConcat.append(")"); + } + + command.append("CREATE TABLE " + HiveUtils.unparseIdentifier(indexTableName)); + if(crtIndexDesc.getStorageHandler() != null) { + command.append(" STORED BY " + HiveUtils.unparseIdentifier(crtIndexDesc.getStorageHandler())); + command.append(serdePropsConcat); + } + else { + if(collItemDelim != null || fieldDelim != null || lineDelim != null || + mapKeys != null || fieldEscape != null || crtIndexDesc.getSerde() != null) { + command.append(" ROW FORMAT"); + + if(crtIndexDesc.getSerde() == null) { + command.append(" DELIMITED"); + + if(fieldDelim != null) { + command.append(" FIELDS TERMINATED BY " + HiveUtils.unparseIdentifierToString(fieldDelim)); + } + if(fieldEscape != null) { + command.append(" ESCAPED BY '" + fieldEscape.replaceAll("\\\\", "\\\\\\\\") + "'"); + } + if(collItemDelim != null) { + command.append(" COLLECTION ITEMS TERMINATED BY " + + HiveUtils.unparseIdentifierToString(collItemDelim)); + } + if(mapKeys != null) { + command.append(" MAP KEYS TERMINATED BY " + + HiveUtils.unparseIdentifierToString(mapKeys)); + } + if(lineDelim != null) { + command.append(" LINES TERMINATED BY " + + HiveUtils.unparseIdentifierToString(lineDelim)); + } + } + else { + command.append(" SERDE " + + HiveUtils.unparseIdentifierToString(crtIndexDesc.getSerde())); + command.append(serdePropsConcat); + } + } + + if(crtIndexDesc.getInputFormat() != null) { + command.append(" STORED AS"); + command.append(" INPUTFORMAT " + + HiveUtils.unparseIdentifierToString(crtIndexDesc.getInputFormat())); + command.append(" OUTPUTFORMAT " + + HiveUtils.unparseIdentifierToString(crtIndexDesc.getOutputFormat())); + } + } + if(crtIndexDesc.getLocation()!=null) { + command.append(" LOCATION " + + HiveUtils.unparseIdentifierToString(crtIndexDesc.getLocation())); + } + + command.append(" AS SELECT "); + command.append(indexCols);; + command.append(","); + command.append(VirtualColumn.FILENAME.getName()); + command.append(","); + command.append(VirtualColumn.BLOCKOFFSET.getName()); + command.append(","); + command.append("EWAH_BITMAP("); + command.append(VirtualColumn.ROWOFFSET.getName()); + command.append(")"); + command.append(" FROM " + HiveUtils.unparseIdentifier(baseTbl.getTableName()) ); + + command.append(" GROUP BY "); + command.append(VirtualColumn.FILENAME.getName()); + command.append(","); + command.append(VirtualColumn.BLOCKOFFSET.getName()); + command.append(","); + command.append(indexCols); + + // TODO: Make this work without map side aggregation + if (!builderConf.get("hive.map.aggr", null).equals("true")) { + throw new HiveException("Cannot construct index without map-side aggregation"); + } + + Task rootTask = IndexUtils.createIndexRootTask(builderConf, inputs, outputs, + command, indexTableName, baseTbl.getDbName()); + + List> indexBuilder = rootTask.getChildTasks(); + for(Task inx : indexBuilder) + { + inx.getChildTasks().removeAll(inx.getChildTasks()); + } + + rootTask.addDependentTask(createIndexDDLTask); + + return rootTask; + } + + + @Override /** * No lower bound on bitmap index query size, so this will always return true */ Index: ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java (revision 1338947) +++ ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java (working copy) @@ -22,8 +22,8 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map.Entry; import java.util.Set; -import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; @@ -45,6 +46,32 @@ protected Configuration configuration; @Override + public Task generateCreateIndexTask( + org.apache.hadoop.hive.ql.metadata.Table baseTbl, Set inputs, + Set outputs, Task createIndexDDLTask, CreateIndexDesc crtIndexDesc, String indexTableName) + throws HiveException{ + try{ + Task createIndexTask; + if (!(baseTbl.isPartitioned())) { + createIndexTask = getCreateIndexMapRedTask(inputs,outputs, + crtIndexDesc, indexTableName, + baseTbl, createIndexDDLTask); + } + else{ + throw new RuntimeException("Please specify deferred rebuild using \" WITH DEFERRED REBUILD \" for partitioned tables."); + } + return createIndexTask; + + }catch (Exception e) { + throw new SemanticException(e); + } + } + + abstract protected Task getCreateIndexMapRedTask(Set inputs, Set outputs, + CreateIndexDesc crtIndexDesc, String indexTableName, + org.apache.hadoop.hive.ql.metadata.Table baseTbl, Task createIndexDDLTask) throws HiveException; + + @Override public List> generateIndexBuildTaskList( org.apache.hadoop.hive.ql.metadata.Table baseTbl, org.apache.hadoop.hive.metastore.api.Index index,