diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 917b3a4..c545830 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -842,7 +842,6 @@ spark.query.files=add_part_multiple.q, \ smb_mapjoin_8.q, \ smb_mapjoin_9.q, \ sort.q, \ - spark_test.q, \ stats0.q, \ stats1.q, \ stats10.q, \ diff --git pom.xml pom.xml index 0ba2e32..97b06bc 100644 --- pom.xml +++ pom.xml @@ -212,17 +212,7 @@ false - - - spark-snapshot - http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark_2.10-1.2-SNAPSHOT/ - - false - - - true - - + diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index a562aa1..daf6cb8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -32,13 +32,13 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; -import java.util.LinkedHashMap; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -48,7 +48,12 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; -import org.apache.hadoop.hive.ql.plan.*; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.ExplainWork; +import org.apache.hadoop.hive.ql.plan.HiveOperation; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.SparkWork; +import org.apache.hadoop.hive.ql.plan.TezWork; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.security.authorization.AuthorizationFactory; import org.apache.hadoop.hive.ql.session.SessionState; diff --git ql/src/test/queries/clientpositive/spark_test.q ql/src/test/queries/clientpositive/spark_test.q deleted file mode 100644 index 225026b..0000000 --- ql/src/test/queries/clientpositive/spark_test.q +++ /dev/null @@ -1,4 +0,0 @@ --- SORT_QUERY_RESULTS - -select key from src; -select key,avg(key) from src group by key; diff --git ql/src/test/results/clientpositive/spark/spark_test.q.out ql/src/test/results/clientpositive/spark/spark_test.q.out deleted file mode 100644 index 89ec52d..0000000 --- ql/src/test/results/clientpositive/spark/spark_test.q.out +++ /dev/null @@ -1,829 +0,0 @@ -PREHOOK: query: -- SORT_QUERY_RESULTS - -select key from src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: -- SORT_QUERY_RESULTS - -select key from src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 -0 -0 -10 -100 -100 -103 -103 -104 -104 -105 -11 -111 -113 -113 -114 -116 -118 -118 -119 -119 -119 -12 -12 -120 -120 -125 -125 -126 -128 -128 -128 -129 -129 -131 -133 -134 -134 -136 -137 -137 -138 -138 -138 -138 -143 -145 -146 -146 -149 -149 -15 -15 -150 -152 -152 -153 -155 -156 -157 -158 -160 -162 -163 -164 -164 -165 -165 -166 -167 -167 -167 -168 -169 -169 -169 -169 -17 -170 -172 -172 -174 -174 -175 -175 -176 -176 -177 -178 -179 -179 -18 -18 -180 -181 -183 -186 -187 -187 -187 -189 -19 -190 -191 -191 -192 -193 -193 -193 -194 -195 -195 -196 -197 -197 -199 -199 -199 -2 -20 -200 -200 -201 -202 -203 -203 -205 -205 -207 -207 -208 -208 -208 -209 -209 -213 -213 -214 -216 -216 -217 -217 -218 -219 -219 -221 -221 -222 -223 -223 -224 -224 -226 -228 -229 -229 -230 -230 -230 -230 -230 -233 -233 -235 -237 -237 -238 -238 -239 -239 -24 -24 -241 -242 -242 -244 -247 -248 -249 -252 -255 -255 -256 -256 -257 -258 -26 -26 -260 -262 -263 -265 -265 -266 -27 -272 -272 -273 -273 -273 -274 -275 -277 -277 -277 -277 -278 -278 -28 -280 -280 -281 -281 -282 -282 -283 -284 -285 -286 -287 -288 -288 -289 -291 -292 -296 -298 -298 -298 -30 -302 -305 -306 -307 -307 -308 -309 -309 -310 -311 -311 -311 -315 -316 -316 -316 -317 -317 -318 -318 -318 -321 -321 -322 -322 -323 -325 -325 -327 -327 -327 -33 -331 -331 -332 -333 -333 -335 -336 -338 -339 -34 -341 -342 -342 -344 -344 -345 -348 -348 -348 -348 -348 -35 -35 -35 -351 -353 -353 -356 -360 -362 -364 -365 -366 -367 -367 -368 -369 -369 -369 -37 -37 -373 -374 -375 -377 -378 -379 -382 -382 -384 -384 -384 -386 -389 -392 -393 -394 -395 -395 -396 -396 -396 -397 -397 -399 -399 -4 -400 -401 -401 -401 -401 -401 -402 -403 -403 -403 -404 -404 -406 -406 -406 -406 -407 -409 -409 -409 -41 -411 -413 -413 -414 -414 -417 -417 -417 -418 -419 -42 -42 -421 -424 -424 -427 -429 -429 -43 -430 -430 -430 -431 -431 -431 -432 -435 -436 -437 -438 -438 -438 -439 -439 -44 -443 -444 -446 -448 -449 -452 -453 -454 -454 -454 -455 -457 -458 -458 -459 -459 -460 -462 -462 -463 -463 -466 -466 -466 -467 -468 -468 -468 -468 -469 -469 -469 -469 -469 -47 -470 -472 -475 -477 -478 -478 -479 -480 -480 -480 -481 -482 -483 -484 -485 -487 -489 -489 -489 -489 -490 -491 -492 -492 -493 -494 -495 -496 -497 -498 -498 -498 -5 -5 -5 -51 -51 -53 -54 -57 -58 -58 -64 -65 -66 -67 -67 -69 -70 -70 -70 -72 -72 -74 -76 -76 -77 -78 -8 -80 -82 -83 -83 -84 -84 -85 -86 -87 -9 -90 -90 -90 -92 -95 -95 -96 -97 -97 -98 -98 -PREHOOK: query: select key,avg(key) from src group by key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select key,avg(key) from src group by key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 0.0 -10 10.0 -100 100.0 -103 103.0 -104 104.0 -105 105.0 -11 11.0 -111 111.0 -113 113.0 -114 114.0 -116 116.0 -118 118.0 -119 119.0 -12 12.0 -120 120.0 -125 125.0 -126 126.0 -128 128.0 -129 129.0 -131 131.0 -133 133.0 -134 134.0 -136 136.0 -137 137.0 -138 138.0 -143 143.0 -145 145.0 -146 146.0 -149 149.0 -15 15.0 -150 150.0 -152 152.0 -153 153.0 -155 155.0 -156 156.0 -157 157.0 -158 158.0 -160 160.0 -162 162.0 -163 163.0 -164 164.0 -165 165.0 -166 166.0 -167 167.0 -168 168.0 -169 169.0 -17 17.0 -170 170.0 -172 172.0 -174 174.0 -175 175.0 -176 176.0 -177 177.0 -178 178.0 -179 179.0 -18 18.0 -180 180.0 -181 181.0 -183 183.0 -186 186.0 -187 187.0 -189 189.0 -19 19.0 -190 190.0 -191 191.0 -192 192.0 -193 193.0 -194 194.0 -195 195.0 -196 196.0 -197 197.0 -199 199.0 -2 2.0 -20 20.0 -200 200.0 -201 201.0 -202 202.0 -203 203.0 -205 205.0 -207 207.0 -208 208.0 -209 209.0 -213 213.0 -214 214.0 -216 216.0 -217 217.0 -218 218.0 -219 219.0 -221 221.0 -222 222.0 -223 223.0 -224 224.0 -226 226.0 -228 228.0 -229 229.0 -230 230.0 -233 233.0 -235 235.0 -237 237.0 -238 238.0 -239 239.0 -24 24.0 -241 241.0 -242 242.0 -244 244.0 -247 247.0 -248 248.0 -249 249.0 -252 252.0 -255 255.0 -256 256.0 -257 257.0 -258 258.0 -26 26.0 -260 260.0 -262 262.0 -263 263.0 -265 265.0 -266 266.0 -27 27.0 -272 272.0 -273 273.0 -274 274.0 -275 275.0 -277 277.0 -278 278.0 -28 28.0 -280 280.0 -281 281.0 -282 282.0 -283 283.0 -284 284.0 -285 285.0 -286 286.0 -287 287.0 -288 288.0 -289 289.0 -291 291.0 -292 292.0 -296 296.0 -298 298.0 -30 30.0 -302 302.0 -305 305.0 -306 306.0 -307 307.0 -308 308.0 -309 309.0 -310 310.0 -311 311.0 -315 315.0 -316 316.0 -317 317.0 -318 318.0 -321 321.0 -322 322.0 -323 323.0 -325 325.0 -327 327.0 -33 33.0 -331 331.0 -332 332.0 -333 333.0 -335 335.0 -336 336.0 -338 338.0 -339 339.0 -34 34.0 -341 341.0 -342 342.0 -344 344.0 -345 345.0 -348 348.0 -35 35.0 -351 351.0 -353 353.0 -356 356.0 -360 360.0 -362 362.0 -364 364.0 -365 365.0 -366 366.0 -367 367.0 -368 368.0 -369 369.0 -37 37.0 -373 373.0 -374 374.0 -375 375.0 -377 377.0 -378 378.0 -379 379.0 -382 382.0 -384 384.0 -386 386.0 -389 389.0 -392 392.0 -393 393.0 -394 394.0 -395 395.0 -396 396.0 -397 397.0 -399 399.0 -4 4.0 -400 400.0 -401 401.0 -402 402.0 -403 403.0 -404 404.0 -406 406.0 -407 407.0 -409 409.0 -41 41.0 -411 411.0 -413 413.0 -414 414.0 -417 417.0 -418 418.0 -419 419.0 -42 42.0 -421 421.0 -424 424.0 -427 427.0 -429 429.0 -43 43.0 -430 430.0 -431 431.0 -432 432.0 -435 435.0 -436 436.0 -437 437.0 -438 438.0 -439 439.0 -44 44.0 -443 443.0 -444 444.0 -446 446.0 -448 448.0 -449 449.0 -452 452.0 -453 453.0 -454 454.0 -455 455.0 -457 457.0 -458 458.0 -459 459.0 -460 460.0 -462 462.0 -463 463.0 -466 466.0 -467 467.0 -468 468.0 -469 469.0 -47 47.0 -470 470.0 -472 472.0 -475 475.0 -477 477.0 -478 478.0 -479 479.0 -480 480.0 -481 481.0 -482 482.0 -483 483.0 -484 484.0 -485 485.0 -487 487.0 -489 489.0 -490 490.0 -491 491.0 -492 492.0 -493 493.0 -494 494.0 -495 495.0 -496 496.0 -497 497.0 -498 498.0 -5 5.0 -51 51.0 -53 53.0 -54 54.0 -57 57.0 -58 58.0 -64 64.0 -65 65.0 -66 66.0 -67 67.0 -69 69.0 -70 70.0 -72 72.0 -74 74.0 -76 76.0 -77 77.0 -78 78.0 -8 8.0 -80 80.0 -82 82.0 -83 83.0 -84 84.0 -85 85.0 -86 86.0 -87 87.0 -9 9.0 -90 90.0 -92 92.0 -95 95.0 -96 96.0 -97 97.0 -98 98.0 diff --git ql/src/test/results/clientpositive/spark_test.q.out ql/src/test/results/clientpositive/spark_test.q.out deleted file mode 100644 index 89ec52d..0000000 --- ql/src/test/results/clientpositive/spark_test.q.out +++ /dev/null @@ -1,829 +0,0 @@ -PREHOOK: query: -- SORT_QUERY_RESULTS - -select key from src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: -- SORT_QUERY_RESULTS - -select key from src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 -0 -0 -10 -100 -100 -103 -103 -104 -104 -105 -11 -111 -113 -113 -114 -116 -118 -118 -119 -119 -119 -12 -12 -120 -120 -125 -125 -126 -128 -128 -128 -129 -129 -131 -133 -134 -134 -136 -137 -137 -138 -138 -138 -138 -143 -145 -146 -146 -149 -149 -15 -15 -150 -152 -152 -153 -155 -156 -157 -158 -160 -162 -163 -164 -164 -165 -165 -166 -167 -167 -167 -168 -169 -169 -169 -169 -17 -170 -172 -172 -174 -174 -175 -175 -176 -176 -177 -178 -179 -179 -18 -18 -180 -181 -183 -186 -187 -187 -187 -189 -19 -190 -191 -191 -192 -193 -193 -193 -194 -195 -195 -196 -197 -197 -199 -199 -199 -2 -20 -200 -200 -201 -202 -203 -203 -205 -205 -207 -207 -208 -208 -208 -209 -209 -213 -213 -214 -216 -216 -217 -217 -218 -219 -219 -221 -221 -222 -223 -223 -224 -224 -226 -228 -229 -229 -230 -230 -230 -230 -230 -233 -233 -235 -237 -237 -238 -238 -239 -239 -24 -24 -241 -242 -242 -244 -247 -248 -249 -252 -255 -255 -256 -256 -257 -258 -26 -26 -260 -262 -263 -265 -265 -266 -27 -272 -272 -273 -273 -273 -274 -275 -277 -277 -277 -277 -278 -278 -28 -280 -280 -281 -281 -282 -282 -283 -284 -285 -286 -287 -288 -288 -289 -291 -292 -296 -298 -298 -298 -30 -302 -305 -306 -307 -307 -308 -309 -309 -310 -311 -311 -311 -315 -316 -316 -316 -317 -317 -318 -318 -318 -321 -321 -322 -322 -323 -325 -325 -327 -327 -327 -33 -331 -331 -332 -333 -333 -335 -336 -338 -339 -34 -341 -342 -342 -344 -344 -345 -348 -348 -348 -348 -348 -35 -35 -35 -351 -353 -353 -356 -360 -362 -364 -365 -366 -367 -367 -368 -369 -369 -369 -37 -37 -373 -374 -375 -377 -378 -379 -382 -382 -384 -384 -384 -386 -389 -392 -393 -394 -395 -395 -396 -396 -396 -397 -397 -399 -399 -4 -400 -401 -401 -401 -401 -401 -402 -403 -403 -403 -404 -404 -406 -406 -406 -406 -407 -409 -409 -409 -41 -411 -413 -413 -414 -414 -417 -417 -417 -418 -419 -42 -42 -421 -424 -424 -427 -429 -429 -43 -430 -430 -430 -431 -431 -431 -432 -435 -436 -437 -438 -438 -438 -439 -439 -44 -443 -444 -446 -448 -449 -452 -453 -454 -454 -454 -455 -457 -458 -458 -459 -459 -460 -462 -462 -463 -463 -466 -466 -466 -467 -468 -468 -468 -468 -469 -469 -469 -469 -469 -47 -470 -472 -475 -477 -478 -478 -479 -480 -480 -480 -481 -482 -483 -484 -485 -487 -489 -489 -489 -489 -490 -491 -492 -492 -493 -494 -495 -496 -497 -498 -498 -498 -5 -5 -5 -51 -51 -53 -54 -57 -58 -58 -64 -65 -66 -67 -67 -69 -70 -70 -70 -72 -72 -74 -76 -76 -77 -78 -8 -80 -82 -83 -83 -84 -84 -85 -86 -87 -9 -90 -90 -90 -92 -95 -95 -96 -97 -97 -98 -98 -PREHOOK: query: select key,avg(key) from src group by key -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: select key,avg(key) from src group by key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -#### A masked pattern was here #### -0 0.0 -10 10.0 -100 100.0 -103 103.0 -104 104.0 -105 105.0 -11 11.0 -111 111.0 -113 113.0 -114 114.0 -116 116.0 -118 118.0 -119 119.0 -12 12.0 -120 120.0 -125 125.0 -126 126.0 -128 128.0 -129 129.0 -131 131.0 -133 133.0 -134 134.0 -136 136.0 -137 137.0 -138 138.0 -143 143.0 -145 145.0 -146 146.0 -149 149.0 -15 15.0 -150 150.0 -152 152.0 -153 153.0 -155 155.0 -156 156.0 -157 157.0 -158 158.0 -160 160.0 -162 162.0 -163 163.0 -164 164.0 -165 165.0 -166 166.0 -167 167.0 -168 168.0 -169 169.0 -17 17.0 -170 170.0 -172 172.0 -174 174.0 -175 175.0 -176 176.0 -177 177.0 -178 178.0 -179 179.0 -18 18.0 -180 180.0 -181 181.0 -183 183.0 -186 186.0 -187 187.0 -189 189.0 -19 19.0 -190 190.0 -191 191.0 -192 192.0 -193 193.0 -194 194.0 -195 195.0 -196 196.0 -197 197.0 -199 199.0 -2 2.0 -20 20.0 -200 200.0 -201 201.0 -202 202.0 -203 203.0 -205 205.0 -207 207.0 -208 208.0 -209 209.0 -213 213.0 -214 214.0 -216 216.0 -217 217.0 -218 218.0 -219 219.0 -221 221.0 -222 222.0 -223 223.0 -224 224.0 -226 226.0 -228 228.0 -229 229.0 -230 230.0 -233 233.0 -235 235.0 -237 237.0 -238 238.0 -239 239.0 -24 24.0 -241 241.0 -242 242.0 -244 244.0 -247 247.0 -248 248.0 -249 249.0 -252 252.0 -255 255.0 -256 256.0 -257 257.0 -258 258.0 -26 26.0 -260 260.0 -262 262.0 -263 263.0 -265 265.0 -266 266.0 -27 27.0 -272 272.0 -273 273.0 -274 274.0 -275 275.0 -277 277.0 -278 278.0 -28 28.0 -280 280.0 -281 281.0 -282 282.0 -283 283.0 -284 284.0 -285 285.0 -286 286.0 -287 287.0 -288 288.0 -289 289.0 -291 291.0 -292 292.0 -296 296.0 -298 298.0 -30 30.0 -302 302.0 -305 305.0 -306 306.0 -307 307.0 -308 308.0 -309 309.0 -310 310.0 -311 311.0 -315 315.0 -316 316.0 -317 317.0 -318 318.0 -321 321.0 -322 322.0 -323 323.0 -325 325.0 -327 327.0 -33 33.0 -331 331.0 -332 332.0 -333 333.0 -335 335.0 -336 336.0 -338 338.0 -339 339.0 -34 34.0 -341 341.0 -342 342.0 -344 344.0 -345 345.0 -348 348.0 -35 35.0 -351 351.0 -353 353.0 -356 356.0 -360 360.0 -362 362.0 -364 364.0 -365 365.0 -366 366.0 -367 367.0 -368 368.0 -369 369.0 -37 37.0 -373 373.0 -374 374.0 -375 375.0 -377 377.0 -378 378.0 -379 379.0 -382 382.0 -384 384.0 -386 386.0 -389 389.0 -392 392.0 -393 393.0 -394 394.0 -395 395.0 -396 396.0 -397 397.0 -399 399.0 -4 4.0 -400 400.0 -401 401.0 -402 402.0 -403 403.0 -404 404.0 -406 406.0 -407 407.0 -409 409.0 -41 41.0 -411 411.0 -413 413.0 -414 414.0 -417 417.0 -418 418.0 -419 419.0 -42 42.0 -421 421.0 -424 424.0 -427 427.0 -429 429.0 -43 43.0 -430 430.0 -431 431.0 -432 432.0 -435 435.0 -436 436.0 -437 437.0 -438 438.0 -439 439.0 -44 44.0 -443 443.0 -444 444.0 -446 446.0 -448 448.0 -449 449.0 -452 452.0 -453 453.0 -454 454.0 -455 455.0 -457 457.0 -458 458.0 -459 459.0 -460 460.0 -462 462.0 -463 463.0 -466 466.0 -467 467.0 -468 468.0 -469 469.0 -47 47.0 -470 470.0 -472 472.0 -475 475.0 -477 477.0 -478 478.0 -479 479.0 -480 480.0 -481 481.0 -482 482.0 -483 483.0 -484 484.0 -485 485.0 -487 487.0 -489 489.0 -490 490.0 -491 491.0 -492 492.0 -493 493.0 -494 494.0 -495 495.0 -496 496.0 -497 497.0 -498 498.0 -5 5.0 -51 51.0 -53 53.0 -54 54.0 -57 57.0 -58 58.0 -64 64.0 -65 65.0 -66 66.0 -67 67.0 -69 69.0 -70 70.0 -72 72.0 -74 74.0 -76 76.0 -77 77.0 -78 78.0 -8 8.0 -80 80.0 -82 82.0 -83 83.0 -84 84.0 -85 85.0 -86 86.0 -87 87.0 -9 9.0 -90 90.0 -92 92.0 -95 95.0 -96 96.0 -97 97.0 -98 98.0