diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index a2ccfe0..51b11ce 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -52,7 +52,8 @@ minimr.query.files=auto_sortmerge_join_16.q,\ temp_table_external.q,\ truncate_column_buckets.q,\ uber_reduce.q,\ - udf_using.q + udf_using.q,\ + orc_mr_pathalias.q # These tests are disabled for minimr # ql_rewrite_gbtoidx.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index 1f262d0..a3d258e 100755 --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -589,6 +589,17 @@ protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass splitPathWithNoSchema, false); } + private static boolean isMatch(String splitPath, String key) { + if (splitPath.equals(key)) { + return true; + } + int index = splitPath.indexOf(key); + if (index == -1) { + return false; + } + return splitPath.charAt(index+key.length()) == '/'; + } + protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass, String splitPath, String splitPathWithNoSchema, boolean nonNative) { if (this.mrwork == null) { @@ -618,8 +629,7 @@ protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass // subdirectories. (Unlike non-native tables, prefix mixups don't seem // to be a potential problem here since we are always dealing with the // path to something deeper than the table location.) - match = - splitPath.startsWith(key) || splitPathWithNoSchema.startsWith(key); + match = isMatch(splitPath, key) || isMatch(splitPathWithNoSchema, key); } if (match) { ArrayList list = entry.getValue(); diff --git ql/src/test/queries/clientpositive/orc_mr_pathalias.q ql/src/test/queries/clientpositive/orc_mr_pathalias.q new file mode 100644 index 0000000..d77d07a --- /dev/null +++ ql/src/test/queries/clientpositive/orc_mr_pathalias.q @@ -0,0 +1,19 @@ +create database if not exists test; +drop table if exists test.test_orc_src; +drop table if exists test.test_orc_src2; +create table test.test_orc_src (a int, b int, c int) stored as orc; +create table test.test_orc_src2 (a int, b int, d int) stored as orc; +insert overwrite table test.test_orc_src select 1,2,3 from src limit 1; +insert overwrite table test.test_orc_src2 select 1,2,4 from src limit 1; + +set hive.auto.convert.join = false; +set hive.execution.engine = mr; + +select + tb.c +from test.test_orc_src tb +join ( + select * from test.test_orc_src2 +) tm +on tb.a = tm.a +where tb.b = 2; diff --git ql/src/test/results/clientpositive/orc_mr_pathalias.q.out ql/src/test/results/clientpositive/orc_mr_pathalias.q.out new file mode 100644 index 0000000..ec2f6a1 --- /dev/null +++ ql/src/test/results/clientpositive/orc_mr_pathalias.q.out @@ -0,0 +1,77 @@ +PREHOOK: query: create database if not exists test +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:test +POSTHOOK: query: create database if not exists test +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:test +PREHOOK: query: drop table if exists test.test_orc_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists test.test_orc_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists test.test_orc_src2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists test.test_orc_src2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table test.test_orc_src (a int, b int, c int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:test +PREHOOK: Output: test@test_orc_src +POSTHOOK: query: create table test.test_orc_src (a int, b int, c int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:test +POSTHOOK: Output: test@test_orc_src +PREHOOK: query: create table test.test_orc_src2 (a int, b int, d int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:test +PREHOOK: Output: test@test_orc_src2 +POSTHOOK: query: create table test.test_orc_src2 (a int, b int, d int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:test +POSTHOOK: Output: test@test_orc_src2 +PREHOOK: query: insert overwrite table test.test_orc_src select 1,2,3 from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: test@test_orc_src +POSTHOOK: query: insert overwrite table test.test_orc_src select 1,2,3 from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: test@test_orc_src +POSTHOOK: Lineage: test_orc_src.a SIMPLE [] +POSTHOOK: Lineage: test_orc_src.b SIMPLE [] +POSTHOOK: Lineage: test_orc_src.c SIMPLE [] +PREHOOK: query: insert overwrite table test.test_orc_src2 select 1,2,4 from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: test@test_orc_src2 +POSTHOOK: query: insert overwrite table test.test_orc_src2 select 1,2,4 from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: test@test_orc_src2 +POSTHOOK: Lineage: test_orc_src2.a SIMPLE [] +POSTHOOK: Lineage: test_orc_src2.b SIMPLE [] +POSTHOOK: Lineage: test_orc_src2.d SIMPLE [] +PREHOOK: query: select + tb.c +from test.test_orc_src tb +join ( + select * from test.test_orc_src2 +) tm +on tb.a = tm.a +where tb.b = 2 +PREHOOK: type: QUERY +PREHOOK: Input: test@test_orc_src +PREHOOK: Input: test@test_orc_src2 +#### A masked pattern was here #### +POSTHOOK: query: select + tb.c +from test.test_orc_src tb +join ( + select * from test.test_orc_src2 +) tm +on tb.a = tm.a +where tb.b = 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: test@test_orc_src +POSTHOOK: Input: test@test_orc_src2 +#### A masked pattern was here #### +3