diff --git build.properties build.properties index 2d293a6..15287c7 100644 --- build.properties +++ build.properties @@ -27,14 +27,14 @@ javac.args= javac.args.warnings= hadoop-0.20.version=0.20.2 -hadoop-0.20S.version=1.0.0 +hadoop-0.20S.version=1.1.2 hadoop-0.23.version=2.0.0-alpha hadoop.version=${hadoop-0.20.version} hadoop.security.version=${hadoop-0.20S.version} # Used to determine which set of Hadoop artifacts we depend on. # - 20: hadoop-core, hadoop-test # - 23: hadoop-common, hadoop-mapreduce-*, etc -hadoop.mr.rev=20 +hadoop.mr.rev=20S build.dir.hive=${hive.root}/build build.dir.hadoop=${build.dir.hive}/hadoopcore diff --git ql/src/test/queries/clientpositive/archive_excludeHadoop20.q ql/src/test/queries/clientpositive/archive_excludeHadoop20.q index 50c0faa..ddc06a9 100644 --- ql/src/test/queries/clientpositive/archive_excludeHadoop20.q +++ ql/src/test/queries/clientpositive/archive_excludeHadoop20.q @@ -1,7 +1,7 @@ set hive.archive.enabled = true; set hive.enforce.bucketing = true; --- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) drop table tstsrc; drop table tstsrcpart; diff --git ql/src/test/queries/clientpositive/auto_join14.q ql/src/test/queries/clientpositive/auto_join14.q index cbdc5e2..b282fb9 100644 --- ql/src/test/queries/clientpositive/auto_join14.q +++ ql/src/test/queries/clientpositive/auto_join14.q @@ -1,7 +1,7 @@ set hive.auto.convert.join = true; --- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE; diff --git ql/src/test/queries/clientpositive/auto_join14_hadoop20.q ql/src/test/queries/clientpositive/auto_join14_hadoop20.q index e7ef6cf..235b7c1 100644 --- ql/src/test/queries/clientpositive/auto_join14_hadoop20.q +++ ql/src/test/queries/clientpositive/auto_join14_hadoop20.q @@ -1,7 +1,7 @@ set hive.auto.convert.join = true; --- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE; diff --git ql/src/test/queries/clientpositive/ctas.q ql/src/test/queries/clientpositive/ctas.q index db91e2d..e595904 100644 --- ql/src/test/queries/clientpositive/ctas.q +++ ql/src/test/queries/clientpositive/ctas.q @@ -1,4 +1,4 @@ --- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) create table nzhang_Tmp(a int, b string); select * from nzhang_Tmp; diff --git ql/src/test/queries/clientpositive/ctas_hadoop20.q ql/src/test/queries/clientpositive/ctas_hadoop20.q index 4927dea..4961b97 100644 --- ql/src/test/queries/clientpositive/ctas_hadoop20.q +++ ql/src/test/queries/clientpositive/ctas_hadoop20.q @@ -1,4 +1,4 @@ --- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) create table nzhang_Tmp(a int, b string); select * from nzhang_Tmp; diff --git ql/src/test/queries/clientpositive/input12.q ql/src/test/queries/clientpositive/input12.q index 72817d5..d4bc409 100644 --- ql/src/test/queries/clientpositive/input12.q +++ ql/src/test/queries/clientpositive/input12.q @@ -2,7 +2,7 @@ set mapreduce.framework.name=yarn; set mapreduce.jobtracker.address=localhost:58; set hive.exec.mode.local.auto=true; --- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; CREATE TABLE dest2(key INT, value STRING) STORED AS TEXTFILE; diff --git ql/src/test/queries/clientpositive/input12_hadoop20.q ql/src/test/queries/clientpositive/input12_hadoop20.q index 86751af..318cd37 100644 --- ql/src/test/queries/clientpositive/input12_hadoop20.q +++ ql/src/test/queries/clientpositive/input12_hadoop20.q @@ -1,7 +1,7 @@ set mapred.job.tracker=localhost:58; set hive.exec.mode.local.auto=true; --- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; CREATE TABLE dest2(key INT, value STRING) STORED AS TEXTFILE; diff --git ql/src/test/queries/clientpositive/join14.q ql/src/test/queries/clientpositive/join14.q index b930b29..83346b4 100644 --- ql/src/test/queries/clientpositive/join14.q +++ ql/src/test/queries/clientpositive/join14.q @@ -1,4 +1,4 @@ --- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE; diff --git ql/src/test/queries/clientpositive/join14_hadoop20.q ql/src/test/queries/clientpositive/join14_hadoop20.q index 0b98138..a12ef1a 100644 --- ql/src/test/queries/clientpositive/join14_hadoop20.q +++ ql/src/test/queries/clientpositive/join14_hadoop20.q @@ -1,4 +1,4 @@ --- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE; diff --git ql/src/test/results/clientpositive/auto_join14.q.out ql/src/test/results/clientpositive/auto_join14.q.out index d0531c4..56d48b2 100644 --- ql/src/test/results/clientpositive/auto_join14.q.out +++ ql/src/test/results/clientpositive/auto_join14.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE -POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE @@ -80,26 +80,19 @@ STAGE PLANS: Position of Big Table: 0 Select Operator expressions: - expr: _col0 - type: string + expr: UDFToInteger(_col0) + type: int expr: _col5 type: string outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col1 - type: string - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Local Work: Map Reduce Local Work @@ -164,26 +157,19 @@ STAGE PLANS: Position of Big Table: 1 Select Operator expressions: - expr: _col0 - type: string + expr: UDFToInteger(_col0) + type: int expr: _col5 type: string outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col1 - type: string - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Local Work: Map Reduce Local Work @@ -239,32 +225,26 @@ STAGE PLANS: outputColumnNames: _col0, _col5 Select Operator expressions: - expr: _col0 - type: string + expr: UDFToInteger(_col0) + type: int expr: _col5 type: string outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col1 - type: string - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value PREHOOK: type: QUERY PREHOOK: Input: default@src +PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Output: default@dest1 @@ -272,6 +252,7 @@ POSTHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src +POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Output: default@dest1 diff --git ql/src/test/results/clientpositive/auto_join14_hadoop20.q.out ql/src/test/results/clientpositive/auto_join14_hadoop20.q.out index b75f26d..8eabae0 100644 --- ql/src/test/results/clientpositive/auto_join14_hadoop20.q.out +++ ql/src/test/results/clientpositive/auto_join14_hadoop20.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE -POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/ctas_hadoop20.q.out ql/src/test/results/clientpositive/ctas_hadoop20.q.out index f4f07e5..ead9ea5 100644 --- ql/src/test/results/clientpositive/ctas_hadoop20.q.out +++ ql/src/test/results/clientpositive/ctas_hadoop20.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) create table nzhang_Tmp(a int, b string) PREHOOK: type: CREATETABLE -POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) create table nzhang_Tmp(a int, b string) POSTHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/input12.q.out ql/src/test/results/clientpositive/input12.q.out index e4bdf24..48a1ff4 100644 --- ql/src/test/results/clientpositive/input12.q.out +++ ql/src/test/results/clientpositive/input12.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE -POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE @@ -69,74 +69,55 @@ STAGE PLANS: type: boolean Select Operator expressions: - expr: key - type: string + expr: UDFToInteger(key) + type: int expr: value type: string outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col1 - type: string - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Filter Operator predicate: expr: ((key >= 100.0) and (key < 200.0)) type: boolean Select Operator expressions: - expr: key - type: string + expr: UDFToInteger(key) + type: int expr: value type: string outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col1 - type: string - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 2 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest2 + File Output Operator + compressed: false + GlobalTableId: 2 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest2 Filter Operator predicate: expr: (key >= 200.0) type: boolean Select Operator expressions: - expr: key - type: string + expr: UDFToInteger(key) + type: int outputColumnNames: _col0 - Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 3 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest3 + File Output Operator + compressed: false + GlobalTableId: 3 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest3 Stage: Stage-9 Conditional Operator diff --git ql/src/test/results/clientpositive/input12_hadoop20.q.out ql/src/test/results/clientpositive/input12_hadoop20.q.out index c69d52e..9772c7b 100644 --- ql/src/test/results/clientpositive/input12_hadoop20.q.out +++ ql/src/test/results/clientpositive/input12_hadoop20.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE -POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/join14.q.out ql/src/test/results/clientpositive/join14.q.out index a0399c9..fd1b44f 100644 --- ql/src/test/results/clientpositive/join14.q.out +++ ql/src/test/results/clientpositive/join14.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE -POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE @@ -76,26 +76,19 @@ STAGE PLANS: outputColumnNames: _col0, _col5 Select Operator expressions: - expr: _col0 - type: string + expr: UDFToInteger(_col0) + type: int expr: _col5 type: string outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col1 - type: string - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest1 Stage: Stage-0 Move Operator @@ -115,6 +108,7 @@ PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value PREHOOK: type: QUERY PREHOOK: Input: default@src +PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Output: default@dest1 @@ -122,6 +116,7 @@ POSTHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src +POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Output: default@dest1 diff --git ql/src/test/results/clientpositive/join14_hadoop20.q.out ql/src/test/results/clientpositive/join14_hadoop20.q.out index 01f63df..41856dd 100644 --- ql/src/test/results/clientpositive/join14_hadoop20.q.out +++ ql/src/test/results/clientpositive/join14_hadoop20.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE -POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20) +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE POSTHOOK: type: CREATETABLE