diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java index 97277b59b4..a5cf3a5e2f 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java @@ -54,7 +54,7 @@ public abstract class AbstractHCatStorerTest extends HCatBaseTest { static Logger LOG = LoggerFactory.getLogger(AbstractHCatStorerTest.class); static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; - String storageFormat; + protected String storageFormat; public AbstractHCatStorerTest() { storageFormat = getStorageFormat(); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java index 477ea660d0..cb0213943d 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java @@ -86,8 +86,6 @@ } }; - private String storageFormat; - @Parameterized.Parameters public static Collection generateParameters() { return StorageFormats.names(); @@ -99,7 +97,7 @@ public TestHCatStorer(String storageFormat) { @Override String getStorageFormat() { - return null; + return this.storageFormat; } @Test diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java b/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java index f60d296fe9..2be864e752 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOConstants.java @@ -35,6 +35,7 @@ public static final String PARQUETFILE = "PARQUETFILE"; public static final String AVRO = "AVRO"; public static final String AVROFILE = "AVROFILE"; + public static final String JSONFILE = "JSONFILE"; /** * The desired TABLE column names and types for input format schema evolution. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/JsonFileStorageFormatDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/io/JsonFileStorageFormatDescriptor.java new file mode 100644 index 0000000000..0020cfe149 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/JsonFileStorageFormatDescriptor.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.Set; + +import org.apache.hadoop.hive.serde2.JsonSerDe; + +import com.google.common.collect.ImmutableSet; + +public class JsonFileStorageFormatDescriptor extends AbstractStorageFormatDescriptor { + @Override + public Set getNames() { + return ImmutableSet.of(IOConstants.JSONFILE); + } + + @Override + public String getInputFormat() { + return IOConstants.TEXTFILE_INPUT; + } + + @Override + public String getOutputFormat() { + return IOConstants.TEXTFILE_OUTPUT; + } + + @Override + public String getSerde() { + return JsonSerDe.class.getName(); + } +} diff --git a/ql/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor b/ql/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor index d858a95cc7..c28a3026d4 100644 --- a/ql/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor +++ b/ql/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor @@ -4,3 +4,4 @@ org.apache.hadoop.hive.ql.io.RCFileStorageFormatDescriptor org.apache.hadoop.hive.ql.io.ORCFileStorageFormatDescriptor org.apache.hadoop.hive.ql.io.ParquetFileStorageFormatDescriptor org.apache.hadoop.hive.ql.io.AvroStorageFormatDescriptor +org.apache.hadoop.hive.ql.io.JsonFileStorageFormatDescriptor \ No newline at end of file diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestStorageFormatDescriptor.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestStorageFormatDescriptor.java index 72acaad872..86d37031ea 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestStorageFormatDescriptor.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestStorageFormatDescriptor.java @@ -40,5 +40,8 @@ public void testNames() { (new ParquetFileStorageFormatDescriptor()).getNames()); Assert.assertEquals(Sets.newHashSet(IOConstants.AVRO, IOConstants.AVROFILE), (new AvroStorageFormatDescriptor()).getNames()); + Assert.assertEquals(Sets.newHashSet(IOConstants.JSONFILE), + (new JsonFileStorageFormatDescriptor()).getNames()); + } } diff --git a/ql/src/test/queries/clientpositive/json_serde1.q b/ql/src/test/queries/clientpositive/json_serde1.q index b805925bfe..fcbf1c0752 100644 --- a/ql/src/test/queries/clientpositive/json_serde1.q +++ b/ql/src/test/queries/clientpositive/json_serde1.q @@ -1,9 +1,8 @@ --! qt:dataset:src -add jar ${system:maven.local.repository}/org/apache/hive/hcatalog/hive-hcatalog-core/${system:hive.version}/hive-hcatalog-core-${system:hive.version}.jar; - drop table if exists json_serde1_1; drop table if exists json_serde1_2; +drop table if exists json_serde1_3; create table json_serde1_1 (a array,b map) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'; @@ -17,7 +16,7 @@ create table json_serde1_2 ( a array, b map, c struct, c4:map, c5:struct> -) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'; +) row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe'; insert into table json_serde1_2 select @@ -33,5 +32,9 @@ insert into table json_serde1_2 select * from json_serde1_2; +create table json_serde1_3 (c1 int, c2 string) stored as jsonfile; +show create table json_serde1_3; + drop table json_serde1_1; drop table json_serde1_2; +drop table json_serde1_3; diff --git a/ql/src/test/results/clientpositive/json_serde1.q.out b/ql/src/test/results/clientpositive/json_serde1.q.out index e14d67463c..341a494e0c 100644 --- a/ql/src/test/results/clientpositive/json_serde1.q.out +++ b/ql/src/test/results/clientpositive/json_serde1.q.out @@ -6,6 +6,10 @@ PREHOOK: query: drop table if exists json_serde1_2 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table if exists json_serde1_2 POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists json_serde1_3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists json_serde1_3 +POSTHOOK: type: DROPTABLE PREHOOK: query: create table json_serde1_1 (a array,b map) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' PREHOOK: type: CREATETABLE @@ -42,7 +46,7 @@ PREHOOK: query: create table json_serde1_2 ( a array, b map, c struct, c4:map, c5:struct> -) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +) row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe' PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@json_serde1_2 @@ -50,7 +54,7 @@ POSTHOOK: query: create table json_serde1_2 ( a array, b map, c struct, c4:map, c5:struct> -) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +) row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe' POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@json_serde1_2 @@ -95,6 +99,34 @@ POSTHOOK: Input: default@json_serde1_2 #### A masked pattern was here #### [3,2,1] {1:"2001-01-01",2:null} {"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"abc":123,"xyz":456},"c5":{"c5_1":"bye","c5_2":88}} [3,2,1] {1:"2001-01-01",2:null} {"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"abc":123,"xyz":456},"c5":{"c5_1":"bye","c5_2":88}} +PREHOOK: query: create table json_serde1_3 (c1 int, c2 string) stored as jsonfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde1_3 +POSTHOOK: query: create table json_serde1_3 (c1 int, c2 string) stored as jsonfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde1_3 +PREHOOK: query: show create table json_serde1_3 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@json_serde1_3 +POSTHOOK: query: show create table json_serde1_3 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@json_serde1_3 +CREATE TABLE `json_serde1_3`( + `c1` int COMMENT 'from deserializer', + `c2` string COMMENT 'from deserializer') +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.JsonSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### PREHOOK: query: drop table json_serde1_1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@json_serde1_1 @@ -111,3 +143,11 @@ POSTHOOK: query: drop table json_serde1_2 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@json_serde1_2 POSTHOOK: Output: default@json_serde1_2 +PREHOOK: query: drop table json_serde1_3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@json_serde1_3 +PREHOOK: Output: default@json_serde1_3 +POSTHOOK: query: drop table json_serde1_3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@json_serde1_3 +POSTHOOK: Output: default@json_serde1_3