From 1703a7723d148f0891a6358cfa6aaca481f7107a Mon Sep 17 00:00:00 2001 From: "David Z. Chen" Date: Tue, 17 Jun 2014 13:01:43 -0700 Subject: [PATCH] HIVE-7286: Parameterize HCatMapReduceTest for testing against all Hive storage formats. --- .gitignore | 1 + hcatalog/core/pom.xml | 7 + .../fileformats/TestOrcDynamicPartitioned.java | 52 ------- .../hive/hcatalog/mapreduce/HCatMapReduceTest.java | 171 ++++++++++++--------- .../mapreduce/TestHCatDynamicPartitioned.java | 9 +- .../TestHCatExternalDynamicPartitioned.java | 16 +- .../mapreduce/TestHCatExternalNonPartitioned.java | 5 + .../mapreduce/TestHCatExternalPartitioned.java | 5 + .../TestHCatMutableDynamicPartitioned.java | 5 + .../mapreduce/TestHCatMutableNonPartitioned.java | 6 +- .../mapreduce/TestHCatMutablePartitioned.java | 5 + .../hcatalog/mapreduce/TestHCatNonPartitioned.java | 10 +- .../hcatalog/mapreduce/TestHCatPartitioned.java | 9 +- ql/pom.xml | 8 + .../apache/hadoop/hive/ql/io/StorageFormats.java | 139 +++++++++++++++++ serde/src/test/resources/alltypes.avsc | 48 ++++++ 16 files changed, 346 insertions(+), 150 deletions(-) delete mode 100644 hcatalog/core/src/test/java/org/apache/hive/hcatalog/fileformats/TestOrcDynamicPartitioned.java create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java create mode 100644 serde/src/test/resources/alltypes.avsc diff --git a/.gitignore b/.gitignore index d0c97d1..fa9773a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ common/src/gen *.iml *.ipr *.iws +*.swp derby.log datanucleus.log .arc diff --git a/hcatalog/core/pom.xml b/hcatalog/core/pom.xml index b5e85cd..09856c6 100644 --- a/hcatalog/core/pom.xml +++ b/hcatalog/core/pom.xml @@ -60,6 +60,13 @@ hive-exec ${project.version} + + org.apache.hive + hive-ql + ${project.version} + test-jar + test + com.google.guava diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/fileformats/TestOrcDynamicPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/fileformats/TestOrcDynamicPartitioned.java deleted file mode 100644 index f68dbb8..0000000 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/fileformats/TestOrcDynamicPartitioned.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.fileformats; - -import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcSerde; -import org.apache.hive.hcatalog.mapreduce.TestHCatDynamicPartitioned; -import org.junit.BeforeClass; - -public class TestOrcDynamicPartitioned extends TestHCatDynamicPartitioned { - - @BeforeClass - public static void generateInputData() throws Exception { - tableName = "testOrcDynamicPartitionedTable"; - generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0); - generateDataColumns(); - } - - @Override - protected String inputFormat() { - return OrcInputFormat.class.getName(); - } - - @Override - protected String outputFormat() { - return OrcOutputFormat.class.getName(); - } - - @Override - protected String serdeClass() { - return OrcSerde.class.getName(); - } - -} diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java index 9ddc3a6..ee57f3f 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java @@ -19,13 +19,15 @@ package org.apache.hive.hcatalog.mapreduce; +import com.google.common.collect.ImmutableSet; + import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; - -import junit.framework.Assert; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -40,10 +42,10 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.hive.ql.io.RCFileInputFormat; -import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; -import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; +import org.apache.hadoop.hive.ql.io.StorageFormats; +import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.avro.AvroSerDe; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; @@ -53,15 +55,23 @@ import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; + import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.DefaultHCatRecord; import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchema; + +import junit.framework.Assert; + import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,42 +79,62 @@ /** * Test for HCatOutputFormat. Writes a partition using HCatOutputFormat and reads - * it back using HCatInputFormat, checks the column values and counts. + * it back using HCatInputFormat, checks the column values and counts. This class + * can be tested to test different partitioning schemes. + * + * This is a parameterized test that tests HCatOutputFormat and HCatInputFormat against Hive's + * native storage formats enumerated using {@link org.apache.hive.hcatalog.mapreduce.StorageFormats}. */ +@RunWith(Parameterized.class) public abstract class HCatMapReduceTest extends HCatBaseTest { - private static final Logger LOG = LoggerFactory.getLogger(HCatMapReduceTest.class); + protected static String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; - protected static String tableName = "testHCatMapReduceTable"; + protected static final String TABLE_NAME = "testHCatMapReduceTable"; private static List writeRecords = new ArrayList(); private static List readRecords = new ArrayList(); - protected abstract List getPartitionKeys(); - - protected abstract List getTableColumns(); - private static FileSystem fs; private String externalTableLocation = null; + protected String tableName; + protected String serdeClass; + protected String inputFormatClass; + protected String outputFormatClass; - protected Boolean isTableExternal() { - return false; - } + /** + * List of SerDe classes that the HCatalog core tests will not be run against. + */ + public static final Set DISABLED_SERDES = ImmutableSet.of( + AvroSerDe.class.getName(), + ParquetHiveSerDe.class.getName()); - protected boolean isTableImmutable() { - return true; + @Parameterized.Parameters + public static Collection generateParameters() { + return StorageFormats.asParameters(); } - protected String inputFormat() { - return RCFileInputFormat.class.getName(); + /** + * Test constructor that sets the storage format class names provided by the test parameter. + */ + public HCatMapReduceTest(String name, String serdeClass, String inputFormatClass, + String outputFormatClass) throws Exception { + this.serdeClass = serdeClass; + this.inputFormatClass = inputFormatClass; + this.outputFormatClass = outputFormatClass; + this.tableName = TABLE_NAME + "_" + name; } - protected String outputFormat() { - return RCFileOutputFormat.class.getName(); + protected abstract List getPartitionKeys(); + + protected abstract List getTableColumns(); + + protected Boolean isTableExternal() { + return false; } - protected String serdeClass() { - return ColumnarSerDe.class.getName(); + protected boolean isTableImmutable() { + return true; } @BeforeClass @@ -143,13 +173,16 @@ public void deleteTable() throws Exception { @Before public void createTable() throws Exception { - String databaseName = (dbName == null) ? MetaStoreUtils.DEFAULT_DATABASE_NAME : dbName; + // Use Junit's Assume to skip running this fixture against any storage formats whose + // SerDe is in the disabled serdes list. + Assume.assumeTrue(!DISABLED_SERDES.contains(serdeClass)); + String databaseName = (dbName == null) ? MetaStoreUtils.DEFAULT_DATABASE_NAME : dbName; try { client.dropTable(databaseName, tableName); } catch (Exception e) { - } //can fail with NoSuchObjectException - + // Can fail with NoSuchObjectException. + } Table tbl = new Table(); tbl.setDbName(databaseName); @@ -160,10 +193,9 @@ public void createTable() throws Exception { tbl.setTableType(TableType.MANAGED_TABLE.toString()); } StorageDescriptor sd = new StorageDescriptor(); - sd.setCols(getTableColumns()); - tbl.setPartitionKeys(getPartitionKeys()); + tbl.setPartitionKeys(getPartitionKeys()); tbl.setSd(sd); sd.setBucketCols(new ArrayList(2)); @@ -171,12 +203,12 @@ public void createTable() throws Exception { sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); - if (isTableExternal()){ + if (isTableExternal()) { sd.getSerdeInfo().getParameters().put("EXTERNAL", "TRUE"); } - sd.getSerdeInfo().setSerializationLib(serdeClass()); - sd.setInputFormat(inputFormat()); - sd.setOutputFormat(outputFormat()); + sd.getSerdeInfo().setSerializationLib(serdeClass); + sd.setInputFormat(inputFormatClass); + sd.setOutputFormat(outputFormatClass); Map tableParams = new HashMap(); if (isTableExternal()) { @@ -190,68 +222,59 @@ public void createTable() throws Exception { client.createTable(tbl); } - //Create test input file with specified number of rows + /* + * Create test input file with specified number of rows + */ private void createInputFile(Path path, int rowCount) throws IOException { - if (fs.exists(path)) { fs.delete(path, true); } FSDataOutputStream os = fs.create(path); - for (int i = 0; i < rowCount; i++) { os.writeChars(i + "\n"); } - os.close(); } - public static class MapCreate extends - Mapper { - - static int writeCount = 0; //test will be in local mode + public static class MapCreate extends Mapper { + // Test will be in local mode. + static int writeCount = 0; @Override - public void map(LongWritable key, Text value, Context context - ) throws IOException, InterruptedException { - { - try { - HCatRecord rec = writeRecords.get(writeCount); - context.write(null, rec); - writeCount++; - - } catch (Exception e) { - - e.printStackTrace(System.err); //print since otherwise exception is lost - throw new IOException(e); - } + public void map(LongWritable key, Text value, Context context) + throws IOException, InterruptedException { + try { + HCatRecord rec = writeRecords.get(writeCount); + context.write(null, rec); + writeCount++; + } catch (Exception e) { + // Print since otherwise exception is lost. + e.printStackTrace(System.err); + throw new IOException(e); } } } - public static class MapRead extends - Mapper { - + public static class MapRead extends Mapper { static int readCount = 0; //test will be in local mode @Override - public void map(WritableComparable key, HCatRecord value, Context context - ) throws IOException, InterruptedException { - { - try { - readRecords.add(value); - readCount++; - } catch (Exception e) { - e.printStackTrace(); //print since otherwise exception is lost - throw new IOException(e); - } + public void map(WritableComparable key, HCatRecord value, Context context) + throws IOException, InterruptedException { + try { + readRecords.add(value); + readCount++; + } catch (Exception e) { + // Print since otherwise exception is lost. + e.printStackTrace(); + throw new IOException(e); } } } - Job runMRCreate(Map partitionValues, - List partitionColumns, List records, - int writeCount, boolean assertWrite) throws Exception { + Job runMRCreate(Map partitionValues, List partitionColumns, + List records, int writeCount, boolean assertWrite) throws Exception { return runMRCreate(partitionValues, partitionColumns, records, writeCount, assertWrite, true, null); } @@ -267,10 +290,9 @@ Job runMRCreate(Map partitionValues, * @return * @throws Exception */ - Job runMRCreate(Map partitionValues, - List partitionColumns, List records, - int writeCount, boolean assertWrite, boolean asSingleMapTask, - String customDynamicPathPattern) throws Exception { + Job runMRCreate(Map partitionValues, List partitionColumns, + List records, int writeCount, boolean assertWrite, boolean asSingleMapTask, + String customDynamicPathPattern) throws Exception { writeRecords = records; MapCreate.writeCount = 0; @@ -355,7 +377,6 @@ Job runMRCreate(Map partitionValues, * @throws Exception */ List runMRRead(int readCount, String filter) throws Exception { - MapRead.readCount = 0; readRecords.clear(); @@ -388,9 +409,7 @@ Job runMRCreate(Map partitionValues, return readRecords; } - protected HCatSchema getTableSchema() throws Exception { - Configuration conf = new Configuration(); Job job = new Job(conf, "hcat mapreduce read schema test"); job.setJarByClass(this.getClass()); diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java index 39e9208..0d87c6c 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java @@ -37,8 +37,10 @@ import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils; + import org.junit.BeforeClass; import org.junit.Test; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,9 +55,10 @@ protected static final int NUM_RECORDS = 20; protected static final int NUM_PARTITIONS = 5; - @BeforeClass - public static void generateInputData() throws Exception { - tableName = "testHCatDynamicPartitionedTable"; + public TestHCatDynamicPartitioned(String formatName, String serdeClass, String inputFormatClass, + String outputFormatClass) throws Exception { + super(formatName, serdeClass, inputFormatClass, outputFormatClass); + tableName = "testHCatDynamicPartitionedTable_" + formatName; generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0); generateDataColumns(); } diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java index 0838765..58764a5 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java @@ -24,18 +24,20 @@ public class TestHCatExternalDynamicPartitioned extends TestHCatDynamicPartitioned { + public TestHCatExternalDynamicPartitioned(String formatName, String serdeClass, + String inputFormatClass, String outputFormatClass) + throws Exception { + super(formatName, serdeClass, inputFormatClass, outputFormatClass); + tableName = "testHCatExternalDynamicPartitionedTable_" + formatName; + generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0); + generateDataColumns(); + } + @Override protected Boolean isTableExternal() { return true; } - @BeforeClass - public static void generateInputData() throws Exception { - tableName = "testHCatExternalDynamicPartitionedTable"; - generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0); - generateDataColumns(); - } - /** * Run the external dynamic partitioning test but with single map task * @throws Exception diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java index 01b2ad6..6e060c0 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java @@ -20,6 +20,11 @@ package org.apache.hive.hcatalog.mapreduce; public class TestHCatExternalNonPartitioned extends TestHCatNonPartitioned { + public TestHCatExternalNonPartitioned(String formatName, String serdeName, + String inputFormatClass, String outputFormatClass) + throws Exception { + super(formatName, serdeName, inputFormatClass, outputFormatClass); + } @Override protected Boolean isTableExternal() { diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java index e5f8d1e..9f16b3b 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java @@ -20,6 +20,11 @@ package org.apache.hive.hcatalog.mapreduce; public class TestHCatExternalPartitioned extends TestHCatPartitioned { + public TestHCatExternalPartitioned(String formatName, String serdeClass, + String inputFormatClass, String outputFormatClass) + throws Exception { + super(formatName, serdeClass, inputFormatClass, outputFormatClass); + } @Override protected Boolean isTableExternal() { diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java index bfc6a4f..5b18739 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java @@ -20,6 +20,11 @@ package org.apache.hive.hcatalog.mapreduce; public class TestHCatMutableDynamicPartitioned extends TestHCatDynamicPartitioned { + public TestHCatMutableDynamicPartitioned(String formatName, String serdeClass, + String inputFormatClass, String outputFormatClass) + throws Exception { + super(formatName, serdeClass, inputFormatClass, outputFormatClass); + } @Override protected boolean isTableImmutable() { diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java index a944023..354ae10 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java @@ -20,7 +20,11 @@ package org.apache.hive.hcatalog.mapreduce; public class TestHCatMutableNonPartitioned extends TestHCatNonPartitioned { - + public TestHCatMutableNonPartitioned(String formatName, String serdeClass, + String inputFormatClass, String outputFormatClass) + throws Exception { + super(formatName, serdeClass, inputFormatClass, outputFormatClass); + } @Override protected boolean isTableImmutable() { diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java index 73d6e80..a22a993 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java @@ -20,6 +20,11 @@ package org.apache.hive.hcatalog.mapreduce; public class TestHCatMutablePartitioned extends TestHCatPartitioned { + public TestHCatMutablePartitioned(String formatName, String serdeClass, + String inputFormatClass, String outputFormatClass) + throws Exception { + super(formatName, serdeClass, inputFormatClass, outputFormatClass); + } @Override protected boolean isTableImmutable() { diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java index d639e99..174a92f 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java @@ -43,16 +43,14 @@ import static org.junit.Assert.assertNull; public class TestHCatNonPartitioned extends HCatMapReduceTest { - private static List writeRecords; static List partitionColumns; - @BeforeClass - public static void oneTimeSetUp() throws Exception { - + public TestHCatNonPartitioned(String formatName, String serdeClass, String inputFormatClass, + String outputFormatClass) throws Exception { + super(formatName, serdeClass, inputFormatClass, outputFormatClass); dbName = null; //test if null dbName works ("default" is used) - tableName = "testHCatNonPartitionedTable"; - + tableName = "testHCatNonPartitionedTable_" + formatName; writeRecords = new ArrayList(); for (int i = 0; i < 20; i++) { diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java index dc05f43..a386415 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java @@ -49,10 +49,10 @@ private static List writeRecords; private static List partitionColumns; - @BeforeClass - public static void oneTimeSetUp() throws Exception { - - tableName = "testHCatPartitionedTable"; + public TestHCatPartitioned(String formatName, String serdeClass, String inputFormatClass, + String outputFormatClass) throws Exception { + super(formatName, serdeClass, inputFormatClass, outputFormatClass); + tableName = "testHCatPartitionedTable_" + formatName; writeRecords = new ArrayList(); for (int i = 0; i < 20; i++) { @@ -68,7 +68,6 @@ public static void oneTimeSetUp() throws Exception { partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c2", serdeConstants.STRING_TYPE_NAME, ""))); } - @Override protected List getPartitionKeys() { List fields = new ArrayList(); diff --git a/ql/pom.xml b/ql/pom.xml index 0729d47..085aad4 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -409,6 +409,14 @@ org.apache.maven.plugins + maven-surefire-plugin + 2.4.2 + + true + + + + org.apache.maven.plugins maven-antrun-plugin diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java b/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java new file mode 100644 index 0000000..92c7fde --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.ServiceLoader; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; +import org.apache.hadoop.hive.ql.io.RCFileStorageFormatDescriptor; +import org.apache.hadoop.hive.ql.io.StorageFormatDescriptor; +import org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat; +import org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; +import org.apache.hadoop.hive.serde2.avro.AvroSerDe; +import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; + +import static org.junit.Assert.assertTrue; + +/** + * Utility class for enumerating Hive native storage formats for testing. Native Storage formats + * are registered via {@link org.apache.hadoop.hive.ql.io.StorageFormatDescriptor}. + */ +public class StorageFormats { + /** + * Table of additional storage formats. These are SerDes or combinations of SerDe with + * InputFormat and OutputFormat that are not registered as a native Hive storage format. + * + * Each row in this table has the following fields: + * - formatName - A string name for the storage format. This is used to give the table created + * for the test a unique name. + * - serdeClass - The name of the SerDe class used by the storage format. + * - inputFormatClass - The name of the InputFormat class. + * - outputFormatClass - The name of the OutputFormat class. + */ + public static final Object[][] ADDITIONAL_STORAGE_FORMATS = new Object[][] { + { + "rcfile_columnar", + ColumnarSerDe.class.getName(), + RCFileInputFormat.class.getName(), + RCFileOutputFormat.class.getName(), + }, { + "avro", + AvroSerDe.class.getName(), + AvroContainerInputFormat.class.getName(), + AvroContainerOutputFormat.class.getName(), + } + }; + + /** + * Create an array of Objects used to populate the test paramters. + * + * @param name Name of the storage format. + * @param serdeClass Name of the SerDe class. + * @param inputFormatClass Name of the InputFormat class. + * @param outputFormatClass Name of the OutputFormat class. + * @return Object array containing the arguments. + */ + protected static Object[] createTestArguments(String name, String serdeClass, + String inputFormatClass, String outputFormatClass) { + Object[] args = { + name, + serdeClass, + inputFormatClass, + outputFormatClass + }; + return args; + } + + /** + * Generates a collection of parameters that can be used as paramters for a JUnit test fixture. + * Each parameter represents one storage format that the fixture will run against. The list + * includes both native Hive storage formats as well as those enumerated in the + * ADDITIONAL_STORAGE_FORMATS table. + * + * @return List of storage format as paramters. + */ + public static Collection asParameters() { + List parameters = new ArrayList(); + + // Add test parameters from official storage formats registered with Hive via + // StorageFormatDescriptor. + final Configuration conf = new Configuration(); + for (StorageFormatDescriptor descriptor : ServiceLoader.load(StorageFormatDescriptor.class)) { + String serdeClass = descriptor.getSerde(); + if (serdeClass == null) { + if (descriptor instanceof RCFileStorageFormatDescriptor) { + serdeClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE); + } else { + serdeClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTSERDE); + } + } + + String[] names = new String[descriptor.getNames().size()]; + names = descriptor.getNames().toArray(names); + Object[] arguments = createTestArguments(names[0], serdeClass, descriptor.getInputFormat(), + descriptor.getOutputFormat()); + parameters.add(arguments); + } + + // Add test parameters from storage formats specified in ADDITIONAL_STORAGE_FORMATS table. + for (int i = 0; i < ADDITIONAL_STORAGE_FORMATS.length; i++) { + String serdeClass = (String) ADDITIONAL_STORAGE_FORMATS[i][1]; + String name = (String) ADDITIONAL_STORAGE_FORMATS[i][0]; + String inputFormatClass = (String) ADDITIONAL_STORAGE_FORMATS[i][2]; + String outputFormatClass = (String) ADDITIONAL_STORAGE_FORMATS[i][3]; + assertTrue("InputFormat for storage format not set", inputFormatClass != null); + assertTrue("OutputFormat for storage format not set", outputFormatClass != null); + Object[] arguments = createTestArguments(name, serdeClass, inputFormatClass, + outputFormatClass); + parameters.add(arguments); + } + + return parameters; + } +} + diff --git a/serde/src/test/resources/alltypes.avsc b/serde/src/test/resources/alltypes.avsc new file mode 100644 index 0000000..39be023 --- /dev/null +++ b/serde/src/test/resources/alltypes.avsc @@ -0,0 +1,48 @@ +{ + "type" : "record", + "name" : "schema", + "fields" : [ { + "name" : "myboolean", + "type" : "boolean" + }, { + "name" : "myint", + "type" : "int" + }, { + "name" : "mylong", + "type" : "long" + }, { + "name" : "myfloat", + "type" : "float" + }, { + "name" : "mydouble", + "type" : "double" + }, { + "name" : "mybytes", + "type" : "bytes" + }, { + "name" : "mystring", + "type" : "string" + }, { + "name" : "myrecord", + "type" : { + "type" : "record", + "name" : "myrecord", + "fields" : [ { + "name" : "mynestedint", + "type" : "int" + } ] + } + }, { + "name" : "myarray", + "type" : { + "type" : "array", + "items" : "int" + } + }, { + "name" : "mymap", + "type" : { + "type" : "map", + "values" : "int" + } + } ] +} -- 1.8.3.4 (Apple Git-47)