From 8368cb1b57397c4e50c112445da05eb101dfe272 Mon Sep 17 00:00:00 2001 From: "David Z. Chen" Date: Tue, 22 Jul 2014 17:54:52 -0700 Subject: [PATCH] HIVE-7457: Minor HCatalog Pig Adapter test clean up. --- .../org/apache/hive/hcatalog/pig/MockLoader.java | 1 + .../apache/hive/hcatalog/pig/TestE2EScenarios.java | 30 ++++++++-------- .../apache/hive/hcatalog/pig/TestHCatLoader.java | 25 +++++++++----- .../hcatalog/pig/TestHCatLoaderComplexSchema.java | 7 ++-- .../apache/hive/hcatalog/pig/TestHCatStorer.java | 40 ++++++++++++++++------ .../hive/hcatalog/pig/TestHCatStorerMulti.java | 27 +++++++++------ .../hive/hcatalog/pig/TestHCatStorerWrapper.java | 3 ++ .../hive/hcatalog/pig/TestOrcHCatPigStorer.java | 31 ----------------- .../hive/hcatalog/pig/TestOrcHCatStorer.java | 14 +++++--- .../hive/hcatalog/pig/TestOrcHCatStorerMulti.java | 28 +++++++++++++++ .../apache/hive/hcatalog/pig/TestPigHCatUtil.java | 4 +++ 11 files changed, 128 insertions(+), 82 deletions(-) delete mode 100644 hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java create mode 100644 hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java index c87b95a..4b52faf 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java @@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; + import org.apache.pig.LoadFunc; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; import org.apache.pig.data.Tuple; diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java index a4b55c8..fe4c9f7 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java @@ -23,9 +23,8 @@ import java.util.HashMap; import java.util.Iterator; -import junit.framework.TestCase; - import org.apache.commons.io.FileUtils; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hive.cli.CliSessionState; @@ -42,6 +41,7 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; + import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.common.HCatContext; @@ -51,12 +51,16 @@ import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil; + import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.Tuple; -public class TestE2EScenarios extends TestCase { +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +public class TestE2EScenarios { private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis(); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; @@ -69,9 +73,8 @@ protected String storageFormat() { return "orc"; } - @Override - protected void setUp() throws Exception { - + @Before + public void setUp() throws Exception { File f = new File(TEST_WAREHOUSE_DIR); if (f.exists()) { FileUtil.fullyDelete(f); @@ -90,8 +93,8 @@ protected void setUp() throws Exception { } - @Override - protected void tearDown() throws Exception { + @After + public void tearDown() throws Exception { try { dropTable("inpy"); dropTable("rc5318"); @@ -146,16 +149,13 @@ private void pigDump(String tableName) throws IOException { System.err.println("==="); } - private void copyTable(String in, String out) throws IOException, InterruptedException { Job ijob = new Job(); Job ojob = new Job(); HCatInputFormat inpy = new HCatInputFormat(); inpy.setInput(ijob , null, in); HCatOutputFormat oupy = new HCatOutputFormat(); - oupy.setOutput(ojob, - OutputJobInfo.create(null, out, new HashMap() - )); + oupy.setOutput(ojob, OutputJobInfo.create(null, out, new HashMap())); // Test HCatContext @@ -207,6 +207,7 @@ private TaskAttemptContext createTaskAttemptContext(Configuration tconf) { } + @Test public void testReadOrcAndRCFromPig() throws Exception { String tableSchema = "ti tinyint, si smallint,i int, bi bigint, f float, d double, b boolean"; @@ -224,15 +225,14 @@ public void testReadOrcAndRCFromPig() throws Exception { driverRun("LOAD DATA LOCAL INPATH '"+TEXTFILE_LOCN+"' OVERWRITE INTO TABLE inpy"); // write it out from hive to an rcfile table, and to an orc table -// driverRun("insert overwrite table rc5318 select * from inpy"); + //driverRun("insert overwrite table rc5318 select * from inpy"); copyTable("inpy","rc5318"); -// driverRun("insert overwrite table orc5318 select * from inpy"); + //driverRun("insert overwrite table orc5318 select * from inpy"); copyTable("inpy","orc5318"); pigDump("inpy"); pigDump("rc5318"); pigDump("orc5318"); - } } diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java index 82fc8a9..ee3e750 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java @@ -34,6 +34,7 @@ import java.util.Properties; import org.apache.commons.io.FileUtils; + import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; @@ -45,11 +46,13 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.mapreduce.Job; + import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.data.Pair; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; + import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.ResourceStatistics; @@ -60,10 +63,13 @@ import org.apache.pig.PigRunner; import org.apache.pig.tools.pigstats.OutputStats; import org.apache.pig.tools.pigstats.PigStats; + import org.joda.time.DateTime; + import org.junit.After; import org.junit.Before; import org.junit.Test; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,7 +78,7 @@ public class TestHCatLoader { private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoader.class); private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") + - File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis()); + File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis()); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data"; private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data"; @@ -93,6 +99,7 @@ protected String storageFormat() { private void dropTable(String tablename) throws IOException, CommandNeedRetryException { dropTable(tablename, driver); } + static void dropTable(String tablename, Driver driver) throws IOException, CommandNeedRetryException { driver.run("drop table if exists " + tablename); } @@ -100,7 +107,8 @@ static void dropTable(String tablename, Driver driver) throws IOException, Comma private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException { createTable(tablename, schema, partitionedBy, driver, storageFormat()); } - static void createTable(String tablename, String schema, String partitionedBy, Driver driver, String storageFormat) + + static void createTable(String tablename, String schema, String partitionedBy, Driver driver, String storageFormat) throws IOException, CommandNeedRetryException { String createTable; createTable = "create table " + tablename + "(" + schema + ") "; @@ -114,6 +122,7 @@ static void createTable(String tablename, String schema, String partitionedBy, D private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException { createTable(tablename, schema, null); } + /** * Execute Hive CLI statement * @param cmd arbitrary statement to execute @@ -125,20 +134,20 @@ static void executeStatementOnDriver(String cmd, Driver driver) throws IOExcepti throw new IOException("Failed to execute \"" + cmd + "\". Driver returned " + cpr.getResponseCode() + " Error: " + cpr.getErrorMessage()); } } + private static void checkProjection(FieldSchema fs, String expectedName, byte expectedPigType) { assertEquals(fs.alias, expectedName); assertEquals("Expected " + DataType.findTypeName(expectedPigType) + "; got " + DataType.findTypeName(fs.type), expectedPigType, fs.type); } - + @Before public void setup() throws Exception { - File f = new File(TEST_WAREHOUSE_DIR); if (f.exists()) { FileUtil.fullyDelete(f); } - if(!(new File(TEST_WAREHOUSE_DIR).mkdirs())) { + if (!(new File(TEST_WAREHOUSE_DIR).mkdirs())) { throw new RuntimeException("Could not create " + TEST_WAREHOUSE_DIR); } @@ -192,7 +201,7 @@ public void setup() throws Exception { server.registerQuery("B = foreach A generate a,b;", ++i); server.registerQuery("B2 = filter B by a < 2;", ++i); server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');", ++i); - + server.registerQuery("C = foreach A generate a,b;", ++i); server.registerQuery("C2 = filter C by a >= 2;", ++i); server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');", ++i); @@ -470,7 +479,7 @@ public void testColumnarStorePushdown() throws Exception { { fs.delete(new Path(PIGOUTPUT_DIR), true); } - }finally { + } finally { new File(PIG_FILE).delete(); } } @@ -534,7 +543,7 @@ public void testConvertBooleanToInt() throws Exception { } /** - * basic tests that cover each scalar type + * basic tests that cover each scalar type * https://issues.apache.org/jira/browse/HIVE-5814 */ private static final class AllTypesTable { diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java index eadbf20..40ec597 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; + import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecException; @@ -44,8 +45,10 @@ import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; + import org.junit.BeforeClass; import org.junit.Test; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -225,7 +228,7 @@ private void verifyWriteRead(String tablename, String pigSchema, String tableSch dropTable(tablename); } } - + private void compareTuples(Tuple t1, Tuple t2) throws ExecException { Assert.assertEquals("Tuple Sizes don't match", t1.size(), t2.size()); for (int i = 0; i < t1.size(); i++) { @@ -237,7 +240,7 @@ private void compareTuples(Tuple t1, Tuple t2) throws ExecException { Assert.assertEquals(msg, noOrder(f1.toString()), noOrder(f2.toString())); } } - + private String noOrder(String s) { char[] chars = s.toCharArray(); Arrays.sort(chars); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java index fcfc642..763af9f 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java @@ -31,8 +31,10 @@ import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; + import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; + import org.apache.pig.EvalFunc; import org.apache.pig.ExecType; import org.apache.pig.PigException; @@ -41,10 +43,13 @@ import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.util.LogUtils; + import org.joda.time.DateTime; import org.joda.time.DateTimeZone; + import org.junit.Assert; import org.junit.Test; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +68,7 @@ public void testWriteTinyint() throws Exception { pigValueRangeTestOverflow("junitTypeTest3", "tinyint", "int", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, Integer.toString(300)); } + @Test public void testWriteSmallint() throws Exception { pigValueRangeTest("junitTypeTest1", "smallint", "int", null, Integer.toString(Short.MIN_VALUE), @@ -72,6 +78,7 @@ public void testWriteSmallint() throws Exception { pigValueRangeTestOverflow("junitTypeTest3", "smallint", "int", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, Integer.toString(Short.MAX_VALUE + 1)); } + @Test public void testWriteChar() throws Exception { pigValueRangeTest("junitTypeTest1", "char(5)", "chararray", null, "xxx", "xxx "); @@ -81,6 +88,7 @@ public void testWriteChar() throws Exception { pigValueRangeTestOverflow("junitTypeTest3", "char(5)", "chararray", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, "too_long2"); } + @Test public void testWriteVarchar() throws Exception { pigValueRangeTest("junitTypeTest1", "varchar(5)", "chararray", null, "xxx", "xxx"); @@ -90,6 +98,7 @@ public void testWriteVarchar() throws Exception { pigValueRangeTestOverflow("junitTypeTest3", "varchar(5)", "chararray", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, "too_long2"); } + @Test public void testWriteDecimalXY() throws Exception { pigValueRangeTest("junitTypeTest1", "decimal(5,2)", "bigdecimal", null, BigDecimal.valueOf(1.2).toString(), @@ -100,6 +109,7 @@ public void testWriteDecimalXY() throws Exception { pigValueRangeTestOverflow("junitTypeTest3", "decimal(5,2)", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, BigDecimal.valueOf(500.123).toString()); } + @Test public void testWriteDecimalX() throws Exception { //interestingly decimal(2) means decimal(2,0) @@ -110,6 +120,7 @@ public void testWriteDecimalX() throws Exception { pigValueRangeTestOverflow("junitTypeTest3", "decimal(2)", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, BigDecimal.valueOf(50.123).toString()); } + @Test public void testWriteDecimal() throws Exception { //decimal means decimal(10,0) @@ -120,9 +131,10 @@ public void testWriteDecimal() throws Exception { pigValueRangeTestOverflow("junitTypeTest3", "decimal", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, BigDecimal.valueOf(12345678900L).toString()); } + /** * because we want to ignore TZ which is included in toString() - * include time to make sure it's 0 + * include time to make sure it's 0 */ private static final String FORMAT_4_DATE = "yyyy-MM-dd HH:mm:ss"; @Test @@ -142,6 +154,7 @@ public void testWriteDate() throws Exception { pigValueRangeTestOverflow("junitTypeTest6", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, d.plusMinutes(1).toString(), FORMAT_4_DATE);//date out of range due to time!=0 } + @Test public void testWriteDate3() throws Exception { DateTime d = new DateTime(1991,10,11,23,10,DateTimeZone.forOffsetHours(-11)); @@ -154,6 +167,7 @@ public void testWriteDate3() throws Exception { pigValueRangeTestOverflow("junitTypeTest6", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, d.plusMinutes(1).toString(), FORMAT_4_DATE); } + @Test public void testWriteDate2() throws Exception { DateTime d = new DateTime(1991,11,12,0,0, DateTimeZone.forID("US/Eastern")); @@ -168,46 +182,48 @@ public void testWriteDate2() throws Exception { pigValueRangeTestOverflow("junitTypeTest3", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw, d.plusMinutes(1).toString(), FORMAT_4_DATE); } + /** - * Note that the value that comes back from Hive will have local TZ on it. Using local is + * Note that the value that comes back from Hive will have local TZ on it. Using local is * arbitrary but DateTime needs TZ (or will assume default) and Hive does not have TZ. * So if you start with Pig value in TZ=x and write to Hive, when you read it back the TZ may * be different. The millis value should match, of course. - * + * * @throws Exception */ @Test public void testWriteTimestamp() throws Exception { DateTime d = new DateTime(1991,10,11,14,23,30, 10);//uses default TZ - pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(), + pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(), d.toDateTime(DateTimeZone.getDefault()).toString()); d = d.plusHours(2); pigValueRangeTest("junitTypeTest2", "timestamp", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Null, d.toString(), d.toDateTime(DateTimeZone.getDefault()).toString()); d = d.toDateTime(DateTimeZone.UTC); - pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(), + pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(), d.toDateTime(DateTimeZone.getDefault()).toString()); d = new DateTime(1991,10,11,23,24,25, 26); - pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(), + pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(), d.toDateTime(DateTimeZone.getDefault()).toString()); d = d.toDateTime(DateTimeZone.UTC); - pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(), + pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(), d.toDateTime(DateTimeZone.getDefault()).toString()); } //End: tests that check values from Pig that are out of range for target column - private void pigValueRangeTestOverflow(String tblName, String hiveType, String pigType, HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, String format) throws Exception { pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, null, format); } + private void pigValueRangeTestOverflow(String tblName, String hiveType, String pigType, HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue) throws Exception { pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, null, null); } + private void pigValueRangeTest(String tblName, String hiveType, String pigType, - HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, + HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, String expectedValue) throws Exception { pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, expectedValue, null); } @@ -218,6 +234,7 @@ private void pigValueRangeTest(String tblName, String hiveType, String pigType, String getStorageFormat() { return "RCFILE"; } + /** * This is used to test how Pig values of various data types which are out of range for Hive target * column are handled. Currently the options are to raise an error or write NULL. @@ -236,7 +253,7 @@ String getStorageFormat() { * @param format date format to use for comparison of values since default DateTime.toString() * includes TZ which is meaningless for Hive DATE type */ - private void pigValueRangeTest(String tblName, String hiveType, String pigType, + private void pigValueRangeTest(String tblName, String hiveType, String pigType, HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, String expectedValue, String format) throws Exception { TestHCatLoader.dropTable(tblName, driver); @@ -309,6 +326,7 @@ private void pigValueRangeTest(String tblName, String hiveType, String pigType, Unfortunately Timestamp.toString() adjusts the value for local TZ and 't' is a String thus the timestamp in 't' doesn't match rawData*/ } + /** * Create a data file with datatypes added in 0.13. Read it with Pig and use * Pig + HCatStorer to write to a Hive table. Then read it using Pig and Hive @@ -365,6 +383,7 @@ public void testDateCharTypes() throws Exception { } Assert.assertEquals("Expected " + NUM_ROWS + " rows; got " + numRowsRead + " file=" + INPUT_FILE_NAME, NUM_ROWS, numRowsRead); } + static void dumpFile(String fileName) throws Exception { File file = new File(fileName); BufferedReader reader = new BufferedReader(new FileReader(file)); @@ -375,6 +394,7 @@ static void dumpFile(String fileName) throws Exception { } reader.close(); } + @Test public void testPartColsInData() throws IOException, CommandNeedRetryException { diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java index 76080f7..9679d3c 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java @@ -25,20 +25,26 @@ import java.util.HashMap; import java.util.Map; -import junit.framework.TestCase; - import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; + import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.Pair; + import org.apache.pig.ExecType; import org.apache.pig.PigServer; -public class TestHCatStorerMulti extends TestCase { +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestHCatStorerMulti { public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName( System.getProperty("user.dir") + "/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName() + "-" + System.currentTimeMillis()); @@ -77,8 +83,8 @@ private void createTable(String tablename, String schema) throws IOException, Co createTable(tablename, schema, null); } - @Override - protected void setUp() throws Exception { + @Before + public void setUp() throws Exception { if (driver == null) { HiveConf hiveConf = new HiveConf(this.getClass()); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); @@ -92,14 +98,13 @@ protected void setUp() throws Exception { cleanup(); } - @Override - protected void tearDown() throws Exception { + @After + public void tearDown() throws Exception { cleanup(); } + @Test public void testStoreBasicTable() throws Exception { - - createTable(BASIC_TABLE, "a int, b string"); populateBasicFile(); @@ -117,6 +122,7 @@ public void testStoreBasicTable() throws Exception { assertEquals(basicInputData.size(), unpartitionedTableValuesReadFromHiveDriver.size()); } + @Test public void testStorePartitionedTable() throws Exception { createTable(PARTITIONED_TABLE, "a int, b string", "bkt string"); @@ -139,9 +145,8 @@ public void testStorePartitionedTable() throws Exception { assertEquals(basicInputData.size(), partitionedTableValuesReadFromHiveDriver.size()); } + @Test public void testStoreTableMulti() throws Exception { - - createTable(BASIC_TABLE, "a int, b string"); createTable(PARTITIONED_TABLE, "a int, b string", "bkt string"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java index 7f0bca7..2cf14aa 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java @@ -25,10 +25,13 @@ import java.util.UUID; import org.apache.hadoop.hive.ql.CommandNeedRetryException; + import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; + import org.apache.pig.ExecType; import org.apache.pig.PigServer; + import org.junit.Assert; import org.junit.Test; diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java deleted file mode 100644 index a9b4521..0000000 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.apache.hive.hcatalog.pig; - -import org.apache.hadoop.hive.ql.CommandNeedRetryException; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -public class TestOrcHCatPigStorer extends TestHCatStorer { - @Override String getStorageFormat() { - return "ORC"; - } -} diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java index 1084092..65769b4 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java @@ -18,11 +18,15 @@ */ package org.apache.hive.hcatalog.pig; -public class TestOrcHCatStorer extends TestHCatStorerMulti { +import java.io.IOException; - @Override - protected String storageFormat() { - return "orc"; +import org.apache.hadoop.hive.ql.CommandNeedRetryException; + +import org.junit.Ignore; +import org.junit.Test; + +public class TestOrcHCatStorer extends TestHCatStorer { + @Override String getStorageFormat() { + return "ORC"; } } - diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java new file mode 100644 index 0000000..77c7979 --- /dev/null +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.pig; + +public class TestOrcHCatStorerMulti extends TestHCatStorerMulti { + + @Override + protected String storageFormat() { + return "orc"; + } +} + diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java index a8ce61a..ca952e7 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java @@ -20,14 +20,18 @@ package org.apache.hive.hcatalog.pig; import com.google.common.collect.Lists; + import junit.framework.Assert; + import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchema; + import org.apache.pig.ResourceSchema; import org.apache.pig.ResourceSchema.ResourceFieldSchema; import org.apache.pig.data.DataType; import org.apache.pig.impl.util.UDFContext; + import org.junit.Test; public class TestPigHCatUtil { -- 1.8.3.4 (Apple Git-47)