commit 05b1dd544d0b5fe1271551ae0624f849537185a8 Author: Sahil Takiar Date: Mon Jun 4 17:03:39 2018 -0500 HIVE-19788: Flaky test: HCatalog-PigServer tests diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java index 8471e5b620..a304e49ae8 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java @@ -109,14 +109,18 @@ protected void logAndRegister(PigServer server, String query, int lineNumber) th server.registerQuery(query, lineNumber); } + public static PigServer createPigServer(boolean stopOnFailure) throws ExecException { + return createPigServer(stopOnFailure, new Properties()); + } + /** * creates PigServer in LOCAL mode. * http://pig.apache.org/docs/r0.12.0/perf.html#error-handling * @param stopOnFailure equivalent of "-stop_on_failure" command line arg, setting to 'true' makes * debugging easier */ - public static PigServer createPigServer(boolean stopOnFailure) throws ExecException { - Properties p = new Properties(); + public static PigServer createPigServer(boolean stopOnFailure, Properties p) throws + ExecException { Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp")); String testId = "HCatBaseTest_" + System.currentTimeMillis(); diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java index 1df3c24550..332c240e52 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java @@ -44,8 +44,6 @@ import org.apache.hive.hcatalog.data.DefaultHCatRecord; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchema; -import org.apache.pig.ExecType; -import org.apache.pig.PigServer; import org.junit.Test; public class TestPassProperties { @@ -55,7 +53,6 @@ private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; private static IDriver driver; - private static PigServer server; private static String[] input; private static HiveConf hiveConf; @@ -81,7 +78,6 @@ public void Initialize() throws Exception { input[i] = i + "," + col1 + "," + col2; } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - server = new PigServer(ExecType.LOCAL); } @Test diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java index 14b22ed8f7..0d72102c1f 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatLoaderTest.java @@ -52,7 +52,6 @@ import org.apache.hive.hcatalog.data.Pair; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; -import org.apache.pig.ExecType; import org.apache.pig.PigRunner; import org.apache.pig.PigServer; import org.apache.pig.ResourceStatistics; @@ -584,7 +583,7 @@ public void testConvertBooleanToInt() throws Exception { Properties properties = new Properties(); properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER, "true"); properties.put("stop.on.failure", Boolean.TRUE.toString()); - PigServer server = new PigServer(ExecType.LOCAL, properties); + PigServer server = createPigServer(true, properties); server.registerQuery( "data = load 'test_convert_boolean_to_int' using org.apache.hive.hcatalog.pig.HCatLoader();"); Schema schema = server.dumpSchema("data"); @@ -689,7 +688,7 @@ private static void testReadDataPrimitiveTypes() throws Exception { // might be the last one to call HCatContext.INSTANCE.setConf(). Make sure setting is false. Properties properties = new Properties(); properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER, "false"); - PigServer server = new PigServer(ExecType.LOCAL, properties); + PigServer server = createPigServer(false, properties); server.registerQuery("X = load '" + ALL_PRIMITIVE_TYPES_TABLE + "' using " + HCatLoader.class.getName() + "();"); Iterator XIter = server.openIterator("X"); int numTuplesRead = 0; diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java index 30b0047788..97277b59b4 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/AbstractHCatStorerTest.java @@ -39,7 +39,6 @@ import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.EvalFunc; -import org.apache.pig.ExecType; import org.apache.pig.PigException; import org.apache.pig.PigServer; import org.apache.pig.data.DataByteArray; @@ -425,7 +424,7 @@ public void testPartColsInData() throws Exception { input[i] = i + "\t1"; } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('b=1');"); @@ -460,7 +459,7 @@ public void testMultiPartColsInData() throws Exception { "111239\tSatya\t01/01/2001\tM\tIN\tKL", "111240\tKavya\t01/01/2002\tF\tIN\tAP" }; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer pig = new PigServer(ExecType.LOCAL); + PigServer pig = createPigServer(false); pig.setBatchOn(); pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," @@ -511,7 +510,7 @@ public void testStoreInPartiitonedTbl() throws Exception { input[i] = i + ""; } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int);"); server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('b=1');"); @@ -541,7 +540,7 @@ public void testStoreInPartiitonedTbl() throws Exception { public void testNoAlias() throws Exception { AbstractHCatLoaderTest.dropTable("junit_parted", driver); AbstractHCatLoaderTest.createTable("junit_parted","a int, b string", "ds string", driver, storageFormat); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); boolean errCaught = false; try { server.setBatchOn(); @@ -601,7 +600,7 @@ public void testStoreMultiTables() throws Exception { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("B = filter A by a < 2;"); @@ -648,7 +647,7 @@ public void testStoreWithNoSchema() throws Exception { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into 'default.junit_unparted' using " @@ -684,7 +683,7 @@ public void testStoreWithNoCtorArgs() throws Exception { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into 'junit_unparted' using " + HCatStorer.class.getName() @@ -720,7 +719,7 @@ public void testEmptyStore() throws Exception { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("B = filter A by a > 100;"); @@ -751,7 +750,7 @@ public void testBagNStruct() throws Exception { HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server .registerQuery("A = load '" @@ -797,7 +796,7 @@ public void testStoreFuncAllSimpleTypes() throws Exception { + "\tbinary-data"; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray);"); @@ -854,7 +853,7 @@ public void testStoreFuncSimple() throws Exception { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into 'default.junit_unparted' using " @@ -888,7 +887,7 @@ public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws Excep "111239\tSatya\t01/01/2001\tM\tIN\tKL", "111240\tKavya\t01/01/2002\tF\tIN\tAP" }; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer pig = new PigServer(ExecType.LOCAL); + PigServer pig = createPigServer(false); pig.setBatchOn(); pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," @@ -921,7 +920,7 @@ public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws Exception "111239\tSatya\t01/01/2001\tM\tIN\tKL", "111240\tKavya\t01/01/2002\tF\tIN\tAP" }; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer pig = new PigServer(ExecType.LOCAL); + PigServer pig = createPigServer(false); pig.setBatchOn(); pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," @@ -952,7 +951,7 @@ public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws Exce String[] inputData = {}; HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer pig = new PigServer(ExecType.LOCAL); + PigServer pig = createPigServer(false); pig.setBatchOn(); pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," @@ -980,7 +979,7 @@ public void testPartitionPublish() throws Exception { input[i] = i + "\tmath"; } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, c:chararray);"); server.registerQuery("B = filter A by " + FailEvalFunc.class.getName() + "($0);"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java index e0f1c89542..19246e6560 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java @@ -47,12 +47,12 @@ import org.apache.hive.hcatalog.common.HCatContext; import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.Tuple; @@ -125,7 +125,7 @@ private void driverRun(String cmd) throws Exception { } private void pigDump(String tableName) throws IOException { - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); System.err.println("==="); System.err.println(tableName+":"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java index eedd722580..8f06d39d39 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.StorageFormats; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.pig.ExecType; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.executionengine.ExecJob; @@ -224,7 +224,7 @@ private void verifyWriteRead(String tablename, String pigSchema, String tableSch MockLoader.setData(tablename + "Input", data); try { createTable(tablename, tableSchema); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + tablename + "Input' using org.apache.hive.hcatalog.pig.MockLoader() AS (" + pigSchema + ");"); Schema dumpedASchema = server.dumpSchema("A"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java index 72ed8dfcba..872d7afd23 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java @@ -65,8 +65,8 @@ import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.data.Pair; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.Tuple; import org.junit.After; @@ -206,7 +206,7 @@ public void setup() throws Exception { } } HcatTestUtils.createTestDataFile(BASIC_FILE_NAME, input); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); int i = 0; server.registerQuery("A = load '" + BASIC_FILE_NAME + "' as (a:int, b:chararray);", ++i); @@ -300,7 +300,7 @@ private String getKeyProviderURI() { @Test public void testReadDataFromEncryptedHiveTableByPig() throws IOException { assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.registerQuery("X = load '" + ENCRYPTED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();"); Iterator XIter = server.openIterator("X"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java index ae40f0bf47..c7f2606132 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java @@ -22,7 +22,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecJob; import org.apache.pig.data.DataType; @@ -127,7 +126,7 @@ public void testSmallTinyInt() throws Exception { Assert.assertEquals(0, driver.run("load data local inpath '" + dataDir.getPath().replaceAll("\\\\", "/") + "' into table " + readTblName).getResponseCode()); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.registerQuery( "data = load '" + readTblName + "' using org.apache.hive.hcatalog.pig.HCatLoader();"); @@ -186,7 +185,7 @@ private void smallTinyIntBoundsCheckHelper(String data, ExecJob.JOB_STATUS expec Assert.assertEquals(0, driver.run("create table test_tbl" + " (my_small_int smallint, my_tiny_int tinyint) stored as rcfile").getResponseCode()); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("data = load '" + data + "' using PigStorage('\t') as (my_small_int:int, my_tiny_int:int);"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java index d6b3ebc935..65ad6ecbb2 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java @@ -37,8 +37,7 @@ import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.Pair; - -import org.apache.pig.ExecType; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.PigServer; import org.junit.After; @@ -123,7 +122,7 @@ public void testStoreBasicTable() throws Exception { populateBasicFile(); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();"); @@ -143,7 +142,7 @@ public void testStorePartitionedTable() throws Exception { populateBasicFile(); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); @@ -168,7 +167,7 @@ public void testStoreTableMulti() throws Exception { populateBasicFile(); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);"); server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java index 0ffab4a339..3edd020017 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java @@ -27,7 +27,6 @@ import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.junit.Assert; @@ -66,7 +65,7 @@ public void testStoreExternalTableWithExternalDir() throws Exception { } } HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = HCatBaseTest.createPigServer(false); server.setBatchOn(); logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);"); logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName() diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java index 4778e1f794..4219757d7e 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java @@ -53,7 +53,7 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.hcatalog.common.HCatUtil; -import org.apache.pig.ExecType; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.PigServer; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; @@ -195,7 +195,7 @@ public void testPigHBaseSchema() throws Exception { assertTrue(doesTableExist); - PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + PigServer server = HCatBaseTest.createPigServer(false, hcatConf.getAllProperties()); server.registerQuery("A = load '"+databaseName+"."+tableName+"' using org.apache.hive.hcatalog.pig.HCatLoader();"); Schema dumpedASchema = server.dumpSchema("A"); @@ -281,7 +281,7 @@ public void testPigFilterProjection() throws Exception { int index=1; - PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + PigServer server = HCatBaseTest.createPigServer(false, hcatConf.getAllProperties()); server.registerQuery("A = load '"+databaseName+"."+tableName+"' using org.apache.hive.hcatalog.pig.HCatLoader();"); server.registerQuery("B = filter A by key < 5;"); server.registerQuery("C = foreach B generate key,testqualifier2;"); @@ -351,7 +351,7 @@ public void testPigPopulation() throws Exception { createTestDataFile(POPTXT_FILE_NAME); - PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + PigServer server = HCatBaseTest.createPigServer(false, hcatConf.getAllProperties()); server.registerQuery("A = load '"+POPTXT_FILE_NAME+"' using PigStorage() as (key:int, testqualifier1:float, testqualifier2:chararray);"); server.registerQuery("B = filter A by (key > 2) AND (key < 8) ;"); server.registerQuery("store B into '"+databaseName.toLowerCase()+"."+tableName.toLowerCase()+"' using org.apache.hive.hcatalog.pig.HCatStorer();"); diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java index bb77005d51..dac6218a67 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveCompatibility.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.Tuple; import org.junit.BeforeClass; @@ -60,7 +59,7 @@ public void testUnpartedReadWrite() throws Exception { Table table = client.getTable("default", "junit_unparted_noisd"); Assert.assertTrue(table.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS)); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); logAndRegister(server, "A = load '" + INPUT_FILE_NAME + "' as (a:int);"); logAndRegister(server, "store A into 'default.junit_unparted_noisd' using org.apache.hive.hcatalog.pig.HCatStorer();"); logAndRegister(server, "B = load 'default.junit_unparted_noisd' using org.apache.hive.hcatalog.pig.HCatLoader();"); @@ -96,7 +95,7 @@ public void testPartedRead() throws Exception { Table table = client.getTable("default", "junit_parted_noisd"); Assert.assertTrue(table.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS)); - PigServer server = new PigServer(ExecType.LOCAL); + PigServer server = createPigServer(false); logAndRegister(server, "A = load '" + INPUT_FILE_NAME + "' as (a:int);"); logAndRegister(server, "store A into 'default.junit_parted_noisd' using org.apache.hive.hcatalog.pig.HCatStorer('b=42');"); logAndRegister(server, "B = load 'default.junit_parted_noisd' using org.apache.hive.hcatalog.pig.HCatLoader();"); diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java index a024c835fd..1c14865cdd 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatHiveThriftCompatibility.java @@ -25,7 +25,6 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.SequenceFile; -import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; @@ -93,7 +92,7 @@ public void testDynamicCols() throws Exception { driver.run("alter table test_thrift add partition (year = '2012') location '" + intStringSeq.getParent() + "'").getResponseCode()); - PigServer pigServer = new PigServer(ExecType.LOCAL); + PigServer pigServer = createPigServer(false); pigServer.registerQuery("A = load 'test_thrift' using org.apache.hive.hcatalog.pig.HCatLoader();"); Schema expectedSchema = new Schema(); diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java index 4ebec1bf1f..ab25ffed8e 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java @@ -96,7 +96,7 @@ public void setup() throws Exception { input[i] = i + "," + col1 + "," + col2; } HcatTestUtils.createTestDataFile(inputFileName, input); - server = new PigServer(ExecType.LOCAL); + server = HCatBaseTest.createPigServer(false); } @After public void teardown() throws IOException {