diff --git a/hcatalog/hcatalog-pig-adapter/pom.xml b/hcatalog/hcatalog-pig-adapter/pom.xml
index 4d2ca51..7589efb 100644
--- a/hcatalog/hcatalog-pig-adapter/pom.xml
+++ b/hcatalog/hcatalog-pig-adapter/pom.xml
@@ -53,6 +53,13 @@
tests
test
+
+ org.apache.hive
+ hive-exec
+ ${project.version}
+ test-jar
+ test
+
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java
index c87b95a..4b52faf 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java
@@ -36,6 +36,7 @@
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.data.Tuple;
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
index a4b55c8..fe4c9f7 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
@@ -23,9 +23,8 @@
import java.util.HashMap;
import java.util.Iterator;
-import junit.framework.TestCase;
-
import org.apache.commons.io.FileUtils;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hive.cli.CliSessionState;
@@ -42,6 +41,7 @@
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
+
import org.apache.hive.hcatalog.HcatTestUtils;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatContext;
@@ -51,12 +51,16 @@
import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat;
import org.apache.hive.hcatalog.mapreduce.OutputJobInfo;
import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil;
+
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
-public class TestE2EScenarios extends TestCase {
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+public class TestE2EScenarios {
private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator
+ TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis();
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
@@ -69,9 +73,8 @@ protected String storageFormat() {
return "orc";
}
- @Override
- protected void setUp() throws Exception {
-
+ @Before
+ public void setUp() throws Exception {
File f = new File(TEST_WAREHOUSE_DIR);
if (f.exists()) {
FileUtil.fullyDelete(f);
@@ -90,8 +93,8 @@ protected void setUp() throws Exception {
}
- @Override
- protected void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
try {
dropTable("inpy");
dropTable("rc5318");
@@ -146,16 +149,13 @@ private void pigDump(String tableName) throws IOException {
System.err.println("===");
}
-
private void copyTable(String in, String out) throws IOException, InterruptedException {
Job ijob = new Job();
Job ojob = new Job();
HCatInputFormat inpy = new HCatInputFormat();
inpy.setInput(ijob , null, in);
HCatOutputFormat oupy = new HCatOutputFormat();
- oupy.setOutput(ojob,
- OutputJobInfo.create(null, out, new HashMap()
- ));
+ oupy.setOutput(ojob, OutputJobInfo.create(null, out, new HashMap()));
// Test HCatContext
@@ -207,6 +207,7 @@ private TaskAttemptContext createTaskAttemptContext(Configuration tconf) {
}
+ @Test
public void testReadOrcAndRCFromPig() throws Exception {
String tableSchema = "ti tinyint, si smallint,i int, bi bigint, f float, d double, b boolean";
@@ -224,15 +225,14 @@ public void testReadOrcAndRCFromPig() throws Exception {
driverRun("LOAD DATA LOCAL INPATH '"+TEXTFILE_LOCN+"' OVERWRITE INTO TABLE inpy");
// write it out from hive to an rcfile table, and to an orc table
-// driverRun("insert overwrite table rc5318 select * from inpy");
+ //driverRun("insert overwrite table rc5318 select * from inpy");
copyTable("inpy","rc5318");
-// driverRun("insert overwrite table orc5318 select * from inpy");
+ //driverRun("insert overwrite table orc5318 select * from inpy");
copyTable("inpy","orc5318");
pigDump("inpy");
pigDump("rc5318");
pigDump("orc5318");
-
}
}
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
index 82fc8a9..0acac19 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
@@ -18,6 +18,8 @@
*/
package org.apache.hive.hcatalog.pig;
+import com.google.common.collect.ImmutableSet;
+
import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
@@ -32,8 +34,10 @@
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import org.apache.commons.io.FileUtils;
+
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -41,15 +45,19 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.io.IOConstants;
+import org.apache.hadoop.hive.ql.io.StorageFormats;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.mapreduce.Job;
+
import org.apache.hive.hcatalog.HcatTestUtils;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.data.Pair;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
+
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.ResourceStatistics;
@@ -60,19 +68,26 @@
import org.apache.pig.PigRunner;
import org.apache.pig.tools.pigstats.OutputStats;
import org.apache.pig.tools.pigstats.PigStats;
+
import org.joda.time.DateTime;
+
import org.junit.After;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assert.*;
+@RunWith(Parameterized.class)
public class TestHCatLoader {
private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoader.class);
private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
- File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis());
+ File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis());
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data";
@@ -85,22 +100,33 @@
private Driver driver;
private Map> basicInputData;
- protected String storageFormat() {
- return "RCFILE tblproperties('hcat.isd'='org.apache.hive.hcatalog.rcfile.RCFileInputDriver'," +
- "'hcat.osd'='org.apache.hive.hcatalog.rcfile.RCFileOutputDriver')";
+ private static final Set DISABLED_STORAGE_FORMATS = ImmutableSet.of(
+ IOConstants.PARQUETFILE);
+
+ private String storageFormat;
+
+ @Parameterized.Parameters
+ public static Collection