diff --git a/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java b/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java index 9b8227e..47a1d5b 100644 --- a/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java +++ b/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java @@ -169,7 +169,12 @@ public void abortJob(JobContext jobContext, State state) throws IOException { src = new Path(jobInfo.getLocation()); } FileSystem fs = src.getFileSystem(jobContext.getConfiguration()); - LOG.info("Job failed. Cleaning up temporary directory [{}].", src); + // Note fs.delete will fail on Windows. The reason is in OutputCommitter, + // Hadoop is still writing to _logs/history. On Linux, OS don't care file is still + // open and remove the directory anyway, but on Windows, OS refuse to remove a + // directory containing open files. So on Windows, we will leave output directory + // behind when job fail. User needs to remove the output directory manually + LOG.info("Job failed. Try cleaning up temporary directory [{}].", src); fs.delete(src, true); } finally { cancelDelegationTokens(jobContext); diff --git a/hcatalog/core/src/test/java/org/apache/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hcatalog/cli/TestPermsGrp.java index 1fdb8b1..a750d27 100644 --- a/hcatalog/core/src/test/java/org/apache/hcatalog/cli/TestPermsGrp.java +++ b/hcatalog/core/src/test/java/org/apache/hcatalog/cli/TestPermsGrp.java @@ -90,6 +90,7 @@ protected void setUp() throws Exception { hcatConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hcatConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); hcatConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + hcatConf.set(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.varname, "60"); clientWH = new Warehouse(hcatConf); msc = new HiveMetaStoreClient(hcatConf, null); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); diff --git a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java index 03d5f21..a069ab5 100644 --- a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java +++ b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java @@ -40,8 +40,8 @@ */ public class HCatBaseTest { protected static final Logger LOG = LoggerFactory.getLogger(HCatBaseTest.class); - protected static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + HCatBaseTest.class.getCanonicalName(); + protected static final String TEST_DATA_DIR = + "/tmp/build/test/data/" + HCatBaseTest.class.getCanonicalName(); protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; protected HiveConf hiveConf = null; diff --git a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java index b870129..320b0e9 100644 --- a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -24,12 +24,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -51,6 +53,7 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.util.Shell; import org.apache.hcatalog.NoExitSecurityManager; import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; import org.apache.hcatalog.data.DefaultHCatRecord; @@ -74,13 +77,17 @@ @BeforeClass public static void setup() throws Exception { + String testDir = System.getProperty("test.data.dir", "./"); + testDir = testDir + "/test_hcat_partitionpublish_" + Math.abs(new Random().nextLong()) + "/"; + File workDir = new File(new File(testDir).getCanonicalPath()); + FileUtil.fullyDelete(workDir); + workDir.mkdirs(); Configuration conf = new Configuration(true); conf.set("yarn.scheduler.capacity.root.queues", "default"); conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); fs = FileSystem.get(conf); - System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory() - .toString(), "/logs").getAbsolutePath()); + System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, @@ -93,6 +100,7 @@ public static void setup() throws Exception { MetaStoreUtils.startMetaStore(msPort, ShimLoader .getHadoopThriftAuthBridge()); + Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); @@ -144,10 +152,12 @@ public void testPartitionPublish() throws Exception { Assert.assertEquals(0, ptns.size()); Table table = msc.getTable(dbName, tableName); Assert.assertTrue(table != null); - // Also make sure that the directory has been deleted in the table - // location. - Assert.assertFalse(fs.exists(new Path(table.getSd().getLocation() - + "/part1=p1value1/part0=p0value1"))); + // In Windows, we cannot remove the output directory when job fail. See + // FileOutputCommitterContainer.abortJob + if (!Shell.WINDOWS) { + Assert.assertFalse(fs.exists(new Path(table.getSd().getLocation() + + "/part1=p1value1/part0=p0value1"))); + } } void runMRCreateFail( diff --git a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java index 1ccae5b..999069b 100644 --- a/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java +++ b/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java @@ -54,8 +54,8 @@ import org.junit.Test; public class TestSequenceFileReadWrite extends TestCase { - private static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + TestSequenceFileReadWrite.class.getCanonicalName(); + private static final String TEST_DATA_DIR = + "/tmp/build/test/data/" + TestSequenceFileReadWrite.class.getCanonicalName(); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java index 47339d9..2644459 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java @@ -50,8 +50,8 @@ import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; public class TestHCatLoader extends TestCase { - private static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + TestHCatLoader.class.getCanonicalName(); + private static final String TEST_DATA_DIR = + "/tmp/build/test/data/" + TestHCatLoader.class.getCanonicalName(); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data"; private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data"; @@ -420,7 +420,7 @@ public void testConvertBooleanToInt() throws Exception { assertEquals(0, driver.run("create external table " + tbl + " (a string, b boolean) row format delimited fields terminated by '\t'" + " stored as textfile location 'file://" + - inputDataDir.getAbsolutePath() + "'").getResponseCode()); + inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode()); Properties properties = new Properties(); properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER, "true"); diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java index 75333b2..c65ad49 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java @@ -68,7 +68,7 @@ public void testSmallTinyInt() throws Exception { " (my_small_int smallint, my_tiny_int tinyint)" + " row format delimited fields terminated by '\t' stored as textfile").getResponseCode()); Assert.assertEquals(0, driver.run("load data local inpath '" + - dataDir.getAbsolutePath() + "' into table " + readTblName).getResponseCode()); + dataDir.getPath().replaceAll("\\\\", "/") + "' into table " + readTblName).getResponseCode()); PigServer server = new PigServer(ExecType.LOCAL); server.registerQuery( @@ -103,7 +103,7 @@ public void testSmallTinyInt() throws Exception { String.format("%d\t%d", Short.MIN_VALUE, Byte.MIN_VALUE), String.format("%d\t%d", Short.MAX_VALUE, Byte.MAX_VALUE) }); - smallTinyIntBoundsCheckHelper(writeDataFile.getAbsolutePath(), ExecJob.JOB_STATUS.COMPLETED); + smallTinyIntBoundsCheckHelper(writeDataFile.getPath().replaceAll("\\\\", "/"), ExecJob.JOB_STATUS.COMPLETED); // Values outside the column type bounds will fail at runtime. HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooSmall.tsv", new String[]{ diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java index b1ac22c..2660fc4 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java @@ -38,8 +38,8 @@ import org.apache.pig.PigServer; public class TestHCatStorerMulti extends TestCase { - private static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName(); + private static final String TEST_DATA_DIR = + "/tmp/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName(); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java index 33200f3..a6680d2 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java @@ -24,7 +24,6 @@ import java.util.Iterator; import java.util.UUID; -import org.apache.commons.lang.SystemUtils; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hcatalog.HcatTestUtils; import org.apache.hcatalog.mapreduce.HCatBaseTest; @@ -47,7 +46,7 @@ @Test public void testStoreExternalTableWithExternalDir() throws IOException, CommandNeedRetryException{ - File tmpExternalDir = new File(SystemUtils.getJavaIoTmpDir(), UUID.randomUUID().toString()); + File tmpExternalDir = new File(TEST_DATA_DIR, UUID.randomUUID().toString()); tmpExternalDir.deleteOnExit(); String part_val = "100"; @@ -70,11 +69,11 @@ public void testStoreExternalTableWithExternalDir() throws IOException, CommandN server.setBatchOn(); logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);"); logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName() - + "('c=" + part_val + "','" + tmpExternalDir.getAbsolutePath() + "');"); + + "('c=" + part_val + "','" + tmpExternalDir.getPath().replaceAll("\\\\", "/") + "');"); server.executeBatch(); Assert.assertTrue(tmpExternalDir.exists()); - Assert.assertTrue(new File(tmpExternalDir.getAbsoluteFile() + "/" + "part-m-00000").exists()); + Assert.assertTrue(new File(tmpExternalDir.getPath().replaceAll("\\\\", "/") + "/" + "part-m-00000").exists()); driver.run("select * from junit_external"); ArrayList res = new ArrayList(); diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java index e52584d..58fd435 100644 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java +++ b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java @@ -307,6 +307,12 @@ ZooKeeper getSession() throws IOException { if (zkSession == null || zkSession.getState() == States.CLOSED) { zkSession = new ZooKeeper(this.connectString, this.DEFAULT_SESSION_TIMEOUT, new ZKWatcher()); + while (zkSession.getState() == States.CONNECTING) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } } } } diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java index cd14143..8557cda 100644 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java +++ b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java @@ -247,7 +247,8 @@ private void setupHBaseCluster() { final int numRegionServers = 1; try { - hbaseDir = new File(workDir, "hbase").getAbsolutePath(); + hbaseDir = new File(workDir, "hbase").toString(); + hbaseDir = hbaseDir.replaceAll("\\\\", "/"); hbaseRoot = "file://" + hbaseDir; if (hbaseConf == null) diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java index d48160c..aefe450 100644 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java +++ b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java @@ -43,7 +43,7 @@ */ public abstract class SkeletonHBaseTest { - protected static String TEST_DIR = System.getProperty("test.data.dir", "./"); + protected static String TEST_DIR = "/tmp/build/test/data/"; protected final static String DEFAULT_CONTEXT_HANDLE = "default"; @@ -172,11 +172,7 @@ public FileSystem getFileSystem() { protected int usageCount = 0; public Context(String handle) { - try { - testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getCanonicalPath(); - } catch (IOException e) { - throw new IllegalStateException("Failed to generate testDir", e); - } + testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath(); System.out.println("Cluster work directory: " + testDir); } diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java index 9d29e2b..9e02d83 100644 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java +++ b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java @@ -181,7 +181,7 @@ public void TestHBaseTableReadMR() throws Exception { String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = new Path(getTestDir(), "hbasedb").toString(); String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'"; diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java index f689cc9..579922d 100644 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java +++ b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java @@ -76,7 +76,7 @@ public void TestSnapshotConversion() throws Exception { String tableName = newTableName("mytableOne"); String databaseName = newTableName("mydatabase"); String fullyQualTableName = databaseName + "." + tableName; - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = new Path(getTestDir(), "hbasedb").toString(); String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'"; String tableQuery = "CREATE TABLE " + fullyQualTableName diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java index 4e91efc..fadbf5e 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java @@ -121,8 +121,12 @@ public void testBasicDDLCommands() throws Exception { assertTrue(testDb.getProperties().size() == 0); String warehouseDir = System .getProperty(ConfVars.METASTOREWAREHOUSE.varname, "/user/hive/warehouse"); + String expectedDir = warehouseDir.replaceAll("\\\\", "/"); + if (!expectedDir.startsWith("/")) { + expectedDir = "/" + expectedDir; + } assertTrue(testDb.getLocation().equals( - "file:" + warehouseDir + "/" + db + ".db")); + "file:" + expectedDir + "/" + db + ".db")); ArrayList cols = new ArrayList(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); cols.add(new HCatFieldSchema("value", Type.STRING, "value comment")); @@ -157,7 +161,7 @@ public void testBasicDDLCommands() throws Exception { assertTrue(table2.getOutputFileFormat().equalsIgnoreCase( IgnoreKeyTextOutputFormat.class.getName())); assertTrue(table2.getLocation().equalsIgnoreCase( - "file:" + warehouseDir + "/" + db + ".db/" + tableTwo)); + "file:" + expectedDir + "/" + db + ".db/" + tableTwo)); client.close(); }