diff --git hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java index 2f74cf1..18e3556 100644 --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java @@ -21,6 +21,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; @@ -645,4 +646,10 @@ public static boolean isHadoop23() { return true; return false; } + /** + * Used by various tests to make sure the path is safe for Windows + */ + public static String makePathASafeFileName(String filePath) { + return new File(filePath).getPath().replaceAll("\\\\", "/"); + } } diff --git hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java index 1f1f2af..6c72275 100644 --- hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java +++ hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java @@ -41,8 +41,7 @@ */ public class HCatBaseTest { protected static final Logger LOG = LoggerFactory.getLogger(HCatBaseTest.class); - protected static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + HCatBaseTest.class.getCanonicalName(); + protected static final String TEST_DATA_DIR = org.apache.hive.hcatalog.mapreduce.HCatBaseTest.TEST_DATA_DIR; protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; protected HiveConf hiveConf = null; diff --git hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java index 957ced6..0a76725 100644 --- hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -79,12 +79,12 @@ @BeforeClass public static void setup() throws Exception { + File workDir = org.apache.hive.hcatalog.mapreduce.TestHCatPartitionPublish.handleWorkDir(); conf.set("yarn.scheduler.capacity.root.queues", "default"); conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); fs = FileSystem.get(conf); - System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory() - .toString(), "/logs").getAbsolutePath()); + System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, @@ -99,6 +99,7 @@ public static void setup() throws Exception { MetaStoreUtils.startMetaStore(msPort, ShimLoader .getHadoopThriftAuthBridge()); + Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); diff --git hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java index f6ddb88..f35bdef 100644 --- hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java +++ hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java @@ -67,7 +67,7 @@ private static final Logger LOG = LoggerFactory.getLogger(TestMultiOutputFormat.class); private static File workDir; - private static Configuration mrConf = null; + private static JobConf mrConf = null; private static FileSystem fs = null; private static MiniMRCluster mrCluster = null; @@ -82,9 +82,12 @@ public static void setup() throws IOException { System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 - mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, - new JobConf(conf)); - mrConf = mrCluster.createJobConf(); + mrConf = new JobConf(conf); + mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, mrConf); + +// mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, +// new JobConf(conf)); +// mrConf = mrCluster.createJobConf(); } private static void createWorkDir() throws IOException { diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java index c42587e..8c7c838 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java @@ -97,4 +97,10 @@ public static void createTestDataFile(String filename, String[] lines) throws IO } } + /** + * Used by various tests to make sure the path is safe for Windows + */ + public static String makePathASafeFileName(String filePath) { + return new File(filePath).getPath().replaceAll("\\\\", "/"); + } } diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java index 461bd99..0fb845f 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.pig.PigServer; import org.junit.Assert; import org.junit.Before; @@ -40,8 +41,8 @@ */ public class HCatBaseTest { protected static final Logger LOG = LoggerFactory.getLogger(HCatBaseTest.class); - protected static final String TEST_DATA_DIR = - "/tmp/build/test/data/" + HCatBaseTest.class.getCanonicalName(); + public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("user.dir") + + "/build/test/data/" + HCatBaseTest.class.getCanonicalName() + "-" + System.currentTimeMillis()); protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; protected HiveConf hiveConf = null; diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index f5a6138..03e3b6c 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -76,13 +76,17 @@ private static SecurityManager securityManager; private static Configuration conf = new Configuration(true); - @BeforeClass - public static void setup() throws Exception { - String testDir = System.getProperty("test.tmp.dir", "./"); + public static File handleWorkDir() throws IOException { + String testDir = System.getProperty("test.data.dir", "./"); testDir = testDir + "/test_hcat_partitionpublish_" + Math.abs(new Random().nextLong()) + "/"; File workDir = new File(new File(testDir).getCanonicalPath()); FileUtil.fullyDelete(workDir); workDir.mkdirs(); + return workDir; + } + @BeforeClass + public static void setup() throws Exception { + File workDir = handleWorkDir(); conf.set("yarn.scheduler.capacity.root.queues", "default"); conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java index 69158f1..7d1a533 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java @@ -45,6 +45,7 @@ import org.apache.hcatalog.HcatTestUtils; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.data.Pair; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.ResourceStatistics; @@ -60,8 +61,9 @@ * @deprecated Use/modify {@link org.apache.hive.hcatalog.pig.TestHCatLoader} instead */ public class TestHCatLoader { - private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator - + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis(); + private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName( + System.getProperty("java.io.tmpdir") + File.separator + TestHCatLoader.class.getCanonicalName() + "-" + + System.currentTimeMillis()); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data"; private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data"; @@ -409,8 +411,8 @@ public void testConvertBooleanToInt() throws Exception { assertEquals(0, driver.run("drop table if exists " + tbl).getResponseCode()); assertEquals(0, driver.run("create external table " + tbl + " (a string, b boolean) row format delimited fields terminated by '\t'" + - " stored as textfile location 'file://" + - inputDataDir.getAbsolutePath() + "'").getResponseCode()); + " stored as textfile location 'file:///" + + inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode()); Properties properties = new Properties(); properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER, "true"); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java index d2f8956..22faf75 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java @@ -69,7 +69,7 @@ public void testSmallTinyInt() throws Exception { " (my_small_int smallint, my_tiny_int tinyint)" + " row format delimited fields terminated by '\t' stored as textfile").getResponseCode()); Assert.assertEquals(0, driver.run("load data local inpath '" + - dataDir.getAbsolutePath() + "' into table " + readTblName).getResponseCode()); + dataDir.getPath().replaceAll("\\\\", "/") + "' into table " + readTblName).getResponseCode()); PigServer server = new PigServer(ExecType.LOCAL); server.registerQuery( @@ -104,7 +104,7 @@ public void testSmallTinyInt() throws Exception { String.format("%d\t%d", Short.MIN_VALUE, Byte.MIN_VALUE), String.format("%d\t%d", Short.MAX_VALUE, Byte.MAX_VALUE) }); - smallTinyIntBoundsCheckHelper(writeDataFile.getAbsolutePath(), ExecJob.JOB_STATUS.COMPLETED); + smallTinyIntBoundsCheckHelper(writeDataFile.getPath().replaceAll("\\\\", "/"), ExecJob.JOB_STATUS.COMPLETED); // Values outside the column type bounds will fail at runtime. HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooSmall.tsv", new String[]{ diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java index 8a4901b..cfa7042 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java @@ -41,8 +41,7 @@ * @deprecated Use/modify {@link org.apache.hive.hcatalog.pig.TestHCatStorerMulti} instead */ public class TestHCatStorerMulti extends TestCase { - private static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName(); + private static final String TEST_DATA_DIR = org.apache.hive.hcatalog.pig.TestHCatStorerMulti.TEST_DATA_DIR; private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java index 976fa3d..f73f6c0 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java @@ -48,7 +48,7 @@ @Test public void testStoreExternalTableWithExternalDir() throws IOException, CommandNeedRetryException{ - File tmpExternalDir = new File(SystemUtils.getJavaIoTmpDir(), UUID.randomUUID().toString()); + File tmpExternalDir = new File(TEST_DATA_DIR, UUID.randomUUID().toString()); tmpExternalDir.deleteOnExit(); String part_val = "100"; @@ -71,11 +71,11 @@ public void testStoreExternalTableWithExternalDir() throws IOException, CommandN server.setBatchOn(); logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);"); logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName() - + "('c=" + part_val + "','" + tmpExternalDir.getAbsolutePath() + "');"); + + "('c=" + part_val + "','" + tmpExternalDir.getPath().replaceAll("\\\\", "/") + "');"); server.executeBatch(); Assert.assertTrue(tmpExternalDir.exists()); - Assert.assertTrue(new File(tmpExternalDir.getAbsoluteFile() + "/" + "part-m-00000").exists()); + Assert.assertTrue(new File(tmpExternalDir.getPath().replaceAll("\\\\", "/") + "/" + "part-m-00000").exists()); driver.run("select * from junit_external"); ArrayList res = new ArrayList(); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java index 268d965..209c6c5 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapreduce.Job; import org.apache.hive.hcatalog.HcatTestUtils; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.data.Pair; import org.apache.pig.ExecType; @@ -57,8 +58,8 @@ import org.junit.Test; public class TestHCatLoader { - private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator - + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis(); + private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") + + File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis()); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data"; private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data"; @@ -420,7 +421,7 @@ public void testConvertBooleanToInt() throws Exception { assertEquals(0, driver.run("drop table if exists " + tbl).getResponseCode()); assertEquals(0, driver.run("create external table " + tbl + " (a string, b boolean) row format delimited fields terminated by '\t'" + - " stored as textfile location 'file://" + + " stored as textfile location 'file:///" + inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode()); Properties properties = new Properties(); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java index c0f3c2f..76080f7 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java @@ -33,13 +33,15 @@ import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.Pair; import org.apache.pig.ExecType; import org.apache.pig.PigServer; public class TestHCatStorerMulti extends TestCase { - private static final String TEST_DATA_DIR = - "/tmp/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName(); + public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName( + System.getProperty("user.dir") + "/build/test/data/" + + TestHCatStorerMulti.class.getCanonicalName() + "-" + System.currentTimeMillis()); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java index a51020b..a59cc62 100644 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java @@ -165,12 +165,8 @@ public FileSystem getFileSystem() { protected int usageCount = 0; public Context(String handle) { - try { - testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getCanonicalPath(); - System.out.println("Cluster work directory: " + testDir); - } catch (IOException e) { - throw new IllegalStateException("Failed to generate testDir", e); - } + testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath(); + System.out.println("Cluster work directory: " + testDir); } public void start() { diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java index 92d923a..57175e5 100644 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java @@ -64,7 +64,7 @@ import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.common.HCatException; -import org.apache.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hcatalog.data.HCatRecord; import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; @@ -188,7 +188,7 @@ public void TestHBaseTableReadMR() throws Exception { String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb"); String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'"; diff --git hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java index 5bbf17d..4e1384a 100644 --- hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java +++ hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java @@ -172,12 +172,8 @@ public FileSystem getFileSystem() { protected int usageCount = 0; public Context(String handle) { - try { - testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getCanonicalPath(); - System.out.println("Cluster work directory: " + testDir); - } catch (IOException e) { - throw new IllegalStateException("Failed to generate testDir", e); - } + testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath(); + System.out.println("Cluster work directory: " + testDir); } public void start() { diff --git hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java index a640b1c..1f416a5 100644 --- hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java +++ hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java @@ -128,7 +128,8 @@ public void testBasicDDLCommands() throws Exception { assertTrue(testDb.getProperties().size() == 0); String warehouseDir = System .getProperty("test.warehouse.dir", "/user/hive/warehouse"); - String expectedDir = warehouseDir.replaceAll("\\\\", "/").replaceFirst("pfile:///", "pfile:/"); + String expectedDir = org.apache.hive.hcatalog.api.TestHCatClient.fixPath(warehouseDir). + replaceFirst("pfile:///", "pfile:/"); assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation()); ArrayList cols = new ArrayList(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); diff --git hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index 4fe1328..360acd7 100644 --- hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -53,6 +53,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertArrayEquals; +import org.apache.hadoop.util.Shell; + public class TestHCatClient { private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class); private static final String msPort = "20101"; @@ -100,7 +102,16 @@ public static void startMetaStoreServer() throws Exception { System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); } - + public static String fixPath(String path) { + if(!Shell.WINDOWS) { + return path; + } + String expectedDir = path.replaceAll("\\\\", "/"); + if (!expectedDir.startsWith("/")) { + expectedDir = "/" + expectedDir; + } + return expectedDir; + } @Test public void testBasicDDLCommands() throws Exception { String db = "testdb"; @@ -121,7 +132,7 @@ public void testBasicDDLCommands() throws Exception { assertTrue(testDb.getProperties().size() == 0); String warehouseDir = System .getProperty("test.warehouse.dir", "/user/hive/warehouse"); - String expectedDir = warehouseDir.replaceAll("\\\\", "/").replaceFirst("pfile:///", "pfile:/"); + String expectedDir = fixPath(warehouseDir).replaceFirst("pfile:///", "pfile:/"); assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation()); ArrayList cols = new ArrayList(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); diff --git itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java index 61d14f1..abf889b 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java +++ itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java @@ -43,7 +43,7 @@ import org.apache.hcatalog.HcatTestUtils; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.common.HCatException; -import org.apache.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hcatalog.data.DefaultHCatRecord; import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; @@ -72,8 +72,8 @@ public void setup() throws Exception { dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator + TestSequenceFileReadWrite.class.getCanonicalName() + "-" + System.currentTimeMillis()); hiveConf = new HiveConf(this.getClass()); - warehouseDir = new File(dataDir, "warehouse").getAbsolutePath(); - inputFileName = new File(dataDir, "input.data").getAbsolutePath(); + warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator + "warehouse"); + inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator + "input.data"); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java index 6bd278e..f907380 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.DataType; @@ -138,7 +139,7 @@ public void testPigHBaseSchema() throws Exception { String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = "testTable"; - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb"); String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'"; @@ -193,7 +194,7 @@ public void testPigFilterProjection() throws Exception { String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb"); String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'"; @@ -255,7 +256,7 @@ public void testPigPopulation() throws Exception { String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb"); String POPTXT_FILE_NAME = db_dir+"testfile.txt"; float f = -100.1f; diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java index e567c23..f4c3c20 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java @@ -69,8 +69,8 @@ public void setup() throws Exception { dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator + TestSequenceFileReadWrite.class.getCanonicalName() + "-" + System.currentTimeMillis()); hiveConf = new HiveConf(this.getClass()); - warehouseDir = new File(dataDir, "warehouse").getAbsolutePath(); - inputFileName = new File(dataDir, "input.data").getAbsolutePath(); + warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator + "warehouse"); + inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator + "input.data"); hiveConf = new HiveConf(this.getClass()); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");