diff --git common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java index a31238b..dfd3dbf 100644 --- common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java +++ common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.util.Shell; import org.apache.hive.common.util.HiveTestUtils; import org.junit.Assert; import org.junit.Test; @@ -35,7 +36,13 @@ @Test public void testHiveSitePath() throws Exception { String expectedPath = HiveTestUtils.getFileFromClasspath("hive-site.xml"); - Assert.assertEquals(expectedPath, new HiveConf().getHiveSiteLocation().getPath()); + String hiveSiteLocation = new HiveConf().getHiveSiteLocation().getPath(); + if (Shell.WINDOWS) { + // Do case-insensitive comparison on Windows, as drive letter can have different case. + expectedPath = expectedPath.toLowerCase(); + hiveSiteLocation = hiveSiteLocation.toLowerCase(); + } + Assert.assertEquals(expectedPath, hiveSiteLocation); } private void checkHadoopConf(String name, String expectedHadoopVal) throws Exception { diff --git ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java new file mode 100644 index 0000000..ce9c207 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.util.Shell; + +public class WindowsPathUtil { + + public static void convertPathsFromWindowsToHdfs(HiveConf conf){ + String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); + conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, getHdfsUriString(orgWarehouseDir)); + + String orgTestTempDir = System.getProperty("test.tmp.dir"); + System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir)); + + String orgTestDataDir = System.getProperty("test.src.data.dir"); + System.setProperty("test.src.data.dir", getHdfsUriString(orgTestDataDir)); + + String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR); + conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir)); + } + + private static String getHdfsUriString(String uriStr) { + assert uriStr != null; + if(Shell.WINDOWS) { + // If the URI conversion is from Windows to HDFS then replace the '\' with '/' + // and remove the windows single drive letter & colon from absolute path. + return uriStr.replace('\\', '/') + .replaceFirst("/[c-zC-Z]:", "/") + .replaceFirst("^[c-zC-Z]:", ""); + } + return uriStr; + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index 91efb58..b548672 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.exec.mr.ExecDriver; import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; @@ -59,6 +60,7 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.util.Shell; /** * Mimics the actual query compiler in generating end to end plans and testing @@ -80,6 +82,11 @@ conf = new HiveConf(ExecDriver.class); SessionState.start(conf); + //convert possible incompatible Windows path in config + if (Shell.WINDOWS) { + WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); + } + fs = FileSystem.get(conf); if (fs.exists(tmppath) && !fs.getFileStatus(tmppath).isDir()) { throw new RuntimeException(tmpdir + " exists but is not a directory"); @@ -161,7 +168,8 @@ private static void fileDiff(String datafile, String testdir) throws Exception { } FSDataInputStream fi_test = fs.open((fs.listStatus(di_test))[0].getPath()); - if (!Utilities.contentsEqual(fi_gold, fi_test, false)) { + boolean ignoreWhitespace = Shell.WINDOWS; + if (!Utilities.contentsEqual(fi_gold, fi_test, ignoreWhitespace)) { LOG.error(di_test.toString() + " does not match " + datafile); assertEquals(false, true); } diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java index e453f56..5b8ec60 100644 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java @@ -34,8 +34,10 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; +import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.util.Shell; import org.apache.thrift.TException; /** @@ -61,6 +63,9 @@ protected void setUp() throws Exception { super.setUp(); hive = Hive.get(); + if (Shell.WINDOWS) { + WindowsPathUtil.convertPathsFromWindowsToHdfs(hive.getConf()); + } checker = new HiveMetaStoreChecker(hive); partCols = new ArrayList();