Index: ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java =================================================================== --- ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (revision 1363034) +++ ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (working copy) @@ -29,6 +29,7 @@ import java.util.List; import java.util.ArrayList; import java.util.regex.Pattern; +import java.util.HashMap; import org.apache.commons.lang.StringUtils; import org.apache.tools.ant.AntClassLoader; @@ -259,6 +260,7 @@ } List qFiles = new ArrayList(); + HashMap qFilesMap = new HashMap(); File outDir = null; File resultsDir = null; File logDir = null; @@ -281,7 +283,7 @@ } else if (queryFileRegex != null && !queryFileRegex.equals("")) { qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileRegexFilter(queryFileRegex)))); } else if (runDisabled != null && runDisabled.equals("true")) { - qFiles.addAll(Arrays.asList(inpDir.listFiles(new DisabledQFileFilter()))); + qFiles.addAll(Arrays.asList(inpDir.listFiles(new DisabledQFileFilter()))); } else { qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileFilter()))); } @@ -298,6 +300,9 @@ } Collections.sort(qFiles); + for (File qFile : qFiles) { + qFilesMap.put(qFile.getName(), getEscapedCanonicalPath(qFile)); + } // Make sure the output directory exists, if it doesn't // then create it. @@ -348,8 +353,9 @@ VelocityContext ctx = new VelocityContext(); ctx.put("className", className); ctx.put("qfiles", qFiles); - ctx.put("resultsDir", resultsDir); - ctx.put("logDir", logDir); + ctx.put("qfilesMap", qFilesMap); + ctx.put("resultsDir", getEscapedCanonicalPath(resultsDir)); + ctx.put("logDir", getEscapedCanonicalPath(logDir)); ctx.put("clusterMode", clusterMode); ctx.put("hadoopVersion", hadoopVersion); @@ -373,4 +379,17 @@ throw new BuildException("Generation failed", e); } } + + private static String getEscapedCanonicalPath(File file) throws IOException { + if (System.getProperty("os.name").toLowerCase().startsWith("win")) { + // Escape the backward slash in CanonicalPath if the unit test runs on windows + // e.g. dir.getCanonicalPath() gets the absolute path of local + // directory. When we embed it directly in the generated java class it results + // in compiler error in windows. Reason : the canonical path contains backward + // slashes "C:\temp\etc\" and it is not a valid string in Java + // unless we escape the backward slashes. + return file.getCanonicalPath().replace("\\", "\\\\"); + } + return file.getCanonicalPath(); + } } Index: shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java =================================================================== --- shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java (revision 1363034) +++ shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java (working copy) @@ -18,19 +18,17 @@ package org.apache.hadoop.fs; -import java.io.*; +import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.util.Shell; /**************************************************************** * A Proxy for LocalFileSystem * * Serves uri's corresponding to 'pfile:///' namespace with using - * a LocalFileSystem + * a LocalFileSystem *****************************************************************/ public class ProxyLocalFileSystem extends FilterFileSystem { @@ -50,10 +48,21 @@ // create a proxy for the local filesystem // the scheme/authority serving as the proxy is derived // from the supplied URI + String scheme = name.getScheme(); + String nameUriString = name.toString(); + if (Shell.WINDOWS) { + // Replace the encoded backward slash with forward slash + // Remove the windows drive letter + // replace the '=' with special string '------' to handle the unsupported char '=' in windows. + nameUriString = nameUriString.replaceAll("%5C", "/") + .replaceFirst("/[c-zC-Z]:", "/") + .replaceFirst("^[c-zC-Z]:", "") + .replaceAll("=", "------"); + name = URI.create(nameUriString); + } - String scheme = name.getScheme(); String authority = name.getAuthority() != null ? name.getAuthority() : ""; - String proxyUriString = name + "://" + authority + "/"; + String proxyUriString = nameUriString + "://" + authority + "/"; fs = new ProxyFileSystem(localFs, URI.create(proxyUriString)); fs.initialize(name, conf); Index: shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java =================================================================== --- shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java (revision 1363034) +++ shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java (working copy) @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.util.Shell; /**************************************************************** * A FileSystem that can serve a given scheme/authority using some @@ -42,14 +43,28 @@ protected String realAuthority; protected URI realUri; - + private Path swizzleParamPath(Path p) { - return new Path (realScheme, realAuthority, p.toUri().getPath()); + String pathUriString = p.toUri().toString(); + if (Shell.WINDOWS) { + // Some of the file paths (Files with partition option) in HDFS '=' + // but Windows file path doesn't support '=' so replace it with special string. + pathUriString = pathUriString.replaceAll("=", "------"); + } + URI newPathUri = URI.create(pathUriString); + return new Path (realScheme, realAuthority, newPathUri.getPath()); } private Path swizzleReturnPath(Path p) { - return new Path (myScheme, myAuthority, p.toUri().getPath()); + String pathUriString = p.toUri().toString(); + if (Shell.WINDOWS) { + // Revert back the special string '------' with '=' when we do the reverse conversion + // from Windows path to HDFS + pathUriString = pathUriString.replaceAll("------", "="); + } + URI newPathUri = URI.create(pathUriString); + return new Path (myScheme, myAuthority, newPathUri.getPath()); } private FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) { @@ -66,14 +81,14 @@ public ProxyFileSystem() { throw new RuntimeException ("Unsupported constructor"); } - + public ProxyFileSystem(FileSystem fs) { throw new RuntimeException ("Unsupported constructor"); } /** * Create a proxy file system for fs. - * + * * @param fs FileSystem to create proxy for * @param myUri URI to use as proxy. Only the scheme and authority from * this are used right now @@ -158,7 +173,7 @@ public boolean rename(Path src, Path dst) throws IOException { return super.rename(swizzleParamPath(src), swizzleParamPath(dst)); } - + @Override public boolean delete(Path f, boolean recursive) throws IOException { return super.delete(swizzleParamPath(f), recursive); @@ -167,8 +182,8 @@ @Override public boolean deleteOnExit(Path f) throws IOException { return super.deleteOnExit(swizzleParamPath(f)); - } - + } + @Override public FileStatus[] listStatus(Path f) throws IOException { FileStatus[] orig = super.listStatus(swizzleParamPath(f)); @@ -178,7 +193,7 @@ } return ret; } - + @Override public Path getHomeDirectory() { return swizzleReturnPath(super.getHomeDirectory()); @@ -188,12 +203,12 @@ public void setWorkingDirectory(Path newDir) { super.setWorkingDirectory(swizzleParamPath(newDir)); } - + @Override public Path getWorkingDirectory() { return swizzleReturnPath(super.getWorkingDirectory()); } - + @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { return super.mkdirs(swizzleParamPath(f), permission); @@ -206,14 +221,14 @@ } @Override - public void copyFromLocalFile(boolean delSrc, boolean overwrite, + public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst) throws IOException { super.copyFromLocalFile(delSrc, overwrite, srcs, swizzleParamPath(dst)); } - + @Override - public void copyFromLocalFile(boolean delSrc, boolean overwrite, + public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException { super.copyFromLocalFile(delSrc, overwrite, src, swizzleParamPath(dst)); @@ -251,7 +266,7 @@ public FileChecksum getFileChecksum(Path f) throws IOException { return super.getFileChecksum(swizzleParamPath(f)); } - + @Override public void setOwner(Path p, String username, String groupname ) throws IOException { @@ -270,4 +285,4 @@ super.setPermission(swizzleParamPath(p), permission); } } - + Index: conf/hive-default.xml.template =================================================================== --- conf/hive-default.xml.template (revision 1363034) +++ conf/hive-default.xml.template (working copy) @@ -69,6 +69,12 @@ + hive.exec.local.scratchdir + /tmp/${user.name} + Local scratch space for Hive jobs + + + hive.test.mode false whether hive is running in test mode. If yes, it turns on sampling and prefixes the output tablename Index: hbase-handler/src/test/templates/TestHBaseCliDriver.vm =================================================================== --- hbase-handler/src/test/templates/TestHBaseCliDriver.vm (revision 1363034) +++ hbase-handler/src/test/templates/TestHBaseCliDriver.vm (working copy) @@ -56,13 +56,10 @@ if ("$clusterMode".equals("miniMR")) { miniMR = true; } + qt = new HBaseQTestUtil("$resultsDir", "$logDir", miniMR, setup); - qt = new HBaseQTestUtil( - "$resultsDir.getCanonicalPath()", - "$logDir.getCanonicalPath()", miniMR, setup); - #foreach ($qf in $qfiles) - qt.addFile("$qf.getCanonicalPath()"); + qt.addFile("$qfilesMap.get($qf.getName())"); #end } catch (Exception e) { System.out.println("Exception: " + e.getMessage()); Index: hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm =================================================================== --- hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm (revision 1363034) +++ hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm (working copy) @@ -40,12 +40,10 @@ miniMR = true; } - qt = new HBaseQTestUtil( - "$resultsDir.getCanonicalPath()", - "$logDir.getCanonicalPath()", miniMR, setup); + qt = new HBaseQTestUtil("$resultsDir", "$logDir", miniMR, setup); #foreach ($qf in $qfiles) - qt.addFile("$qf.getCanonicalPath()"); + qt.addFile("$qfilesMap.get($qf.getName())"); #end } catch (Exception e) { System.out.println("Exception: " + e.getMessage()); Index: build.xml =================================================================== --- build.xml (revision 1363034) +++ build.xml (working copy) @@ -532,9 +532,6 @@ - - - Index: testutils/hadoop.cmd =================================================================== --- testutils/hadoop.cmd (revision 0) +++ testutils/hadoop.cmd (revision 0) @@ -0,0 +1,253 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + + +@rem The Hadoop command script +@rem +@rem Environment Variables +@rem +@rem JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +@rem +@rem HADOOP_CLASSPATH Extra Java CLASSPATH entries. +@rem +@rem HADOOP_HEAPSIZE The maximum amount of heap to use, in MB. +@rem Default is 1000. +@rem +@rem HADOOP_OPTS Extra Java runtime options. +@rem +@rem HADOOP_NAMENODE_OPTS These options are added to HADOOP_OPTS +@rem HADOOP_CLIENT_OPTS when the respective command is run. +@rem HADOOP_{COMMAND}_OPTS etc HADOOP_JT_OPTS applies to JobTracker +@rem for e.g. HADOOP_CLIENT_OPTS applies to +@rem more than one command (fs, dfs, fsck, +@rem dfsadmin etc) +@rem +@rem HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +@rem +@rem HADOOP_ROOT_LOGGER The root appender. Default is INFO,console +@rem + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) +call :updatepath %HADOOP_BIN_PATH% + +set BIN=%~dp0 +for %%i in (%BIN%.) do ( + set BIN=%%~dpi +) +if "%BIN:~-1%" == "\" ( + set BIN=%BIN:~0,-1% +) + + +@rem +@rem setup java environment variables +@rem + +if not defined JAVA_HOME ( + echo Error: JAVA_HOME is not set. + goto :eof +) + +if not exist %JAVA_HOME%\bin\java.exe ( + echo Error: JAVA_HOME is incorrectly set. + goto :eof +) + +set JAVA=%JAVA_HOME%\bin\java +set JAVA_HEAP_MAX=-Xmx1000m + +@rem +@rem check envvars which might override default args +@rem + +if defined HADOOP_HEAPSIZE ( + set JAVA_HEAP_MAX=-Xmx%HADOOP_HEAPSIZE%m +) + +@rem +@rem CLASSPATH initially contains %HADOOP_CONF_DIR% +@rem + +set CLASSPATH=%HADOOP_CONF_DIR% +set CLASSPATH=%CLASSPATH%;%JAVA_HOME%\lib\tools.jar + + +set BUILD_ROOT="%BIN%"/build + + +if not defined HIVE_HADOOP_TEST_CLASSPATH ( + @echo Error: HIVE_HADOOP_TEST_CLASSPATH not defined. + goto :eof +) + + + +set CLASSPATH=%CLASSPATH%;%HIVE_HADOOP_TEST_CLASSPATH% +if not exist %BUILD_ROOT%/test/hadoop/logs ( + mkdir %BUILD_ROOT%/test/hadoop/logs +) + +@rem +@rem add user-specified CLASSPATH last +@rem + +if defined HADOOP_CLASSPATH ( + set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH% +) + +if not defined HADOOP_LOG_DIR ( + set HADOOP_LOG_DIR=%BUILD_ROOT%\logs +) + +if not defined HADOOP_LOGFILE ( + set HADOOP_LOGFILE=hadoop.log +) + +if not defined HADOOP_ROOT_LOGGER ( + set HADOOP_ROOT_LOGGER=INFO,console,DRFA +) + +@rem +@rem default policy file for service-level authorization +@rem + +if not defined HADOOP_POLICYFILE ( + set HADOOP_POLICYFILE=hadoop-policy.xml +) +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR% +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.file=%HADOOP_LOGFILE% +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.root.logger=%HADOOP_ROOT_LOGGER% + +if defined HADOOP_PREFIX ( + set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.home.dir=%HADOOP_PREFIX% +) + +if defined HADOOP_IDENT_STRING ( + set HADOOP_OPTS=%$HADOOP_OPTS% -Dhadoop.id.str=%HADOOP_IDENT_STRING% +) + +if defined JAVA_LIBRARY_PATH ( + set HADOOP_OPTS=%HADOOP_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH% +) +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.policy.file=%HADOOP_POLICYFILE% + +@rem Disable ipv6 as it can cause issues +set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true + +:main + setlocal enabledelayedexpansion + + set hadoop-command=%1 + if not defined hadoop-command ( + goto print_usage + ) + + call :make_command_arguments %* + set corecommands=fs version jar distcp daemonlog archive + for %%i in ( %corecommands% ) do ( + if %hadoop-command% == %%i set corecommand=true + ) + if defined corecommand ( + call :%hadoop-command% + ) else ( + set CLASSPATH=%CLASSPATH%;%CD% + set CLASS=%hadoop-command% + ) + set path=%HADOOP_BIN_PATH%;%windir%\system32;%windir% + call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% %hadoop-command-arguments% + + goto :eof + +:version + set CLASS=org.apache.hadoop.util.VersionInfo + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + goto :eof + +:jar + set CLASS=org.apache.hadoop.util.RunJar + goto :eof + +:distcp + set CLASS=org.apache.hadoop.tools.DistCp + set CLASSPATH=%CLASSPATH%;%TOOL_PATH% + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + goto :eof + +:daemonlog + set CLASS=org.apache.hadoop.log.LogLevel + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + goto :eof + +:archive + set CLASS=org.apache.hadoop.tools.HadoopArchives + set CLASSPATH=%CLASSPATH%;%TOOL_PATH% + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + goto :eof + +:updatepath + set path_to_add=%* + set current_path_comparable=%path:(x86)=% + set current_path_comparable=%current_path_comparable: =_% + set path_to_add_comparable=%path_to_add:(x86)=% + set path_to_add_comparable=%path_to_add_comparable: =_% + for %%i in ( %current_path_comparable% ) do ( + if /i "%%i" == "%path_to_add_comparable%" ( + set path_to_add_exist=true + ) + ) + set system_path_comparable= + set path_to_add_comparable= + if not defined path_to_add_exist path=%path_to_add%;%path% + set path_to_add= + goto :eof + +:make_command_arguments + if "%2" == "" goto :eof + set _count=0 + set _shift=1 + for %%i in (%*) do ( + set /a _count=!_count!+1 + if !_count! GTR %_shift% ( + if not defined _arguments ( + set _arguments=%%i + ) else ( + set _arguments=!_arguments! %%i + ) + ) + ) + + set hadoop-command-arguments=%_arguments% + goto :eof + +:print_usage + @echo Usage: hadoop COMMAND + @echo where COMMAND is one of: + @echo fs run a generic filesystem user client + @echo version print the version + @echo jar ^ run a jar file + @echo. + @echo distcp ^ ^ copy file or directories recursively + @echo archive -archiveName NAME ^* ^ create a hadoop archive + @echo daemonlog get/set the log level for each daemon + @echo Most commands print help when invoked w/o parameters. + +endlocal Index: data/conf/hive-site.xml =================================================================== --- data/conf/hive-site.xml (revision 1363034) +++ data/conf/hive-site.xml (working copy) @@ -47,6 +47,12 @@ + hive.exec.local.scratchdir + ${build.dir}/scratchdir/local/ + Local scratch space for Hive jobs + + + javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=../build/test/junit_metastore_db;create=true Index: build-common.xml =================================================================== --- build-common.xml (revision 1363034) +++ build-common.xml (working copy) @@ -109,6 +109,8 @@ + + @@ -366,7 +368,7 @@ + depends="test-conditions,gen-test,compile-test,test-jar,test-init"> @@ -378,25 +380,37 @@ + + + + + + + + + + + + - + - + - + @@ -413,7 +427,7 @@ - + Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1363034) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -35,9 +35,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell; /** * Hive Configuration. @@ -163,6 +165,7 @@ SCRIPTWRAPPER("hive.exec.script.wrapper", null), PLAN("hive.exec.plan", ""), SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-" + System.getProperty("user.name")), + LOCALSCRATCHDIR("hive.exec.local.scratchdir", "/tmp/" + System.getProperty("user.name")), SUBMITVIACHILD("hive.exec.submitviachild", false), SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", 100000), ALLOWPARTIALCONSUMP("hive.exec.script.allow.partial.consumption", false), @@ -690,8 +693,10 @@ val = System.getenv("HADOOP_PREFIX"); } // and if all else fails we can at least try /usr/bin/hadoop - return (val == null ? File.separator + "usr" : val) + val = (val == null ? File.separator + "usr" : val) + File.separator + "bin" + File.separator + "hadoop"; + // Launch hadoop command file on windows. + return val + (Shell.WINDOWS ? ".cmd" : ""); } enum VarType { @@ -933,6 +938,7 @@ hiveJar = this.get(ConfVars.HIVEJAR.varname); } + hiveJar = new Path(hiveJar).toString(); if (auxJars == null) { auxJars = this.get(ConfVars.HIVEAUXJARS.varname); } Index: contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java (revision 1363034) +++ contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java (working copy) @@ -24,6 +24,8 @@ import junit.framework.TestCase; +import org.apache.hadoop.util.Shell; + /** * TestGenericMR. * @@ -61,8 +63,7 @@ final StringWriter out = new StringWriter(); new GenericMR().map(new StringReader(in), out, identityMapper()); - - assertEquals(in + "\n", out.toString()); + assertEquals(in + "\n", getOsSpecificOutput(out.toString())); } public void testKVSplitMap() throws Exception { @@ -79,7 +80,7 @@ } }); - assertEquals(expected, out.toString()); + assertEquals(expected, getOsSpecificOutput(out.toString())); } public void testIdentityReduce() throws Exception { @@ -88,7 +89,7 @@ new GenericMR().reduce(new StringReader(in), out, identityReducer()); - assertEquals(in + "\n", out.toString()); + assertEquals(in + "\n", getOsSpecificOutput(out.toString())); } public void testWordCountReduce() throws Exception { @@ -111,7 +112,7 @@ final String expected = "hello\t3\nokay\t12\n"; - assertEquals(expected, out.toString()); + assertEquals(expected, getOsSpecificOutput(out.toString())); } private Mapper identityMapper() { @@ -134,4 +135,9 @@ } }; } + + private static String getOsSpecificOutput(String outStr){ + assert outStr != null; + return Shell.WINDOWS ? outStr.replaceAll("\\r", "") : outStr; + } } Index: odbc/build.xml =================================================================== --- odbc/build.xml (revision 1363034) +++ odbc/build.xml (working copy) @@ -29,6 +29,21 @@ + + + + + + + + + + + + + + + @@ -49,7 +64,7 @@ - + @@ -63,7 +78,7 @@ - + @@ -73,7 +88,7 @@ - + @@ -84,7 +99,7 @@ - + @@ -94,7 +109,7 @@ - + @@ -104,7 +119,4 @@ - - - Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 1363034) +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -33,7 +33,6 @@ import java.io.PrintStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; -import java.lang.UnsupportedOperationException; import java.util.ArrayList; import java.util.Arrays; import java.util.Deque; @@ -46,7 +45,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; @@ -82,6 +80,7 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.util.Shell; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.zookeeper.ZooKeeper; @@ -205,19 +204,63 @@ } public void initConf() throws Exception { + + convertPathsFromWindowsToHdfs(); + if (miniMr) { assert dfs != null; assert mr != null; // set fs.default.name to the uri of mini-dfs - conf.setVar(HiveConf.ConfVars.HADOOPFS, dfs.getFileSystem().getUri().toString()); + String dfsUriString = getHdfsUriString(dfs.getFileSystem().getUri().toString()); + conf.setVar(HiveConf.ConfVars.HADOOPFS, dfsUriString); // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, - (new Path(dfs.getFileSystem().getUri().toString(), + (new Path(dfsUriString, "/build/ql/test/data/warehouse/")).toString()); conf.setVar(HiveConf.ConfVars.HADOOPJT, "localhost:" + mr.getJobTrackerPort()); } } + private void convertPathsFromWindowsToHdfs() { + // Following local paths are used as HDFS paths in unit tests. + // It works well in Unix as the path notation in Unix and HDFS is more or less same. + // But when it comes to Windows, drive letter separator ':' & backslash '\" are invalid + // characters in HDFS so we need to converts these local paths to HDFS paths before using them + // in unit tests. + if (Shell.WINDOWS) { + // hive.exec.scratchdir needs to be set relative to the mini-dfs + String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); + conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, getHdfsUriString(orgWarehouseDir)); + + String orgTestTempDir = System.getProperty("test.tmp.dir"); + System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir)); + + String orgTestDataDir = System.getProperty("test.src.data.dir"); + System.setProperty("test.src.data.dir", getHdfsUriString(orgTestDataDir)); + + String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR); + conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir)); + + if (miniMr) { + String orgAuxJarFolder = conf.getAuxJars(); + conf.setAuxJars(getHdfsUriString("file://" + orgAuxJarFolder)); + } + } + } + + private String getHdfsUriString(String uriStr) { + assert uriStr != null; + if(Shell.WINDOWS) { + // If the URI conversion is from Windows to HDFS then replace the '\' with '/' + // and remove the windows single drive letter & colon from absolute path. + return uriStr.replace('\\', '/') + .replaceFirst("/[c-zC-Z]:", "/") + .replaceFirst("^[c-zC-Z]:", ""); + } + + return uriStr; + } + public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) throws Exception { this.outDir = outDir; @@ -231,7 +274,7 @@ if (miniMr) { dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null); FileSystem fs = dfs.getFileSystem(); - mr = new MiniMRCluster(4, fs.getUri().toString(), 1); + mr = new MiniMRCluster(4, getHdfsUriString(fs.getUri().toString()), 1); } initConf(); @@ -242,8 +285,7 @@ dataDir = new File(".").getAbsolutePath() + "/data/files"; } - testFiles = dataDir.replace('\\', '/') - .replace("c:", ""); + testFiles = dataDir; String ow = System.getProperty("test.output.overwrite"); if ((ow != null) && ow.equalsIgnoreCase("true")) { @@ -285,7 +327,7 @@ // Look for a hint to not run a test on some Hadoop versions Pattern pattern = Pattern.compile("-- (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS\\((.*)\\)"); - + boolean excludeQuery = false; boolean includeQuery = false; Set versionSet = new HashSet(); @@ -313,7 +355,7 @@ + " contains more than one reference to (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS"; throw new UnsupportedOperationException(message); } - + String prefix = matcher.group(1); if ("EX".equals(prefix)) { excludeQuery = true; @@ -330,7 +372,7 @@ qsb.append(line + "\n"); } qMap.put(qf.getName(), qsb.toString()); - + if (excludeQuery && versionSet.contains(hadoopVer)) { System.out.println("QTestUtil: " + qf.getName() + " EXCLUDE list contains Hadoop Version " + hadoopVer + ". Skipping..."); @@ -469,7 +511,7 @@ // db.createPartition(srcpart, part_spec); fpath = new Path(testFiles, "kv1.txt"); // db.loadPartition(fpath, srcpart.getName(), part_spec, true); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='" + hr + "')"); } @@ -481,7 +523,7 @@ // IgnoreKeyTextOutputFormat.class, 2, bucketCols); for (String fname : new String[] {"srcbucket0.txt", "srcbucket1.txt"}) { fpath = new Path(testFiles, fname); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE srcbucket"); } @@ -492,7 +534,7 @@ for (String fname : new String[] {"srcbucket20.txt", "srcbucket21.txt", "srcbucket22.txt", "srcbucket23.txt"}) { fpath = new Path(testFiles, fname); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE srcbucket2"); } @@ -520,25 +562,25 @@ // load the input data into the src table fpath = new Path(testFiles, "kv1.txt"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src"); + runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src"); // load the input data into the src table fpath = new Path(testFiles, "kv3.txt"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src1"); + runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src1"); // load the input data into the src_sequencefile table fpath = new Path(testFiles, "kv1.seq"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src_sequencefile"); // load the input data into the src_thrift table fpath = new Path(testFiles, "complex.seq"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src_thrift"); // load the json data into the src_json table fpath = new Path(testFiles, "json.txt"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src_json"); conf.setBoolean("hive.test.init.phase", false); } @@ -712,26 +754,9 @@ outfd.write(e.getMessage()); outfd.close(); - String cmdLine = "diff " + outf.getPath() + " " + expf; - System.out.println(cmdLine); - - Process executor = Runtime.getRuntime().exec(cmdLine); - - StreamPrinter outPrinter = new StreamPrinter( - executor.getInputStream(), null, SessionState.getConsole().getChildOutStream()); - StreamPrinter errPrinter = new StreamPrinter( - executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream()); - - outPrinter.start(); - errPrinter.start(); - - int exitVal = executor.waitFor(); - + int exitVal = executeDiffCommand(outf.getPath(), expf, false); if (exitVal != 0 && overWrite) { - System.out.println("Overwriting results"); - cmdLine = "cp " + outf.getPath() + " " + expf; - executor = Runtime.getRuntime().exec(cmdLine); - exitVal = executor.waitFor(); + exitVal = overwriteResults(outf.getPath(), expf); } return exitVal; @@ -751,26 +776,10 @@ outfd.write(tree.toStringTree()); outfd.close(); - String cmdLine = "diff " + outf.getPath() + " " + expf; - System.out.println(cmdLine); + int exitVal = executeDiffCommand(outf.getPath(), expf, false); - Process executor = Runtime.getRuntime().exec(cmdLine); - - StreamPrinter outPrinter = new StreamPrinter( - executor.getInputStream(), null, SessionState.getConsole().getChildOutStream()); - StreamPrinter errPrinter = new StreamPrinter( - executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream()); - - outPrinter.start(); - errPrinter.start(); - - int exitVal = executor.waitFor(); - if (exitVal != 0 && overWrite) { - System.out.println("Overwriting results"); - cmdLine = "cp " + outf.getPath() + " " + expf; - executor = Runtime.getRuntime().exec(cmdLine); - exitVal = executor.waitFor(); + exitVal = overwriteResults(outf.getPath(), expf); } return exitVal; @@ -804,31 +813,10 @@ }; maskPatterns(patterns, outf.getPath()); - String[] cmdArray = new String[] { - "diff", - "-b", - outf.getPath(), - planFile - }; - System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' ')); + int exitVal = executeDiffCommand(outf.getPath(), planFile, true); - Process executor = Runtime.getRuntime().exec(cmdArray); - - StreamPrinter outPrinter = new StreamPrinter( - executor.getInputStream(), null, SessionState.getConsole().getChildOutStream()); - StreamPrinter errPrinter = new StreamPrinter( - executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream()); - - outPrinter.start(); - errPrinter.start(); - - int exitVal = executor.waitFor(); - if (exitVal != 0 && overWrite) { - System.out.println("Overwriting results"); - String cmdLine = "cp " + outf.getPath() + " " + planFile; - executor = Runtime.getRuntime().exec(cmdLine); - exitVal = executor.waitFor(); + exitVal = overwriteResults(outf.getPath(), planFile); } return exitVal; @@ -883,6 +871,10 @@ in = new BufferedReader(new FileReader(fname)); out = new BufferedWriter(new FileWriter(fname + ".orig")); while (null != (line = in.readLine())) { + // Ignore the empty lines on windows + if (line.isEmpty() && Shell.WINDOWS) { + continue; + } out.write(line); out.write('\n'); } @@ -954,13 +946,56 @@ "^Deleted.*", }; maskPatterns(patterns, (new File(logDir, tname + ".out")).getPath()); + int exitVal = executeDiffCommand((new File(logDir, tname + ".out")).getPath(), + outFileName, false); - cmdArray = new String[] { - "diff", "-a", - (new File(logDir, tname + ".out")).getPath(), - outFileName + if (exitVal != 0 && overWrite) { + exitVal = overwriteResults((new File(logDir, tname + ".out")).getPath(), outFileName); + } + + return exitVal; + } + + private static int overwriteResults(String inFileName, String outFileName) throws Exception { + // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING) + // once Hive uses JAVA 7. + System.out.println("Overwriting results"); + String[] cmdArray = new String[] { + "cp", + Shell.WINDOWS ? getQuotedString(inFileName) : inFileName, + Shell.WINDOWS ? getQuotedString(outFileName) : outFileName }; + Process executor = Runtime.getRuntime().exec(cmdArray); + return executor.waitFor(); + } + private static int executeDiffCommand(String inFileName, + String outFileName, + boolean ignoreWhiteSpace) throws Exception { + ArrayList diffCommandArgs = new ArrayList(); + diffCommandArgs.add("diff"); + + // Text file comparison + diffCommandArgs.add("-a"); + + // Ignore changes in the amount of white space + if (ignoreWhiteSpace || Shell.WINDOWS) { + diffCommandArgs.add("-b"); + } + + // Files created on Windows machines have different line endings + // than files created on Unix/Linux. Windows uses carriage return and line feed + // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n"). + // Also StringBuilder.toString(), Stream to String conversions adds extra + // spaces at the end of the line. + if (Shell.WINDOWS) { + diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input + diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank + } + // Add files to compare to the arguments list + diffCommandArgs.add(Shell.WINDOWS ? getQuotedString(inFileName) : inFileName); + diffCommandArgs.add(Shell.WINDOWS ? getQuotedString(outFileName) : outFileName); + String[] cmdArray =(String [])diffCommandArgs.toArray(new String [diffCommandArgs.size ()]); System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' ')); Process executor = Runtime.getRuntime().exec(cmdArray); @@ -973,19 +1008,11 @@ outPrinter.start(); errPrinter.start(); - int exitVal = executor.waitFor(); + return executor.waitFor(); + } - if (exitVal != 0 && overWrite) { - System.out.println("Overwriting results"); - cmdArray = new String[3]; - cmdArray[0] = "cp"; - cmdArray[1] = (new File(logDir, tname + ".out")).getPath(); - cmdArray[2] = outFileName; - executor = Runtime.getRuntime().exec(cmdArray); - exitVal = executor.waitFor(); - } - - return exitVal; + private static String getQuotedString(String str){ + return String.format("\"%s\"", str); } public ASTNode parseQuery(String tname) throws Exception { @@ -1210,4 +1237,4 @@ + "or try \"ant test ... -Dtest.silent=false\" to get more logs."); System.err.flush(); } -} +} \ No newline at end of file Index: ql/src/test/templates/TestCliDriver.vm =================================================================== --- ql/src/test/templates/TestCliDriver.vm (revision 1363034) +++ ql/src/test/templates/TestCliDriver.vm (working copy) @@ -48,7 +48,7 @@ if ("$clusterMode".equals("miniMR")) miniMR = true; hadoopVer = "$hadoopVersion"; - qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR, hadoopVer); + qt = new QTestUtil("$resultsDir", "$logDir", miniMR, hadoopVer); // do a one time initialization qt.cleanUp(); @@ -124,7 +124,7 @@ try { System.out.println("Begin query: " + "$fname"); - qt.addFile("$qf.getCanonicalPath()"); + qt.addFile("$qfilesMap.get($fname)"); if (qt.shouldBeSkipped("$fname")) { return; Index: ql/src/test/templates/TestNegativeCliDriver.vm =================================================================== --- ql/src/test/templates/TestNegativeCliDriver.vm (revision 1363034) +++ ql/src/test/templates/TestNegativeCliDriver.vm (working copy) @@ -41,7 +41,7 @@ if ("$clusterMode".equals("miniMR")) miniMR = true; hadoopVer = "$hadoopVersion"; - qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR, hadoopVer); + qt = new QTestUtil("$resultsDir", "$logDir", miniMR, hadoopVer); // do a one time initialization qt.cleanUp(); qt.createSources(); @@ -115,7 +115,7 @@ try { System.out.println("Begin query: " + "$fname"); - qt.addFile("$qf.getCanonicalPath()"); + qt.addFile("$qfilesMap.get($fname)"); if (qt.shouldBeSkipped("$fname")) { System.out.println("Test $fname skipped"); Index: ql/src/test/templates/TestParse.vm =================================================================== --- ql/src/test/templates/TestParse.vm (revision 1363034) +++ ql/src/test/templates/TestParse.vm (working copy) @@ -37,7 +37,7 @@ if ("$clusterMode".equals("miniMR")) miniMR = true; String hadoopVer = "$hadoopVersion"; - qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR, hadoopVer); + qt = new QTestUtil("$resultsDir", "$logDir", miniMR, hadoopVer); } catch (Exception e) { System.out.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -98,7 +98,7 @@ try { System.out.println("Begin query: " + "$fname"); - qt.addFile("$qf.getCanonicalPath()"); + qt.addFile("$qfilesMap.get($fname)"); qt.init("$fname"); ASTNode tree = qt.parseQuery("$fname"); Index: ql/src/test/templates/TestParseNegative.vm =================================================================== --- ql/src/test/templates/TestParseNegative.vm (revision 1363034) +++ ql/src/test/templates/TestParseNegative.vm (working copy) @@ -37,8 +37,7 @@ if ("$clusterMode".equals("miniMR")) miniMR = true; String hadoopVer = "$hadoopVersion"; - qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", - miniMR, hadoopVer); + qt = new QTestUtil("$resultsDir", "$logDir", miniMR, hadoopVer); } catch (Exception e) { System.out.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -98,7 +97,7 @@ try { System.out.println("Begin query: " + "$fname"); - qt.addFile("$qf.getCanonicalPath()"); + qt.addFile("$qfilesMap.get($fname)"); qt.init("$fname"); ASTNode tree = qt.parseQuery("$fname"); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (revision 1363034) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (working copy) @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.plan.CopyWork; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.ql.plan.MoveWork; +import org.apache.hadoop.util.Shell; /** * LoadSemanticAnalyzer. @@ -81,10 +82,10 @@ // directory if (!path.startsWith("/")) { if (isLocal) { - path = new Path(System.getProperty("user.dir"), path).toString(); + path = new Path(System.getProperty("user.dir"), path).toUri().toString(); } else { path = new Path(new Path("/user/" + System.getProperty("user.name")), - path).toString(); + path).toString(); } } @@ -231,7 +232,7 @@ } // create final load/move work - + String loadTmpPath = ctx.getExternalTmpFileURI(toURI); Map partSpec = ts.getPartSpec(); if (partSpec == null) { Index: ql/src/java/org/apache/hadoop/hive/ql/Context.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Context.java (revision 1363034) +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java (working copy) @@ -98,16 +98,13 @@ this.conf = conf; this.executionId = executionId; - // non-local tmp location is configurable. however it is the same across + // local & non-local tmp location is configurable. however it is the same across // all external file systems nonLocalScratchPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), executionId); - - // local tmp location is not configurable for now - localScratchDir = System.getProperty("java.io.tmpdir") - + Path.SEPARATOR + System.getProperty("user.name") + Path.SEPARATOR - + executionId; + localScratchDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR), + executionId).toUri().getPath(); } /**