diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java index 6e531dd..1984fa9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java @@ -317,8 +317,6 @@ private void commitOutputPathToFinalPath(Path specPath, Path outPath = getOperatorOutputPath(specPath); Path finalPath = getOperatorFinalPath(specPath); FileSystem fs = outPath.getFileSystem(hconf); - // for local file system in Hadoop-0.17.2.1, it will throw IOException when - // file not existing. try { if (!fs.rename(outPath, finalPath)) { throw new IOException("Unable to rename output to: " + finalPath); @@ -328,9 +326,6 @@ private void commitOutputPathToFinalPath(Path specPath, throw e; } } catch (IOException e) { - if (!fs.exists(outPath) && ignoreNonExisting) { - return; - } throw e; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 113227d..0bf99e1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -1408,10 +1408,6 @@ public static String realFile(String newFile, Configuration conf) throws IOExcep } String file = path.makeQualified(fs).toString(); - // For compatibility with hadoop 0.17, change file:/a/b/c to file:///a/b/c - if (StringUtils.startsWith(file, "file:/") && !StringUtils.startsWith(file, "file:///")) { - file = "file:///" + file.substring("file:/".length()); - } return file; } @@ -1913,6 +1909,26 @@ public static String getResourceFiles(Configuration conf, SessionState.ResourceT } /** + * Create a URL from a string representing a path to a local file. + * The path string can be just a path, or can start with file:/, file:/// + * @param onestr path string + * @return + */ + private static URL urlFromPathString(String onestr) { + URL oneurl = null; + try { + if (StringUtils.indexOf(onestr, "file:/") == 0) { + oneurl = new URL(onestr); + } else { + oneurl = new File(onestr).toURL(); + } + } catch (Exception err) { + LOG.error("Bad URL " + onestr + ", ignoring path"); + } + return oneurl; + } + + /** * Add new elements to the classpath. * * @param newPaths @@ -1930,13 +1946,8 @@ public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) curPath = newPath; for (String onestr : newPaths) { - // special processing for hadoop-17. file:// needs to be removed - if (StringUtils.indexOf(onestr, "file://") == 0) { - onestr = StringUtils.substring(onestr, 7); - } - - URL oneurl = (new File(onestr)).toURL(); - if (!curPath.contains(oneurl)) { + URL oneurl = urlFromPathString(onestr); + if (oneurl != null && !curPath.contains(oneurl)) { curPath.add(oneurl); } } @@ -1956,13 +1967,10 @@ public static void removeFromClassPath(String[] pathsToRemove) throws Exception Set newPath = new HashSet(Arrays.asList(loader.getURLs())); for (String onestr : pathsToRemove) { - // special processing for hadoop-17. file:// needs to be removed - if (StringUtils.indexOf(onestr, "file://") == 0) { - onestr = StringUtils.substring(onestr, 7); + URL oneurl = urlFromPathString(onestr); + if (oneurl != null) { + newPath.remove(oneurl); } - - URL oneurl = (new File(onestr)).toURL(); - newPath.remove(oneurl); } loader = new URLClassLoader(newPath.toArray(new URL[0])); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java index 574c8f7..07a789f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java @@ -213,13 +213,6 @@ private static void getTargetPathsFromSymlinksDirs( } } - /** - * For backward compatibility with hadoop 0.17. - */ - public void validateInput(JobConf job) throws IOException { - // do nothing - } - @Override public ContentSummary getContentSummary(Path p, JobConf job) throws IOException { diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index 82c1263..17edcd1 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -47,7 +47,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; @@ -334,18 +334,13 @@ public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } - if (field instanceof BytesWritable) { - BytesWritable b = (BytesWritable) field; - // For backward-compatibility with hadoop 0.17 + if (field instanceof BinaryComparable) { + BinaryComparable b = (BinaryComparable) field; byteArrayRef.setData(b.getBytes()); cachedLazyStruct.init(byteArrayRef, 0, b.getLength()); - } else if (field instanceof Text) { - Text t = (Text) field; - byteArrayRef.setData(t.getBytes()); - cachedLazyStruct.init(byteArrayRef, 0, t.getLength()); } else { throw new SerDeException(getClass().toString() - + ": expects either BytesWritable or Text object!"); + + ": expects BinaryComparable object, got " + field.getClass().toString()); } lastOperationSerialize = false; lastOperationDeserialize = true; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java index b188c3f..563bff6 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java @@ -166,19 +166,11 @@ public Object deserialize(Writable field) throws SerDeException { if (b.getLength() == 0) { return null; } - // For backward-compatibility with hadoop 0.17 byteArrayRef.setData(b.getBytes()); cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getLength()); - } else if (field instanceof Text) { - Text t = (Text) field; - if (t.getLength() == 0) { - return null; - } - byteArrayRef.setData(t.getBytes()); - cachedLazyBinaryStruct.init(byteArrayRef, 0, t.getLength()); } else { throw new SerDeException(getClass().toString() - + ": expects either BinaryComparable or Text object!"); + + ": expects BinaryComparable object, got " + field.getClass().toString()); } lastOperationSerialize = false; lastOperationDeserialize = true;