commit 90c15633ddfd479e1923eb9e62b6cd7238f9e97a Author: Ivan Suller Date: Fri Jun 28 12:11:22 2019 +0200 HIVE-21933 Change-Id: Iee9c4853284f4f034d21991831e8bb6eac2f6e73 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index d91cd60668..3f7488f898 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -405,7 +405,7 @@ public static void cacheBaseWork(Configuration conf, String name, BaseWork work, /** * Pushes work into the global work map */ - public static void setBaseWork(Configuration conf, String name, BaseWork work) { + private static void setBaseWork(Configuration conf, String name, BaseWork work) { Path path = getPlanPath(conf, name); setHasWork(conf, name); gWorkMap.get(conf).put(path, work); @@ -754,14 +754,6 @@ public static String getTaskId(Configuration hconf) { } } - public static HashMap makeMap(Object... olist) { - HashMap ret = new HashMap(); - for (int i = 0; i < olist.length; i += 2) { - ret.put(olist[i], olist[i + 1]); - } - return (ret); - } - public static Properties makeProperties(String... olist) { Properties ret = new Properties(); for (int i = 0; i < olist.length; i += 2) { @@ -800,37 +792,11 @@ public static PartitionDesc getPartitionDesc(Partition part, TableDesc tableDesc return new PartitionDesc(part, tableDesc); } - public static PartitionDesc getPartitionDesc(Partition part) throws HiveException { - return new PartitionDesc(part); - } - public static PartitionDesc getPartitionDescFromTableDesc(TableDesc tblDesc, Partition part, boolean usePartSchemaProperties) throws HiveException { return new PartitionDesc(part, tblDesc, usePartSchemaProperties); } - private static String getOpTreeSkel_helper(Operator op, String indent) { - if (op == null) { - return StringUtils.EMPTY; - } - - StringBuilder sb = new StringBuilder(); - sb.append(indent); - sb.append(op.toString()); - sb.append("\n"); - if (op.getChildOperators() != null) { - for (Object child : op.getChildOperators()) { - sb.append(getOpTreeSkel_helper((Operator) child, indent + " ")); - } - } - - return sb.toString(); - } - - public static String getOpTreeSkel(Operator op) { - return getOpTreeSkel_helper(op, StringUtils.EMPTY); - } - private static boolean isWhitespace(int c) { if (c == -1) { return false; @@ -926,22 +892,6 @@ public static StreamStatus readColumn(DataInput in, OutputStream out) throws IOE // Unreachable } - /** - * Convert an output stream to a compressed output stream based on codecs and compression options - * specified in the Job Configuration. - * - * @param jc - * Job Configuration - * @param out - * Output Stream to be converted into compressed output stream - * @return compressed output stream - */ - public static OutputStream createCompressedStream(JobConf jc, OutputStream out) - throws IOException { - boolean isCompressed = FileOutputFormat.getCompressOutput(jc); - return createCompressedStream(jc, out, isCompressed); - } - /** * Convert an output stream to a compressed output stream based on codecs codecs in the Job * Configuration. Caller specifies directly whether file is compressed or not @@ -966,22 +916,6 @@ public static OutputStream createCompressedStream(JobConf jc, OutputStream out, } } - /** - * Based on compression option and configured output codec - get extension for output file. This - * is only required for text files - not sequencefiles - * - * @param jc - * Job Configuration - * @param isCompressed - * Whether the output file is compressed or not - * @return the required file extension (example: .gz) - * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} - */ - @Deprecated - public static String getFileExtension(JobConf jc, boolean isCompressed) { - return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); - } - /** * Based on compression option, output format, and configured output codec - * get extension for output file. Text files require an extension, whereas @@ -1014,27 +948,6 @@ public static String getFileExtension(JobConf jc, boolean isCompressed, return StringUtils.EMPTY; } - /** - * Create a sequencefile output stream based on job configuration. - * - * @param jc - * Job configuration - * @param fs - * File System to create file in - * @param file - * Path to be created - * @param keyClass - * Java Class for key - * @param valClass - * Java Class for value - * @return output stream over the created sequencefile - */ - public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path file, - Class keyClass, Class valClass, Progressable progressable) throws IOException { - boolean isCompressed = FileOutputFormat.getCompressOutput(jc); - return createSequenceWriter(jc, fs, file, keyClass, valClass, isCompressed, progressable); - } - /** * Create a sequencefile output stream based on job configuration Uses user supplied compression * flag (rather than obtaining it from the Job Configuration). @@ -1056,11 +969,11 @@ public static String getFileExtension(JobConf jc, boolean isCompressed, throws IOException { CompressionCodec codec = null; CompressionType compressionType = CompressionType.NONE; - Class codecClass = null; + Class codecClass = null; if (isCompressed) { compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc); codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); - codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc); + codec = ReflectionUtil.newInstance(codecClass, jc); } return SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType, codec, progressable); @@ -1157,7 +1070,7 @@ public static Path toTempPath(String orig) { /** * Detect if the supplied file is a temporary path. */ - public static boolean isTempPath(FileStatus file) { + private static boolean isTempPath(FileStatus file) { String name = file.getPath().getName(); // in addition to detecting hive temporary files, we also check hadoop // temporary folders that used to show up in older releases @@ -1182,46 +1095,6 @@ public static void rename(FileSystem fs, Path src, Path dst) throws IOException, } } - /** - * Moves files from src to dst if it is within the specified set of paths - * @param fs - * @param src - * @param dst - * @param filesToMove - * @throws IOException - * @throws HiveException - */ - private static void moveSpecifiedFiles(FileSystem fs, Path src, Path dst, Set filesToMove) - throws IOException, HiveException { - if (!fs.exists(dst)) { - fs.mkdirs(dst); - } - - FileStatus[] files = fs.listStatus(src); - for (FileStatus file : files) { - if (filesToMove.contains(file.getPath())) { - Utilities.moveFile(fs, file, dst); - } else if (file.isDir()) { - // Traverse directory contents. - // Directory nesting for dst needs to match src. - Path nestedDstPath = new Path(dst, file.getPath().getName()); - Utilities.moveSpecifiedFiles(fs, file.getPath(), nestedDstPath, filesToMove); - } - } - } - - public static void moveSpecifiedFiles(FileSystem fs, Path dst, Set filesToMove) - throws IOException, HiveException { - if (!fs.exists(dst)) { - fs.mkdirs(dst); - } - - for (Path path: filesToMove) { - FileStatus fsStatus = fs.getFileStatus(path); - Utilities.moveFile(fs, fsStatus, dst); - } - } - private static void moveFile(FileSystem fs, FileStatus file, Path dst) throws IOException, HiveException { Path srcFilePath = file.getPath(); @@ -1334,7 +1207,7 @@ public static String getTaskIdFromFilename(String filename) { * @param filename * filename to extract taskid from */ - public static String getPrefixedTaskIdFromFilename(String filename) { + private static String getPrefixedTaskIdFromFilename(String filename) { return getTaskIdFromFilename(filename, FILE_NAME_PREFIXED_TASK_ID_REGEX); } @@ -1342,7 +1215,7 @@ private static String getTaskIdFromFilename(String filename, Pattern pattern) { return getIdFromFilename(filename, pattern, 1); } - public static int getAttemptIdFromFilename(String filename) { + private static int getAttemptIdFromFilename(String filename) { String attemptStr = getIdFromFilename(filename, FILE_NAME_PREFIXED_TASK_ID_REGEX, 3); return Integer.parseInt(attemptStr.substring(1)); } @@ -1365,14 +1238,6 @@ private static String getIdFromFilename(String filename, Pattern pattern, int gr return taskId; } - public static String getFileNameFromDirName(String dirName) { - int dirEnd = dirName.lastIndexOf(Path.SEPARATOR); - if (dirEnd != -1) { - return dirName.substring(dirEnd + 1); - } - return dirName; - } - /** * Replace the task id from the filename. It is assumed that the filename is derived from the * output of getTaskId @@ -1482,7 +1347,7 @@ private static String replaceTaskIdFromFilename(String filename, String oldTaskI } - public static boolean shouldAvoidRename(FileSinkDesc conf, Configuration hConf) { + private static boolean shouldAvoidRename(FileSinkDesc conf, Configuration hConf) { // we are avoiding rename/move only if following conditions are met // * execution engine is tez // * query cache is disabled @@ -1648,7 +1513,7 @@ public static void removeTempOrDuplicateFiles(FileSystem fs, Path path, boolean return removeTempOrDuplicateFiles(fs, stats, dpCtx, conf, hconf, isBaseDir); } - public static List removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats, + private static List removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats, DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, boolean isBaseDir) throws IOException { return removeTempOrDuplicateFiles(fs, fileStats, dpCtx, conf, hconf, null, isBaseDir); } @@ -1658,7 +1523,7 @@ public static void removeTempOrDuplicateFiles(FileSystem fs, Path path, boolean * * @return a list of path names corresponding to should-be-created empty buckets. */ - public static List removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats, + private static List removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats, DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set filesKept, boolean isBaseDir) throws IOException { int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(), @@ -2712,7 +2577,6 @@ public boolean skipProcessing(Task task) { } } - @SuppressWarnings("unchecked") private static List getTasks(List> tasks, TaskFilterFunction function) { DAGTraversal.traverse(tasks, function); @@ -3667,7 +3531,6 @@ public static void createTmpDirs(Configuration conf, MapWork mWork) * @param rWork Used to find FileSinkOperators * @throws IOException */ - @SuppressWarnings("unchecked") public static void createTmpDirs(Configuration conf, ReduceWork rWork) throws IOException { if (rWork == null) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java index 305b467439..2d48449c9d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java @@ -101,10 +101,6 @@ public void testGetFileExtension() { getFileExtension(jc, false, new HiveIgnoreKeyTextOutputFormat())); assertEquals("Deflate for uncompressed text format", ".deflate", getFileExtension(jc, true, new HiveIgnoreKeyTextOutputFormat())); - assertEquals("No extension for uncompressed default format", "", - getFileExtension(jc, false)); - assertEquals("Deflate for uncompressed default format", ".deflate", - getFileExtension(jc, true)); String extension = ".myext"; jc.set("hive.output.file.extension", extension); @@ -349,7 +345,7 @@ public void testGetInputPathsWithMultipleThreadsAndEmptyPartitions() throws Exce String testPartitionName = "p=" + i; testPartitionsPaths[i] = new Path(testTablePath, "p=" + i); mapWork.getPathToAliases().put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName)); - mapWork.getAliasToWork().put(testPartitionName, (Operator) mock(Operator.class)); + mapWork.getAliasToWork().put(testPartitionName, mock(Operator.class)); mapWork.getPathToPartitionInfo().put(testPartitionsPaths[i], mockPartitionDesc); } @@ -501,7 +497,7 @@ private void runTestGetInputPaths(JobConf jobConf, int numOfPartitions) throws E pathToAliasTable.put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName)); - mapWork.getAliasToWork().put(testPartitionName, (Operator) mock(Operator.class)); + mapWork.getAliasToWork().put(testPartitionName, mock(Operator.class)); } mapWork.setPathToAliases(pathToAliasTable); @@ -638,6 +634,7 @@ public CountingWrappingTask(Task dep) { super.addDependentTask(wrappedDep); } + @Override public boolean addDependentTask(Task dependent) { return wrappedDep.addDependentTask(dependent); }