diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/MsckDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/MsckDesc.java index 32a51fe1a2..4f6f31e285 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/MsckDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/MsckDesc.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.ql.ddl.misc; import java.io.Serializable; -import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -36,19 +34,16 @@ private static final long serialVersionUID = 1L; private final String tableName; - private final ArrayList> partitionsSpecs; + private final List> partitionsSpecs; private final String resFile; private final boolean repairPartitions; private final boolean addPartitions; private final boolean dropPartitions; - public MsckDesc(String tableName, List> partitionSpecs, Path resFile, + public MsckDesc(String tableName, List> partitionsSpecs, Path resFile, boolean repairPartitions, boolean addPartitions, boolean dropPartitions) { this.tableName = tableName; - this.partitionsSpecs = new ArrayList>(partitionSpecs.size()); - for (Map partSpec : partitionSpecs) { - this.partitionsSpecs.add(new LinkedHashMap<>(partSpec)); - } + this.partitionsSpecs = partitionsSpecs; this.resFile = resFile.toString(); this.repairPartitions = repairPartitions; this.addPartitions = addPartitions; @@ -61,7 +56,7 @@ public String getTableName() { } @Explain(displayName = "partitions specs", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public ArrayList> getPartitionsSpecs() { + public List> getPartitionsSpecs() { return partitionsSpecs; } diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/MsckOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/MsckOperation.java index dea0a05fa0..5bb99eb580 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/MsckOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/MsckOperation.java @@ -51,7 +51,7 @@ public int execute() throws HiveException, IOException { String[] names = Utilities.getDbTableName(desc.getTableName()); MsckInfo msckInfo = new MsckInfo(SessionState.get().getCurrentCatalog(), names[0], names[1], desc.getPartitionsSpecs(), desc.getResFile(), desc.isRepairPartitions(), desc.isAddPartitions(), - desc.isDropPartitions(), -1); + desc.isDropPartitions(), -1L); return msck.repair(msckInfo); } catch (MetaException e) { LOG.error("Unable to create msck instance.", e); diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java index 64ce2fe550..e6dc5f8cdf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.ddl.table.storage; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -34,20 +33,25 @@ * DDL task description for ALTER TABLE ... [PARTITION ... ] CONCATENATE commands. */ @Explain(displayName = "Concatenate", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +@SuppressWarnings("rawtypes") public class AlterTableConcatenateDesc implements DDLDesc { - private String tableName; - private Map partSpec; - private ListBucketingCtx lbCtx; // context for list bucketing. - - private List inputDir = new ArrayList(); - private Path outputDir = null; - private Class inputFormatClass; - private TableDesc tableDesc; - - public AlterTableConcatenateDesc(String tableName, - Map partSpec) { + private final String tableName; + private final Map partitionSpec; + private final ListBucketingCtx lbCtx; + private final Path inputDir; + private final Path outputDir; + private final Class inputFormatClass; + private final TableDesc tableDesc; + + public AlterTableConcatenateDesc(String tableName, Map partitionSpec, ListBucketingCtx lbCtx, + Path inputDir, Path outputDir, Class inputFormatClass, TableDesc tableDesc) { this.tableName = tableName; - this.partSpec = partSpec; + this.partitionSpec = partitionSpec; + this.lbCtx = lbCtx; + this.inputDir = inputDir; + this.outputDir = outputDir; + this.inputFormatClass = inputFormatClass; + this.tableDesc = tableDesc; } @Explain(displayName = "table name") @@ -55,47 +59,28 @@ public String getTableName() { return tableName; } - @Explain(displayName = "partition desc") - public Map getPartSpec() { - return partSpec; + /** For Explain only. */ + @Explain(displayName = "partition spec") + public Map getPartitionSpec() { + return partitionSpec; } - public Path getOutputDir() { - return outputDir; - } - - public void setOutputDir(Path outputDir) { - this.outputDir = outputDir; + public ListBucketingCtx getLbCtx() { + return lbCtx; } - public List getInputDir() { + public Path getInputDir() { return inputDir; } - public void setInputDir(List inputDir) { - this.inputDir = inputDir; - } - - public ListBucketingCtx getLbCtx() { - return lbCtx; - } - - public void setLbCtx(ListBucketingCtx lbCtx) { - this.lbCtx = lbCtx; + public Path getOutputDir() { + return outputDir; } public Class getInputFormatClass() { return inputFormatClass; } - public void setInputFormatClass(Class inputFormatClass) { - this.inputFormatClass = inputFormatClass; - } - - public void setTableDesc(TableDesc tableDesc) { - this.tableDesc = tableDesc; - } - public TableDesc getTableDesc() { return tableDesc; } diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java index 0afc357b82..932508a56d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.ql.ddl.table.storage; -import java.io.Serializable; -import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -43,6 +43,8 @@ import org.apache.hadoop.hive.ql.plan.RCFileMergeDesc; import org.apache.hadoop.hive.ql.plan.TezWork; +import com.google.common.collect.Lists; + /** * Operation process of concatenating the files of a table/partition. */ @@ -53,29 +55,48 @@ public AlterTableConcatenateOperation(DDLOperationContext context, AlterTableCon @Override public int execute() throws HiveException { - ListBucketingCtx lbCtx = desc.getLbCtx(); - boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); - int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel(); + CompilationOpContext opContext = context.getDriverContext().getCtx().getOpContext(); + + MergeFileWork mergeWork = getMergeFileWork(opContext); + Task task = getTask(mergeWork); + return executeTask(opContext, task); + } + + private MergeFileWork getMergeFileWork(CompilationOpContext opContext) { + List inputDirList = Lists.newArrayList(desc.getInputDir()); // merge work only needs input and output. - MergeFileWork mergeWork = new MergeFileWork(desc.getInputDir(), desc.getOutputDir(), + MergeFileWork mergeWork = new MergeFileWork(inputDirList, desc.getOutputDir(), desc.getInputFormatClass().getName(), desc.getTableDesc()); - LinkedHashMap> pathToAliases = new LinkedHashMap<>(); - ArrayList inputDirstr = new ArrayList(1); - inputDirstr.add(desc.getInputDir().toString()); - pathToAliases.put(desc.getInputDir().get(0), inputDirstr); - mergeWork.setPathToAliases(pathToAliases); mergeWork.setListBucketingCtx(desc.getLbCtx()); mergeWork.resolveConcatenateMerge(context.getDb().getConf()); mergeWork.setMapperCannotSpanPartns(true); mergeWork.setSourceTableInputFormat(desc.getInputFormatClass().getName()); - final FileMergeDesc fmd; - if (desc.getInputFormatClass().equals(RCFileInputFormat.class)) { - fmd = new RCFileMergeDesc(); - } else { - // safe to assume else is ORC as semantic analyzer will check for RC/ORC - fmd = new OrcFileMergeDesc(); - } + + Map> pathToAliases = new LinkedHashMap<>(); + List inputDirStr = Lists.newArrayList(inputDirList.toString()); + pathToAliases.put(desc.getInputDir(), inputDirStr); + mergeWork.setPathToAliases(pathToAliases); + + FileMergeDesc fmd = getFileMergeDesc(); + Operator mergeOp = OperatorFactory.get(opContext, fmd); + LinkedHashMap> aliasToWork = + new LinkedHashMap>(); + aliasToWork.put(inputDirList.toString(), mergeOp); + mergeWork.setAliasToWork(aliasToWork); + + return mergeWork; + } + + private FileMergeDesc getFileMergeDesc() { + // safe to assume else is ORC as semantic analyzer will check for RC/ORC + FileMergeDesc fmd = (desc.getInputFormatClass().equals(RCFileInputFormat.class)) ? + new RCFileMergeDesc() : + new OrcFileMergeDesc(); + + ListBucketingCtx lbCtx = desc.getLbCtx(); + boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); + int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel(); fmd.setDpCtx(null); fmd.setHasDynamicPartitions(false); @@ -83,32 +104,30 @@ public int execute() throws HiveException { fmd.setListBucketingDepth(lbd); fmd.setOutputPath(desc.getOutputDir()); - CompilationOpContext opContext = context.getDriverContext().getCtx().getOpContext(); - Operator mergeOp = OperatorFactory.get(opContext, fmd); + return fmd; + } - LinkedHashMap> aliasToWork = - new LinkedHashMap>(); - aliasToWork.put(desc.getInputDir().toString(), mergeOp); - mergeWork.setAliasToWork(aliasToWork); - DriverContext driverCxt = new DriverContext(); - Task task; + private Task getTask(MergeFileWork mergeWork) { if (context.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { TezWork tezWork = new TezWork(context.getQueryState().getQueryId(), context.getConf()); mergeWork.setName("File Merge"); tezWork.add(mergeWork); - task = new TezTask(); + Task task = new TezTask(); ((TezTask) task).setWork(tezWork); + return task; } else { - task = new MergeFileTask(); + Task task = new MergeFileTask(); ((MergeFileTask) task).setWork(mergeWork); + return task; } + } - // initialize the task and execute + private int executeTask(CompilationOpContext opContext, Task task) { + DriverContext driverCxt = new DriverContext(); task.initialize(context.getQueryState(), context.getQueryPlan(), driverCxt, opContext); - Task subtask = task; int ret = task.execute(driverCxt); - if (subtask.getException() != null) { - context.getTask().setException(subtask.getException()); + if (task.getException() != null) { + context.getTask().setException(task.getException()); } return ret; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index 1cbc272207..c16aad81c0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -343,7 +343,7 @@ else if (partRawRowObjectInspector.equals(tblRawRowObjectInspector)) { private Map cloneConfsForNestedColPruning(Configuration hconf) { Map tableNameToConf = new HashMap<>(); - for (Map.Entry> e : conf.getPathToAliases().entrySet()) { + for (Map.Entry> e : conf.getPathToAliases().entrySet()) { List aliases = e.getValue(); if (aliases == null || aliases.isEmpty()) { continue; @@ -426,7 +426,7 @@ public void setChildren(Configuration hconf) throws Exception { Map tableNameToConf = cloneConfsForNestedColPruning(hconf); Map convertedOI = getConvertedOI(tableNameToConf); - for (Map.Entry> entry : conf.getPathToAliases().entrySet()) { + for (Map.Entry> entry : conf.getPathToAliases().entrySet()) { Path onefile = entry.getKey(); List aliases = entry.getValue(); PartitionDesc partDesc = conf.getPathToPartitionInfo().get(onefile); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index bc75ec0b56..68986dd446 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -2512,7 +2512,7 @@ public void interrupt() { final Configuration myConf = conf; final JobConf myJobConf = jobConf; final Map> aliasToWork = work.getAliasToWork(); - final Map> pathToAlias = work.getPathToAliases(); + final Map> pathToAlias = work.getPathToAliases(); final PartitionDesc partDesc = work.getPathToPartitionInfo().get(path); Runnable r = new Runnable() { @Override @@ -3348,8 +3348,7 @@ public static String getVertexCounterName(String counter, String vertexName) { LOG.info("Processing alias {}", alias); // The alias may not have any path - Collection>> pathToAliases = - work.getPathToAliases().entrySet(); + Collection>> pathToAliases = work.getPathToAliases().entrySet(); if (!skipDummy) { // ConcurrentModification otherwise if adding dummy. pathToAliases = new ArrayList<>(pathToAliases); @@ -3357,7 +3356,7 @@ public static String getVertexCounterName(String counter, String vertexName) { boolean isEmptyTable = true; boolean hasLogged = false; - for (Map.Entry> e : pathToAliases) { + for (Map.Entry> e : pathToAliases) { if (lDrvStat != null && lDrvStat.isAborted()) { throw new IOException("Operation is Canceled."); } @@ -3576,8 +3575,8 @@ private static Path createDummyFileForEmptyTable(JobConf job, MapWork work, // update the work - LinkedHashMap> pathToAliases = work.getPathToAliases(); - ArrayList newList = new ArrayList(1); + Map> pathToAliases = work.getPathToAliases(); + List newList = new ArrayList(1); newList.add(alias); pathToAliases.put(newPath, newList); @@ -3639,7 +3638,7 @@ public static void setInputAttributes(Configuration conf, MapWork mWork) { public static void createTmpDirs(Configuration conf, MapWork mWork) throws IOException { - Map> pa = mWork.getPathToAliases(); + Map> pa = mWork.getPathToAliases(); if (MapUtils.isNotEmpty(pa)) { // common case: 1 table scan per map-work // rare case: smb joins diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index f06ac37b3a..3278dfea06 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -261,7 +261,7 @@ public URI apply(Path path) { Set fileSinkUris = new HashSet(); List topNodes = new ArrayList(); - LinkedHashMap> aliasToWork = mapWork.getAliasToWork(); + Map> aliasToWork = mapWork.getAliasToWork(); for (Operator operator : aliasToWork.values()) { topNodes.add(operator); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java index 5a903d3617..308de1a443 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java @@ -28,11 +28,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.AbstractMapOperator; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -46,7 +44,6 @@ import org.apache.hadoop.hive.ql.plan.VectorPartitionDesc; import org.apache.hadoop.hive.ql.plan.VectorPartitionDesc.VectorMapOperatorReadType; import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; @@ -561,7 +558,7 @@ private void internalSetChildren(Configuration hconf) throws Exception { HashMap partitionContextMap = new HashMap(); - for (Map.Entry> entry : conf.getPathToAliases().entrySet()) { + for (Map.Entry> entry : conf.getPathToAliases().entrySet()) { Path path = entry.getKey(); PartitionDesc partDesc = conf.getPathToPartitionInfo().get(path); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java index 5f2539fefe..1844ce05a8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java @@ -334,7 +334,7 @@ public int hashCode() { Map pathToPartitionInfo) throws IOException { init(job); - Map> pathToAliases = mrwork.getPathToAliases(); + Map> pathToAliases = mrwork.getPathToAliases(); Map> aliasToWork = mrwork.getAliasToWork(); CombineFileInputFormatShim combine = ShimLoader.getHadoopShims() @@ -608,11 +608,11 @@ private void processPaths(JobConf job, CombineFileInputFormatShim combine, * @return the sampled splits */ private List sampleSplits(List splits) { - HashMap nameToSamples = mrwork.getNameToSplitSample(); + Map nameToSamples = mrwork.getNameToSplitSample(); List retLists = new ArrayList(); Map> aliasToSplitList = new HashMap>(); - Map> pathToAliases = mrwork.getPathToAliases(); - Map> pathToAliasesNoScheme = removeScheme(pathToAliases); + Map> pathToAliases = mrwork.getPathToAliases(); + Map> pathToAliasesNoScheme = removeScheme(pathToAliases); // Populate list of exclusive splits for every sampled alias // @@ -681,9 +681,9 @@ private void processPaths(JobConf job, CombineFileInputFormatShim combine, return retLists; } - Map> removeScheme(Map> pathToAliases) { - Map> result = new HashMap<>(); - for (Map.Entry > entry : pathToAliases.entrySet()) { + Map> removeScheme(Map> pathToAliases) { + Map> result = new HashMap<>(); + for (Map.Entry > entry : pathToAliases.entrySet()) { Path newKey = Path.getPathWithoutSchemeAndAuthority(entry.getKey()); StringInternUtils.internUriStringsInPath(newKey); result.put(newKey, entry.getValue()); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java index 07824c0e37..0d2eb0a0a9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -53,7 +52,7 @@ extends HiveContextAwareRecordReader { private org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(CombineHiveRecordReader.class); - private LinkedHashMap pathToPartInfo; + private Map pathToPartInfo; public CombineHiveRecordReader(InputSplit split, Configuration conf, Reporter reporter, Integer partition, RecordReader preReader) throws IOException { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java index f76b026bbc..96e7bf870b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java @@ -23,13 +23,11 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Properties; import java.util.Set; @@ -66,7 +64,6 @@ import org.apache.hadoop.mapred.TaskAttemptContext; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.util.Shell; import org.apache.hive.common.util.ReflectionUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -419,8 +416,7 @@ private static RecordUpdater getRecordUpdater(JobConf jc, return pathToPartitionInfo.get(path); } - private static boolean foundAlias(Map> pathToAliases, - Path path) { + private static boolean foundAlias(Map> pathToAliases, Path path) { List aliases = pathToAliases.get(path); if ((aliases == null) || (aliases.isEmpty())) { return false; @@ -428,8 +424,7 @@ private static boolean foundAlias(Map> pathToAliases, return true; } - private static Path getMatchingPath(Map> pathToAliases, - Path dir) { + private static Path getMatchingPath(Map> pathToAliases, Path dir) { // First find the path to be searched Path path = dir; if (foundAlias(pathToAliases, path)) { @@ -462,7 +457,7 @@ private static Path getMatchingPath(Map> pathToAliases, * @param dir The path to look for **/ public static List> doGetWorksFromPath( - Map> pathToAliases, + Map> pathToAliases, Map> aliasToWork, Path dir) { List> opList = new ArrayList>(); @@ -479,9 +474,7 @@ private static Path getMatchingPath(Map> pathToAliases, * @param pathToAliases mapping from path to aliases * @param dir The path to look for **/ - public static List doGetAliasesFromPath( - Map> pathToAliases, - Path dir) { + public static List doGetAliasesFromPath(Map> pathToAliases, Path dir) { if (pathToAliases == null) { return new ArrayList(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index 4bd4a24195..cff7e04b9a 100755 --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -878,13 +878,12 @@ protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass } ArrayList aliases = new ArrayList(); - Iterator>> iterator = this.mrwork - .getPathToAliases().entrySet().iterator(); + Iterator>> iterator = this.mrwork.getPathToAliases().entrySet().iterator(); Set splitParentPaths = null; int pathsSize = this.mrwork.getPathToAliases().entrySet().size(); while (iterator.hasNext()) { - Entry> entry = iterator.next(); + Entry> entry = iterator.next(); Path key = entry.getKey(); boolean match; if (nonNative) { @@ -914,7 +913,7 @@ protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass } } if (match) { - ArrayList list = entry.getValue(); + List list = entry.getValue(); for (String val : list) { aliases.add(val); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java index 26f7733d71..30957ca860 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/SymbolicInputFormat.java @@ -43,7 +43,7 @@ public void rework(HiveConf job, MapredWork work) throws IOException { Map pathToParts = work.getMapWork().getPathToPartitionInfo(); List toRemovePaths = new ArrayList<>(); Map toAddPathToPart = new HashMap<>(); - Map> pathToAliases = work.getMapWork().getPathToAliases(); + Map> pathToAliases = work.getMapWork().getPathToAliases(); for (Map.Entry pathPartEntry : pathToParts.entrySet()) { Path path = pathPartEntry.getKey(); @@ -62,7 +62,7 @@ public void rework(HiveConf job, MapredWork work) throws IOException { symlinks = fileSystem.listStatus(path, FileUtils.HIDDEN_FILES_PATH_FILTER); } toRemovePaths.add(path); - ArrayList aliases = pathToAliases.remove(path); + List aliases = pathToAliases.remove(path); for (FileStatus symlink : symlinks) { BufferedReader reader = null; try { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java index 3044603918..594289eda4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; @Explain(displayName = "Merge File Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) @@ -132,7 +131,7 @@ public void setHasDynamicPartitions(boolean hasDynamicPartitions) { public void resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf, Path path, TableDesc tblDesc, - ArrayList aliases, + List aliases, PartitionDesc partDesc) { super.resolveDynamicPartitionStoredAsSubDirsMerge(conf, path, tblDesc, aliases, partDesc); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java index 0444562ce7..6d525ba269 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java @@ -18,11 +18,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import org.apache.hadoop.hive.common.StringInternUtils; @@ -87,7 +85,7 @@ private void pushProjectionsAndFilters(final JobConf jobConf, final Set aliases = new HashSet(); try { - ArrayList a = HiveFileFormatUtils.getFromPathRecursively( + List a = HiveFileFormatUtils.getFromPathRecursively( mapWork.getPathToAliases(), new Path(splitPath), null, false, true); if (a != null) { aliases.addAll(a); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 3277765a7c..5d6143d6a4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -117,7 +117,6 @@ import org.apache.hadoop.hive.ql.plan.TezWork; import org.apache.hadoop.hive.ql.session.LineageState; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.InputFormat; @@ -862,8 +861,7 @@ public static void setKeyAndValueDescForTaskTree(Task ta } else if (task instanceof ExecDriver) { MapredWork work = (MapredWork) task.getWork(); work.getMapWork().deriveExplainAttributes(); - HashMap> opMap = work - .getMapWork().getAliasToWork(); + Map> opMap = work.getMapWork().getAliasToWork(); if (opMap != null && !opMap.isEmpty()) { for (Operator op : opMap.values()) { setKeyAndValueDesc(work.getReduceWork(), op); @@ -985,7 +983,7 @@ public static MapredWork getMapRedWorkFromConf(HiveConf conf) { conf.getBoolVar( HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS); work.setMapperCannotSpanPartns(mapperCannotSpanPartns); - work.setPathToAliases(new LinkedHashMap>()); + work.setPathToAliases(new LinkedHashMap>()); work.setPathToPartitionInfo(new LinkedHashMap()); work.setAliasToWork(new LinkedHashMap>()); return mrWork; @@ -1164,13 +1162,13 @@ static boolean hasBranchFinished(Object... children) { */ public static void replaceMapWork(String sourceAlias, String targetAlias, MapWork source, MapWork target) { - Map> sourcePathToAliases = source.getPathToAliases(); + Map> sourcePathToAliases = source.getPathToAliases(); Map sourcePathToPartitionInfo = source.getPathToPartitionInfo(); Map> sourceAliasToWork = source.getAliasToWork(); Map sourceAliasToPartnInfo = source.getAliasToPartnInfo(); - LinkedHashMap> targetPathToAliases = target.getPathToAliases(); - LinkedHashMap targetPathToPartitionInfo = target.getPathToPartitionInfo(); + Map> targetPathToAliases = target.getPathToAliases(); + Map targetPathToPartitionInfo = target.getPathToPartitionInfo(); Map> targetAliasToWork = target.getAliasToWork(); Map targetAliasToPartnInfo = target.getAliasToPartnInfo(); @@ -1192,8 +1190,8 @@ public static void replaceMapWork(String sourceAlias, String targetAlias, targetAliasToWork.remove(targetAlias); targetAliasToPartnInfo.remove(targetAlias); List pathsToRemove = new ArrayList<>(); - for (Entry> entry: targetPathToAliases.entrySet()) { - ArrayList aliases = entry.getValue(); + for (Entry> entry: targetPathToAliases.entrySet()) { + List aliases = entry.getValue(); aliases.remove(targetAlias); if (aliases.isEmpty()) { pathsToRemove.add(entry.getKey()); @@ -1209,8 +1207,8 @@ public static void replaceMapWork(String sourceAlias, String targetAlias, targetAliasToPartnInfo.putAll(sourceAliasToPartnInfo); targetPathToPartitionInfo.putAll(sourcePathToPartitionInfo); List pathsToAdd = new ArrayList<>(); - for (Entry> entry: sourcePathToAliases.entrySet()) { - ArrayList aliases = entry.getValue(); + for (Entry> entry: sourcePathToAliases.entrySet()) { + List aliases = entry.getValue(); if (aliases.contains(sourceAlias)) { pathsToAdd.add(entry.getKey()); } @@ -1653,7 +1651,7 @@ public static MapWork createMergeTask(FileSinkDesc fsInputDesc, Path finalName, // create the merge file work MergeFileWork work = new MergeFileWork(inputDirs, finalName, hasDynamicPartitions, tblDesc.getInputFileFormatClass().getName(), tblDesc); - LinkedHashMap> pathToAliases = new LinkedHashMap<>(); + Map> pathToAliases = new LinkedHashMap<>(); pathToAliases.put(inputDir, inputDirstr); work.setMapperCannotSpanPartns(true); work.setPathToAliases(pathToAliases); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index 1256e1ce58..5ed43c7996 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -147,14 +147,14 @@ private static void genMapJoinLocalWork(MapredWork newWork, MapJoinOperator mapJ smallTableAliasList.add(alias); // get input path and remove this alias from pathToAlias // because this file will be fetched by fetch operator - LinkedHashMap> pathToAliases = newWork.getMapWork().getPathToAliases(); + Map> pathToAliases = newWork.getMapWork().getPathToAliases(); // keep record all the input path for this alias HashSet pathSet = new HashSet<>(); HashSet emptyPath = new HashSet<>(); - for (Map.Entry> entry2 : pathToAliases.entrySet()) { + for (Map.Entry> entry2 : pathToAliases.entrySet()) { Path path = entry2.getKey(); - ArrayList list = entry2.getValue(); + List list = entry2.getValue(); if (list.contains(alias)) { // add to path set pathSet.add(path); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java index 0b5de81158..4ac256762c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.ql.optimizer.physical; import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Stack; @@ -119,8 +117,8 @@ protected void replaceTask( } public long getTotalKnownInputSize(Context context, MapWork currWork, - Map> pathToAliases, - HashMap aliasToSize) throws SemanticException { + Map> pathToAliases, + Map aliasToSize) throws SemanticException { try { // go over all the input paths, and calculate a known total size, known // size for each input alias. @@ -130,7 +128,7 @@ public long getTotalKnownInputSize(Context context, MapWork currWork, // is chosen as big table, what's the total size of left tables, which // are going to be small tables. long aliasTotalKnownInputSize = 0L; - for (Map.Entry> entry : pathToAliases.entrySet()) { + for (Map.Entry> entry : pathToAliases.entrySet()) { Path path = entry.getKey(); List aliasList = entry.getValue(); ContentSummary cs = context.getCS(path); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java index e564daf6f7..0d9d5e0169 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java @@ -241,7 +241,7 @@ private void mergeMapJoinTaskIntoItsChildMapRedTask(MapRedTask mapJoinTask, Conf String childMRAlias = childMRAliases.get(0); // Sanity check to make sure there is no alias conflict after merge. - for (Entry> entry : childMapWork.getPathToAliases().entrySet()) { + for (Entry> entry : childMapWork.getPathToAliases().entrySet()) { Path path = entry.getKey(); List aliases = entry.getValue(); @@ -392,7 +392,7 @@ public static boolean cannotConvert(long aliasKnownSize, // Must be deterministic order map for consistent q-test output across Java versions HashMap, Set> taskToAliases = new LinkedHashMap, Set>(); - HashMap> pathToAliases = currWork.getPathToAliases(); + Map> pathToAliases = currWork.getPathToAliases(); Map> aliasToWork = currWork.getAliasToWork(); // start to generate multiple map join tasks diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java index ec9813d72c..b7dd90d4fe 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java @@ -148,14 +148,14 @@ private void processAlias(MapWork work, Set tableScans) { tso.getConf().setIsMetadataOnly(true); } // group path alias according to work - Map> candidates = new HashMap<>(); + Map> candidates = new HashMap<>(); for (Path path : work.getPaths()) { - ArrayList aliasesAffected = work.getPathToAliases().get(path); + List aliasesAffected = work.getPathToAliases().get(path); if (CollectionUtils.isNotEmpty(aliasesAffected)) { candidates.put(path, aliasesAffected); } } - for (Entry> entry : candidates.entrySet()) { + for (Entry> entry : candidates.entrySet()) { processAlias(work, entry.getKey(), entry.getValue(), aliases); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java index af3edf892b..ebf17085fc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.optimizer.MapJoinProcessor; -import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin; import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin.ConditionalResolverCommonJoinCtx; @@ -77,7 +76,7 @@ private void genSMBJoinWork(MapWork currWork, SMBMapJoinOperator smbJoinOp) { Map aliasToPartitionInfo = currWork.getAliasToPartnInfo(); List removePaths = new ArrayList<>(); - for (Map.Entry> entry : currWork.getPathToAliases().entrySet()) { + for (Map.Entry> entry : currWork.getPathToAliases().entrySet()) { boolean keepPath = false; for (String alias : entry.getValue()) { if (aliasToPartitionInfo.containsKey(alias)) { @@ -260,7 +259,7 @@ private boolean isEligibleForOptimization(SMBMapJoinOperator originalSMBJoinOp) HashMap, Set> taskToAliases = new LinkedHashMap, Set>(); // Note that pathToAlias will behave as if the original plan was a join plan - HashMap> pathToAliases = currJoinWork.getMapWork().getPathToAliases(); + Map> pathToAliases = currJoinWork.getMapWork().getPathToAliases(); // generate a map join task for the big table SMBJoinDesc originalSMBJoinDesc = originalSMBJoinOp.getConf(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 52e8dcb090..b97a3a2916 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -1206,7 +1206,7 @@ private void addMapWorkRules(Map opRules, NodeProcessor np) // Eliminate MR plans with more than one TableScanOperator. - LinkedHashMap> aliasToWork = mapWork.getAliasToWork(); + Map> aliasToWork = mapWork.getAliasToWork(); if ((aliasToWork == null) || (aliasToWork.size() == 0)) { setNodeIssue("Vectorized map work requires work"); return null; @@ -1713,8 +1713,8 @@ private void setValidateInputFormatAndSchemaEvolutionExplain(MapWork mapWork, List tableDataColumnList = null; List tableDataTypeInfoList = null; - LinkedHashMap> pathToAliases = mapWork.getPathToAliases(); - LinkedHashMap pathToPartitionInfo = mapWork.getPathToPartitionInfo(); + Map> pathToAliases = mapWork.getPathToAliases(); + Map pathToPartitionInfo = mapWork.getPathToPartitionInfo(); // Remember the input file formats we validated and why. Set inputFileFormatClassNameSet = new HashSet(); @@ -1725,7 +1725,7 @@ private void setValidateInputFormatAndSchemaEvolutionExplain(MapWork mapWork, Set inputFormatSupportSet = new TreeSet(); boolean outsideLoopIsFirstPartition = true; - for (Entry> entry: pathToAliases.entrySet()) { + for (Entry> entry: pathToAliases.entrySet()) { final boolean isFirstPartition = outsideLoopIsFirstPartition; outsideLoopIsFirstPartition = false; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index bd4edb729d..4c5bfe0706 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1992,10 +1992,7 @@ private void analyzeAlterTableLocation(ASTNode ast, String tableName, Map partSpec) throws SemanticException { - AlterTableConcatenateDesc mergeDesc = new AlterTableConcatenateDesc( - tableName, partSpec); - List inputDir = new ArrayList(); Path oldTblPartLoc = null; Path newTblPartLoc = null; Table tblObj = null; @@ -2016,7 +2013,6 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); return; } - mergeDesc.setTableDesc(Utilities.getTableDesc(tblObj)); List bucketCols = null; Class inputFormatClass = null; @@ -2061,11 +2057,9 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, } // throw a HiveException for other than rcfile and orcfile. - if (!((inputFormatClass.equals(RCFileInputFormat.class) || - (inputFormatClass.equals(OrcInputFormat.class))))) { + if (!(inputFormatClass.equals(RCFileInputFormat.class) || inputFormatClass.equals(OrcInputFormat.class))) { throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_FILE_FORMAT.getMsg()); } - mergeDesc.setInputFormatClass(inputFormatClass); // throw a HiveException if the table/partition is bucketized if (bucketCols != null && bucketCols.size() > 0) { @@ -2087,19 +2081,14 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_NOT_MANAGED.getMsg()); } - inputDir.add(oldTblPartLoc); - - mergeDesc.setInputDir(inputDir); - - mergeDesc.setLbCtx(lbCtx); - addInputsOutputsAlterTable(tableName, partSpec, null, AlterTableType.MERGEFILES, false); + TableDesc tblDesc = Utilities.getTableDesc(tblObj); + Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc); + AlterTableConcatenateDesc mergeDesc = new AlterTableConcatenateDesc(tableName, partSpec, lbCtx, oldTblPartLoc, + queryTmpdir, inputFormatClass, Utilities.getTableDesc(tblObj)); DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), mergeDesc); ddlWork.setNeedLock(true); Task mergeTask = TaskFactory.get(ddlWork); - TableDesc tblDesc = Utilities.getTableDesc(tblObj); - Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc); - mergeDesc.setOutputDir(queryTmpdir); // No need to handle MM tables - unsupported path. LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc, partSpec == null ? new HashMap<>() : partSpec); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java index 41a3b00932..42637dfa77 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -97,7 +96,7 @@ public void init(QueryState queryState, LogHelper console, Hive db) { protected void setInputFormat(Task task) { if (task instanceof ExecDriver) { MapWork work = ((MapredWork) task.getWork()).getMapWork(); - HashMap> opMap = work.getAliasToWork(); + Map> opMap = work.getAliasToWork(); if (!opMap.isEmpty()) { for (Operator op : opMap.values()) { setInputFormat(work, op); @@ -141,8 +140,8 @@ private void setInputFormat(MapWork work, Operator op) { private void breakTaskTree(Task task) { if (task instanceof ExecDriver) { - HashMap> opMap = ((MapredWork) task - .getWork()).getMapWork().getAliasToWork(); + Map> opMap = + ((MapredWork) task.getWork()).getMapWork().getAliasToWork(); if (!opMap.isEmpty()) { for (Operator op : opMap.values()) { breakOperatorTree(op); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 8c3ee0c7af..5000ba47b3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -697,7 +697,7 @@ protected void setInputFormat(Task task) { for (BaseWork w: all) { if (w instanceof MapWork) { MapWork mapWork = (MapWork) w; - HashMap> opMap = mapWork.getAliasToWork(); + Map> opMap = mapWork.getAliasToWork(); if (!opMap.isEmpty()) { for (Operator op : opMap.values()) { setInputFormat(mapWork, op); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java index 5bf55020e3..6bc592546f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java @@ -530,7 +530,7 @@ protected void setInputFormat(Task task) { for (BaseWork w: all) { if (w instanceof MapWork) { MapWork mapWork = (MapWork) w; - HashMap> opMap = mapWork.getAliasToWork(); + Map> opMap = mapWork.getAliasToWork(); if (!opMap.isEmpty()) { for (Operator op : opMap.values()) { setInputFormat(mapWork, op); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java index 7c1dc45b52..cc5baeef9c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java @@ -21,11 +21,9 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,8 +51,8 @@ private static final long serialVersionUID = 1L; private HashMap, Set> taskToAliases; - HashMap> pathToAliases; - HashMap aliasToKnownSize; + Map> pathToAliases; + Map aliasToKnownSize; private Task commonJoinTask; private Path localTmpDir; @@ -79,7 +77,7 @@ public void setCommonJoinTask(Task commonJoinTask) { this.commonJoinTask = commonJoinTask; } - public HashMap getAliasToKnownSize() { + public Map getAliasToKnownSize() { return aliasToKnownSize == null ? aliasToKnownSize = new HashMap() : aliasToKnownSize; } @@ -88,11 +86,11 @@ public void setAliasToKnownSize(HashMap aliasToKnownSize) { this.aliasToKnownSize = aliasToKnownSize; } - public HashMap> getPathToAliases() { + public Map> getPathToAliases() { return pathToAliases; } - public void setPathToAliases(final HashMap> pathToAliases) { + public void setPathToAliases(Map> pathToAliases) { this.pathToAliases = pathToAliases; } @@ -214,10 +212,10 @@ protected void resolveUnknownSizes(ConditionalResolverCommonJoinCtx ctx, HiveCon Set aliases = getParticipants(ctx); Map aliasToKnownSize = ctx.getAliasToKnownSize(); - Map> pathToAliases = ctx.getPathToAliases(); + Map> pathToAliases = ctx.getPathToAliases(); Set unknownPaths = new HashSet<>(); - for (Map.Entry> entry : pathToAliases.entrySet()) { + for (Map.Entry> entry : pathToAliases.entrySet()) { for (String alias : entry.getValue()) { if (aliases.contains(alias) && !aliasToKnownSize.containsKey(alias)) { unknownPaths.add(entry.getKey()); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java index 54c9659c76..a828809349 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java @@ -248,10 +248,10 @@ private void generateActualTasks(HiveConf conf, List> pta = work.getPathToAliases(); + Map> pta = work.getPathToAliases(); assert pta.size() == 1; path = pta.keySet().iterator().next(); - ArrayList aliases = pta.get(path); + List aliases = pta.get(path); work.removePathToAlias(path); // the root path is not useful anymore // populate pathToPartitionInfo and pathToAliases w/ DP paths diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index bb063c52be..ca271fc093 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -94,15 +94,15 @@ // use LinkedHashMap to make sure the iteration order is // deterministic, to ease testing - private LinkedHashMap> pathToAliases = new LinkedHashMap<>(); + private Map> pathToAliases = new LinkedHashMap<>(); - private LinkedHashMap pathToPartitionInfo = new LinkedHashMap<>(); + private Map pathToPartitionInfo = new LinkedHashMap<>(); - private LinkedHashMap> aliasToWork = new LinkedHashMap>(); + private Map> aliasToWork = new LinkedHashMap>(); - private LinkedHashMap aliasToPartnInfo = new LinkedHashMap(); + private Map aliasToPartnInfo = new LinkedHashMap(); - private HashMap nameToSplitSample = new LinkedHashMap(); + private Map nameToSplitSample = new LinkedHashMap(); // If this map task has a FileSinkOperator, and bucketing/sorting metadata can be // inferred about the data being written by that operator, these are mappings from the directory @@ -181,25 +181,25 @@ public MapWork(String name) { } @Explain(displayName = "Path -> Alias", explainLevels = { Level.EXTENDED }) - public LinkedHashMap> getPathToAliases() { + public Map> getPathToAliases() { // return pathToAliases; } - public void setPathToAliases(final LinkedHashMap> pathToAliases) { + public void setPathToAliases(Map> pathToAliases) { for (Path p : pathToAliases.keySet()) { StringInternUtils.internUriStringsInPath(p); } this.pathToAliases = pathToAliases; } - public void addPathToAlias(Path path, ArrayList aliases){ + public void addPathToAlias(Path path, List aliases){ StringInternUtils.internUriStringsInPath(path); pathToAliases.put(path, aliases); } public void addPathToAlias(Path path, String newAlias){ - ArrayList aliases = pathToAliases.get(path); + List aliases = pathToAliases.get(path); if (aliases == null) { aliases = new ArrayList<>(1); StringInternUtils.internUriStringsInPath(path); @@ -225,26 +225,25 @@ public void removePathToAlias(Path path){ * @return */ @Explain(displayName = "Truncated Path -> Alias", explainLevels = { Level.EXTENDED }) - public Map> getTruncatedPathToAliases() { - Map> trunPathToAliases = new LinkedHashMap>(); - Iterator>> itr = this.pathToAliases.entrySet().iterator(); + public Map> getTruncatedPathToAliases() { + Map> trunPathToAliases = new LinkedHashMap>(); + Iterator>> itr = this.pathToAliases.entrySet().iterator(); while (itr.hasNext()) { - final Entry> entry = itr.next(); + Entry> entry = itr.next(); Path origiKey = entry.getKey(); String newKey = PlanUtils.removePrefixFromWarehouseConfig(origiKey.toString()); - ArrayList value = entry.getValue(); + List value = entry.getValue(); trunPathToAliases.put(newKey, value); } return trunPathToAliases; } @Explain(displayName = "Path -> Partition", explainLevels = { Level.EXTENDED }) - public LinkedHashMap getPathToPartitionInfo() { + public Map getPathToPartitionInfo() { return pathToPartitionInfo; } - public void setPathToPartitionInfo(final LinkedHashMap pathToPartitionInfo) { + public void setPathToPartitionInfo(final Map pathToPartitionInfo) { for (Path p : pathToPartitionInfo.keySet()) { StringInternUtils.internUriStringsInPath(p); } @@ -364,7 +363,7 @@ public void internTable(Interner interner) { /** * @return the aliasToPartnInfo */ - public LinkedHashMap getAliasToPartnInfo() { + public Map getAliasToPartnInfo() { return aliasToPartnInfo; } @@ -377,17 +376,16 @@ public void setAliasToPartnInfo( this.aliasToPartnInfo = aliasToPartnInfo; } - public LinkedHashMap> getAliasToWork() { + public Map> getAliasToWork() { return aliasToWork; } - public void setAliasToWork( - final LinkedHashMap> aliasToWork) { + public void setAliasToWork(Map> aliasToWork) { this.aliasToWork = aliasToWork; } @Explain(displayName = "Split Sample", explainLevels = { Level.EXTENDED }) - public HashMap getNameToSplitSample() { + public Map getNameToSplitSample() { return nameToSplitSample; } @@ -417,7 +415,7 @@ public void setNumMapTasks(Integer numMapTasks) { public void addMapWork(Path path, String alias, Operator work, PartitionDesc pd) { StringInternUtils.internUriStringsInPath(path); - ArrayList curAliases = pathToAliases.get(path); + List curAliases = pathToAliases.get(path); if (curAliases == null) { assert (pathToPartitionInfo.get(path) == null); curAliases = new ArrayList<>(); @@ -450,7 +448,7 @@ public void setInputFormatSorted(boolean inputFormatSorted) { } public void resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf, Path path, - TableDesc tblDesc, ArrayList aliases, PartitionDesc partDesc) { + TableDesc tblDesc, List aliases, PartitionDesc partDesc) { StringInternUtils.internUriStringsInPath(path); pathToAliases.put(path, aliases); pathToPartitionInfo.put(path, partDesc); @@ -522,7 +520,7 @@ public void replaceRoots(Map, Operator> replacementMap) { public void mergeAliasedInput(String alias, Path pathDir, PartitionDesc partitionInfo) { StringInternUtils.internUriStringsInPath(pathDir); alias = alias.intern(); - ArrayList aliases = pathToAliases.get(pathDir); + List aliases = pathToAliases.get(pathDir); if (aliases == null) { aliases = new ArrayList<>(Arrays.asList(alias)); pathToAliases.put(pathDir, aliases); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestGetInputSummary.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestGetInputSummary.java index a946b4fbed..5f55cebb98 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestGetInputSummary.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestGetInputSummary.java @@ -26,12 +26,12 @@ import static org.mockito.Mockito.when; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -382,10 +382,8 @@ private ContentSummary runTestGetInputSummary(JobConf jobConf, context.addCS(partitionPath.toString(), entry.getValue()); } - LinkedHashMap pathToPartitionInfo = - new LinkedHashMap<>(); - LinkedHashMap> pathToAliasTable = - new LinkedHashMap<>(); + Map pathToPartitionInfo = new LinkedHashMap<>(); + Map> pathToAliasTable = new LinkedHashMap<>(); TableScanOperator scanOp = new TableScanOperator(); PartitionDesc partitionDesc = new PartitionDesc( diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java index c7cd4ad3f6..a363b22aae 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java @@ -299,10 +299,10 @@ public void testMapOperator() throws Throwable { new Path("hdfs:///testDir/testFile")); // initialize pathToAliases - ArrayList aliases = new ArrayList(); + List aliases = new ArrayList(); aliases.add("a"); aliases.add("b"); - LinkedHashMap> pathToAliases = new LinkedHashMap<>(); + Map> pathToAliases = new LinkedHashMap<>(); pathToAliases.put(new Path("hdfs:///testDir"), aliases); // initialize pathToTableInfo diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java index 3aaf56145f..d8e4347440 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java @@ -22,6 +22,8 @@ import java.io.File; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -64,7 +66,7 @@ public void testPlan() throws Exception { ArrayList aliasList = new ArrayList(); aliasList.add("a"); - LinkedHashMap> pa = new LinkedHashMap<>(); + Map> pa = new LinkedHashMap<>(); pa.put(new Path("/tmp/testfolder"), aliasList); TableDesc tblDesc = Utilities.defaultTd; diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java index 305b467439..dc851c0f24 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java @@ -370,7 +370,7 @@ public void testGetInputPathsWithMultipleThreadsAndEmptyPartitions() throws Exce assertEquals(mapWork.getPathToPartitionInfo().size(), numPartitions); assertEquals(mapWork.getAliasToWork().size(), numPartitions); - for (Map.Entry> entry : mapWork.getPathToAliases().entrySet()) { + for (Map.Entry> entry : mapWork.getPathToAliases().entrySet()) { assertNotNull(entry.getKey()); assertNotNull(entry.getValue()); assertEquals(entry.getValue().size(), 1); @@ -489,7 +489,7 @@ private void runTestGetInputPaths(JobConf jobConf, int numOfPartitions) throws E MapWork mapWork = new MapWork(); Path scratchDir = new Path(HiveConf.getVar(jobConf, HiveConf.ConfVars.LOCALSCRATCHDIR)); - LinkedHashMap> pathToAliasTable = new LinkedHashMap<>(); + Map> pathToAliasTable = new LinkedHashMap<>(); String testTableName = "testTable"; diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java index b67aec371d..befeb4f2dd 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java @@ -133,14 +133,14 @@ public Edge answer(InvocationOnMock invocation) throws Throwable { op = mock(Operator.class); - LinkedHashMap> map + Map> map = new LinkedHashMap>(); map.put("foo", op); mws[0].setAliasToWork(map); mws[1].setAliasToWork(map); - LinkedHashMap> pathMap = new LinkedHashMap<>(); - ArrayList aliasList = new ArrayList(); + Map> pathMap = new LinkedHashMap<>(); + List aliasList = new ArrayList(); aliasList.add("foo"); pathMap.put(new Path("foo"), aliasList); diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index 9a8ae3b0a0..b5958fa9cc 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -2201,8 +2201,8 @@ JobConf createMockExecutionEnvironment(Path workDir, mapWork.setVectorizedRowBatchCtx(vectorizedRowBatchCtx); } mapWork.setUseBucketizedHiveInputFormat(false); - LinkedHashMap> aliasMap = new LinkedHashMap<>(); - ArrayList aliases = new ArrayList(); + Map> aliasMap = new LinkedHashMap<>(); + List aliases = new ArrayList(); aliases.add(tableName); LinkedHashMap partMap = new LinkedHashMap<>(); for(int p=0; p < partitions; ++p) { diff --git ql/src/test/org/apache/hadoop/hive/ql/plan/TestConditionalResolverCommonJoin.java ql/src/test/org/apache/hadoop/hive/ql/plan/TestConditionalResolverCommonJoin.java index 3a8b5e7331..3fc82adcf8 100644 --- ql/src/test/org/apache/hadoop/hive/ql/plan/TestConditionalResolverCommonJoin.java +++ ql/src/test/org/apache/hadoop/hive/ql/plan/TestConditionalResolverCommonJoin.java @@ -32,6 +32,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.Set; public class TestConditionalResolverCommonJoin { @@ -40,7 +42,7 @@ public void testResolvingDriverAlias() throws Exception { ConditionalResolverCommonJoin resolver = new ConditionalResolverCommonJoin(); - HashMap> pathToAliases = new HashMap<>(); + Map> pathToAliases = new HashMap<>(); pathToAliases.put(new Path("path1"), new ArrayList(Arrays.asList("alias1", "alias2"))); pathToAliases.put(new Path("path2"), new ArrayList(Arrays.asList("alias3"))); diff --git ql/src/test/org/apache/hadoop/hive/ql/plan/TestMapWork.java ql/src/test/org/apache/hadoop/hive/ql/plan/TestMapWork.java index 1756711db6..3e0d834175 100644 --- ql/src/test/org/apache/hadoop/hive/ql/plan/TestMapWork.java +++ ql/src/test/org/apache/hadoop/hive/ql/plan/TestMapWork.java @@ -19,8 +19,9 @@ import static org.junit.Assert.assertEquals; -import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.apache.hadoop.fs.Path; import org.junit.Test; @@ -31,11 +32,11 @@ @Test public void testGetAndSetConsistency() { MapWork mw = new MapWork(); - LinkedHashMap> pathToAliases = new LinkedHashMap<>(); + Map> pathToAliases = new LinkedHashMap<>(); pathToAliases.put(new Path("p0"), Lists.newArrayList("a1", "a2")); mw.setPathToAliases(pathToAliases); - LinkedHashMap> pta = mw.getPathToAliases(); + Map> pta = mw.getPathToAliases(); assertEquals(pathToAliases, pta); } diff --git ql/src/test/results/clientpositive/llap/orc_merge10.q.out ql/src/test/results/clientpositive/llap/orc_merge10.q.out index 3af8190e6a..7b69b39cd3 100644 --- ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -632,7 +632,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: ds 1 part 0 table name: default.orcfile_merge1 diff --git ql/src/test/results/clientpositive/llap/orc_merge6.q.out ql/src/test/results/clientpositive/llap/orc_merge6.q.out index 95cbe1ec58..7021220588 100644 --- ql/src/test/results/clientpositive/llap/orc_merge6.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge6.q.out @@ -522,7 +522,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: hour 24 year 2000 table name: default.orc_merge5a_n1 diff --git ql/src/test/results/clientpositive/llap/orc_merge7.q.out ql/src/test/results/clientpositive/llap/orc_merge7.q.out index 4155dc6123..1f7b7b7af1 100644 --- ql/src/test/results/clientpositive/llap/orc_merge7.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge7.q.out @@ -653,7 +653,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: st 80.0 table name: default.orc_merge5a_n0 diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out index 48217cd44d..207541b135 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out @@ -338,7 +338,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: st 80.0 table name: default.orc_merge5a diff --git ql/src/test/results/clientpositive/orc_merge10.q.out ql/src/test/results/clientpositive/orc_merge10.q.out index e8ebd4e717..1f70773226 100644 --- ql/src/test/results/clientpositive/orc_merge10.q.out +++ ql/src/test/results/clientpositive/orc_merge10.q.out @@ -603,7 +603,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: ds 1 part 0 table name: default.orcfile_merge1 diff --git ql/src/test/results/clientpositive/orc_merge6.q.out ql/src/test/results/clientpositive/orc_merge6.q.out index a07cfd3cb4..bc05d2fadf 100644 --- ql/src/test/results/clientpositive/orc_merge6.q.out +++ ql/src/test/results/clientpositive/orc_merge6.q.out @@ -486,7 +486,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: hour 24 year 2000 table name: default.orc_merge5a_n1 diff --git ql/src/test/results/clientpositive/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/orc_merge_incompat2.q.out index 71ae287285..177bc25ed1 100644 --- ql/src/test/results/clientpositive/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/orc_merge_incompat2.q.out @@ -333,7 +333,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: st 80.0 table name: default.orc_merge5a diff --git ql/src/test/results/clientpositive/spark/orc_merge6.q.out ql/src/test/results/clientpositive/spark/orc_merge6.q.out index 982b614e34..b3d1ca405d 100644 --- ql/src/test/results/clientpositive/spark/orc_merge6.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge6.q.out @@ -422,7 +422,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: hour 24 year 2000 table name: default.orc_merge5a_n1 diff --git ql/src/test/results/clientpositive/spark/orc_merge7.q.out ql/src/test/results/clientpositive/spark/orc_merge7.q.out index a641ed73b7..0c2b8a0209 100644 --- ql/src/test/results/clientpositive/spark/orc_merge7.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge7.q.out @@ -553,7 +553,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: st 80.0 table name: default.orc_merge5a_n0 diff --git ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out index 1bfbf4d8df..8da08bf64a 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out @@ -294,7 +294,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Concatenate - partition desc: + partition spec: st 80.0 table name: default.orc_merge5a diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java index c2ba3b0cc1..871cff20cb 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java @@ -108,15 +108,16 @@ public int repair(MsckInfo msckInfo) { boolean success = false; long txnId = -1; int ret = 0; + long partitionExpirySeconds = msckInfo.getPartitionExpirySeconds(); try { Table table = getMsc().getTable(msckInfo.getCatalogName(), msckInfo.getDbName(), msckInfo.getTableName()); qualifiedTableName = Warehouse.getCatalogQualifiedTableName(table); if (getConf().getBoolean(MetastoreConf.ConfVars.MSCK_REPAIR_ENABLE_PARTITION_RETENTION.getHiveName(), false)) { - msckInfo.setPartitionExpirySeconds(PartitionManagementTask.getRetentionPeriodInSeconds(table)); + partitionExpirySeconds = PartitionManagementTask.getRetentionPeriodInSeconds(table); LOG.info("{} - Retention period ({}s) for partition is enabled for MSCK REPAIR..", - qualifiedTableName, msckInfo.getPartitionExpirySeconds()); + qualifiedTableName, partitionExpirySeconds); } - HiveMetaStoreChecker checker = new HiveMetaStoreChecker(getMsc(), getConf(), msckInfo.getPartitionExpirySeconds()); + HiveMetaStoreChecker checker = new HiveMetaStoreChecker(getMsc(), getConf(), partitionExpirySeconds); // checkMetastore call will fill in result with partitions that are present in filesystem // and missing in metastore - accessed through getPartitionsNotInMs // And partitions that are not present in filesystem and metadata exists in metastore - @@ -253,7 +254,7 @@ public int repair(MsckInfo msckInfo) { firstWritten |= writeMsckResult(result.getPartitionsNotOnFs(), "Partitions missing from filesystem:", resultOut, firstWritten); firstWritten |= writeMsckResult(result.getExpiredPartitions(), - "Expired partitions (retention period: " + msckInfo.getPartitionExpirySeconds() + "s) :", resultOut, firstWritten); + "Expired partitions (retention period: " + partitionExpirySeconds + "s) :", resultOut, firstWritten); // sorting to stabilize qfile output (msck_repair_drop.q) Collections.sort(repairOutput); for (String rout : repairOutput) { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MsckInfo.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MsckInfo.java index 81bcb56b1d..25d0c648ae 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MsckInfo.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MsckInfo.java @@ -17,29 +17,27 @@ */ package org.apache.hadoop.hive.metastore; -import java.util.ArrayList; -import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; /** * Metadata related to Msck. */ public class MsckInfo { - private String catalogName; - private String dbName; - private String tableName; - private ArrayList> partSpecs; - private String resFile; - private boolean repairPartitions; - private boolean addPartitions; - private boolean dropPartitions; - private long partitionExpirySeconds; - - public MsckInfo(final String catalogName, final String dbName, final String tableName, - final ArrayList> partSpecs, final String resFile, final boolean repairPartitions, - final boolean addPartitions, - final boolean dropPartitions, - final long partitionExpirySeconds) { + private final String catalogName; + private final String dbName; + private final String tableName; + private final List> partSpecs; + private final String resFile; + private final boolean repairPartitions; + private final boolean addPartitions; + private final boolean dropPartitions; + private final long partitionExpirySeconds; + + public MsckInfo(String catalogName, String dbName, String tableName, List> partSpecs, + String resFile, boolean repairPartitions, boolean addPartitions, boolean dropPartitions, + long partitionExpirySeconds) { this.catalogName = catalogName; this.dbName = dbName; this.tableName = tableName; @@ -55,71 +53,35 @@ public String getCatalogName() { return catalogName; } - public void setCatalogName(final String catalogName) { - this.catalogName = catalogName; - } - public String getDbName() { return dbName; } - public void setDbName(final String dbName) { - this.dbName = dbName; - } - public String getTableName() { return tableName; } - public void setTableName(final String tableName) { - this.tableName = tableName; - } - - public ArrayList> getPartSpecs() { + public List> getPartSpecs() { return partSpecs; } - public void setPartSpecs(final ArrayList> partSpecs) { - this.partSpecs = partSpecs; - } - public String getResFile() { return resFile; } - public void setResFile(final String resFile) { - this.resFile = resFile; - } - public boolean isRepairPartitions() { return repairPartitions; } - public void setRepairPartitions(final boolean repairPartitions) { - this.repairPartitions = repairPartitions; - } - public boolean isAddPartitions() { return addPartitions; } - public void setAddPartitions(final boolean addPartitions) { - this.addPartitions = addPartitions; - } - public boolean isDropPartitions() { return dropPartitions; } - public void setDropPartitions(final boolean dropPartitions) { - this.dropPartitions = dropPartitions; - } - public long getPartitionExpirySeconds() { return partitionExpirySeconds; } - - public void setPartitionExpirySeconds(final long partitionExpirySeconds) { - this.partitionExpirySeconds = partitionExpirySeconds; - } }