commit fcf2cdb6344960ec5e17fa1d35ed8b13f5f0eeb1 Author: Ivan Suller Date: Thu Jan 23 17:13:13 2020 +0100 HIVE-22768 Change-Id: I2f2645de97cd19935b1d73853c9e27815f900eb2 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index aa62e90c0e1..82fe7b0c525 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -773,14 +773,6 @@ public static Properties makeProperties(String... olist) { return (ret); } - public static ArrayList makeList(Object... olist) { - ArrayList ret = new ArrayList(); - for (Object element : olist) { - ret.add(element); - } - return (ret); - } - public static TableDesc getTableDesc(Table tbl) { Properties props = tbl.getMetadata(); props.put(serdeConstants.SERIALIZATION_LIB, tbl.getDeserializer().getClass().getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 73ca658d9c6..e27dbe38288 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1039,14 +1039,14 @@ public static TableScanOperator createTemporaryFile( // Connect parent to fileSinkOp parent.replaceChild(child, fileSinkOp); - fileSinkOp.setParentOperators(Utilities.makeList(parent)); + fileSinkOp.setParentOperators(Collections.singletonList(parent)); // Create a dummy TableScanOperator for the file generated through fileSinkOp TableScanOperator tableScanOp = createTemporaryTableScanOperator( parent.getCompilationOpContext(), parent.getSchema()); // Connect this TableScanOperator to child. - tableScanOp.setChildOperators(Utilities.makeList(child)); + tableScanOp.setChildOperators(Collections.singletonList(child)); child.replaceParent(parent, tableScanOp); return tableScanOp; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java index d2cf78bee54..c99caccd7e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java @@ -42,8 +42,6 @@ import org.apache.hadoop.hive.ql.exec.ScriptOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication.ReduceSinkDeduplicateProcCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -473,37 +471,6 @@ protected static void isNullOperator(Operator operator) throws SemanticExcept } } - /** - * @param newOperator the operator will be inserted between child and parent - * @param child - * @param parent - * @throws SemanticException - */ - protected static void insertOperatorBetween( - Operator newOperator, Operator parent, Operator child) - throws SemanticException { - isNullOperator(newOperator); - isNullOperator(parent); - isNullOperator(child); - - if (parent != getSingleParent(child)) { - throw new SemanticException("Operator " + parent.getName() + " (ID: " + - parent.getIdentifier() + ") is not the only parent of Operator " + - child.getName() + " (ID: " + child.getIdentifier() + ")"); - } - if (child != getSingleChild(parent)) { - throw new SemanticException("Operator " + child.getName() + " (ID: " + - child.getIdentifier() + ") is not the only child of Operator " + - parent.getName() + " (ID: " + parent.getIdentifier() + ")"); - } - - newOperator.setParentOperators(Utilities.makeList(parent)); - newOperator.setChildOperators(Utilities.makeList(child)); - - child.setParentOperators(Utilities.makeList(newOperator)); - parent.setChildOperators(Utilities.makeList(newOperator)); - } - public static void removeOperator(Operator target, ParseContext context) { assert target.getNumParent() == 1 && target.getNumChild() == 1; removeOperator(target, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/QueryPlanTreeTransformation.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/QueryPlanTreeTransformation.java index f8a6f4dc3a5..f348efcdd6b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/QueryPlanTreeTransformation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/QueryPlanTreeTransformation.java @@ -19,21 +19,19 @@ package org.apache.hadoop.hive.ql.optimizer.correlation; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.DemuxOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.optimizer.correlation.CorrelationOptimizer.CorrelationNodeProcCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -42,6 +40,8 @@ import org.apache.hadoop.hive.ql.plan.MuxDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * QueryPlanTreeTransformation contains static methods used to transform @@ -178,11 +178,11 @@ protected static void applyCorrelation( if (hasBottomReduceSinkOperators && hasNonBottomReduceSinkOperators) { child.setParentOperators(parentsWithMultipleDemux); } else { - child.setParentOperators(Utilities.makeList(demuxOp)); + child.setParentOperators(Collections.singletonList(demuxOp)); } } for (Operator parent: parentRSsOfDemux) { - parent.setChildOperators(Utilities.makeList(demuxOp)); + parent.setChildOperators(Collections.singletonList(demuxOp)); } // replace all ReduceSinkOperators which are not at the bottom of @@ -205,10 +205,10 @@ protected static void applyCorrelation( parentsOfMux.add(parentOp); Operator mux = OperatorFactory.get( childOP.getCompilationOpContext(), new MuxDesc(parentsOfMux)); - mux.setChildOperators(Utilities.makeList(childOP)); + mux.setChildOperators(Collections.singletonList(childOP)); mux.setParentOperators(parentsOfMux); - childOP.setParentOperators(Utilities.makeList(mux)); - parentOp.setChildOperators(Utilities.makeList(mux)); + childOP.setParentOperators(Collections.singletonList(mux)); + parentOp.setChildOperators(Collections.singletonList(mux)); } else { List> parentsOfMux = new ArrayList>(); @@ -236,7 +236,7 @@ protected static void applyCorrelation( MuxDesc muxDesc = new MuxDesc(siblingOPs); Operator mux = OperatorFactory.get( rsop.getCompilationOpContext(), muxDesc); - mux.setChildOperators(Utilities.makeList(childOP)); + mux.setChildOperators(Collections.singletonList(childOP)); mux.setParentOperators(parentsOfMux); for (Operator op: parentsOfMux) { @@ -250,10 +250,10 @@ protected static void applyCorrelation( } else { // op is not a DemuxOperator, so it should have // a single child. - op.setChildOperators(Utilities.makeList(mux)); + op.setChildOperators(Collections.singletonList(mux)); } } - childOP.setParentOperators(Utilities.makeList(mux)); + childOP.setParentOperators(Collections.singletonList(mux)); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java index e54e517904b..02c2d73d28a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSMBJoinHintOptimizer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.optimizer.spark; +import java.util.Collections; import java.util.List; import java.util.Stack; @@ -26,7 +27,6 @@ import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; @@ -80,7 +80,6 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, * In SMB join these are not expected for any parents, either from small or big tables. * @param mapJoinOp */ - @SuppressWarnings("unchecked") private void removeSmallTableReduceSink(MapJoinOperator mapJoinOp) { SMBJoinDesc smbJoinDesc = new SMBJoinDesc(mapJoinOp.getConf()); List> parentOperators = mapJoinOp.getParentOperators(); @@ -93,7 +92,7 @@ private void removeSmallTableReduceSink(MapJoinOperator mapJoinOp) { "AssertionError: expect # of parents to be 1, but was " + grandParents.size()); Operator grandParent = grandParents.get(0); grandParent.removeChild(par); - grandParent.setChildOperators(Utilities.makeList(mapJoinOp)); + grandParent.setChildOperators(Collections.singletonList(mapJoinOp)); mapJoinOp.getParentOperators().set(i, grandParent); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SplitOpTreeForDPP.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SplitOpTreeForDPP.java index b9c2c1c33ba..4a520c49c5d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SplitOpTreeForDPP.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SplitOpTreeForDPP.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.parse.spark; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -29,7 +30,6 @@ import org.apache.hadoop.hive.ql.exec.OperatorUtils; import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; @@ -133,7 +133,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Set> sinkSet = new LinkedHashSet<>(); for (Operator sel : firstNodesOfPruningBranch) { SparkUtilities.collectOp(sinkSet, sel, SparkPartitionPruningSinkOperator.class); - sel.setParentOperators(Utilities.makeList(newBranchingOp)); + sel.setParentOperators(Collections.singletonList(newBranchingOp)); } context.pruningSinkSet.addAll(sinkSet); return null; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index 83db48e7588..e0dd4f758db 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.FileInputStream; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -112,7 +114,7 @@ } } LOG.info("Directory of actual files: " + tmppath); - for (Object one : Utilities.makeList("mapplan1.out", "mapplan2.out", + for (Object one : Arrays.asList("mapplan1.out", "mapplan2.out", "mapredplan1.out", "mapredplan2.out", "mapredplan3.out", "mapredplan4.out", "mapredplan5.out", "mapredplan6.out")) { Path onedir = new Path(tmppath, (String) one); @@ -250,9 +252,9 @@ private void populateMapRedPlan1(Table src) throws SemanticException { outputColumns.add("_col" + i); } // map-side work - Operator op1 = OperatorFactory.get(ctx, PlanUtils - .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), - Utilities.makeList(getStringColumn("value")), outputColumns, true, + Operator op1 = OperatorFactory.get(ctx, + PlanUtils.getReduceSinkDesc(Collections.singletonList(getStringColumn("key")), + Collections.singletonList(getStringColumn("value")), outputColumns, true, -1, 1, -1, AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST)); addMapWork(mr, src, "a", op1); @@ -282,10 +284,9 @@ private void populateMapRedPlan2(Table src) throws Exception { outputColumns.add("_col" + i); } // map-side work - Operator op1 = OperatorFactory.get(ctx, PlanUtils - .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), - Utilities - .makeList(getStringColumn("key"), getStringColumn("value")), + Operator op1 = OperatorFactory.get(ctx, + PlanUtils.getReduceSinkDesc(Collections.singletonList(getStringColumn("key")), + Arrays.asList(getStringColumn("key"), getStringColumn("value")), outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST)); addMapWork(mr, src, "a", op1); @@ -320,15 +321,15 @@ private void populateMapRedPlan3(Table src, Table src2) throws SemanticException } // map-side work Operator op1 = OperatorFactory.get(ctx, PlanUtils - .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), - Utilities.makeList(getStringColumn("value")), outputColumns, true, + .getReduceSinkDesc(Collections.singletonList(getStringColumn("key")), + Collections.singletonList(getStringColumn("value")), outputColumns, true, Byte.valueOf((byte) 0), 1, -1, AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST)); addMapWork(mr, src, "a", op1); Operator op2 = OperatorFactory.get(ctx, PlanUtils - .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), - Utilities.makeList(getStringColumn("key")), outputColumns, true, + .getReduceSinkDesc(Collections.singletonList(getStringColumn("key")), + Collections.singletonList(getStringColumn("key")), outputColumns, true, Byte.valueOf((byte) 1), Integer.MAX_VALUE, -1, AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST)); addMapWork(mr, src2, "b", op2); @@ -345,11 +346,11 @@ private void populateMapRedPlan3(Table src, Table src2) throws SemanticException Operator op4 = OperatorFactory.get(ctx, new FileSinkDesc(new Path(TMPDIR + File.separator + "mapredplan3.out"), Utilities.defaultTd, false)); - Operator op5 = OperatorFactory.get(new SelectDesc(Utilities - .makeList(new ExprNodeFieldDesc(TypeInfoFactory.stringTypeInfo, - new ExprNodeColumnDesc(TypeInfoFactory.getListTypeInfo(TypeInfoFactory.stringTypeInfo), - Utilities.ReduceField.VALUE.toString(), "", false), "0", false)), - Utilities.makeList(outputColumns.get(0))), op4); + Operator op5 = OperatorFactory.get(new SelectDesc( + Collections.singletonList(new ExprNodeFieldDesc(TypeInfoFactory.stringTypeInfo, + new ExprNodeColumnDesc(TypeInfoFactory.getListTypeInfo(TypeInfoFactory.stringTypeInfo), + Utilities.ReduceField.VALUE.toString(), "", false), "0", false)), + Collections.singletonList(outputColumns.get(0))), op4); rWork.setReducer(op5); } @@ -362,11 +363,10 @@ private void populateMapRedPlan4(Table src) throws SemanticException { for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); } - Operator op1 = OperatorFactory.get(ctx, PlanUtils - .getReduceSinkDesc(Utilities.makeList(getStringColumn("tkey")), - Utilities.makeList(getStringColumn("tkey"), - getStringColumn("tvalue")), outputColumns, false, -1, 1, -1, - AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST)); + Operator op1 = OperatorFactory.get(ctx, + PlanUtils.getReduceSinkDesc(Collections.singletonList(getStringColumn("tkey")), + Arrays.asList(getStringColumn("tkey"), getStringColumn("tvalue")), outputColumns, false, -1, 1, + -1, AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST)); Operator op0 = OperatorFactory.get(new ScriptDesc("cat", PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key,value"), @@ -374,8 +374,8 @@ private void populateMapRedPlan4(Table src) throws SemanticException { + Utilities.tabCode, "tkey,tvalue"), TextRecordReader.class, TextRecordReader.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key")), op1); - Operator op4 = OperatorFactory.get(new SelectDesc(Utilities - .makeList(getStringColumn("key"), getStringColumn("value")), + Operator op4 = OperatorFactory.get(new SelectDesc( + Arrays.asList(getStringColumn("key"), getStringColumn("value")), outputColumns), op0); addMapWork(mr, src, "a", op4); @@ -409,12 +409,11 @@ private void populateMapRedPlan5(Table src) throws SemanticException { outputColumns.add("_col" + i); } Operator op0 = OperatorFactory.get(ctx, PlanUtils - .getReduceSinkDesc(Utilities.makeList(getStringColumn("0")), Utilities - .makeList(getStringColumn("0"), getStringColumn("1")), + .getReduceSinkDesc(Collections.singletonList(getStringColumn("0")), + Arrays.asList(getStringColumn("0"), getStringColumn("1")), outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST)); - Operator op4 = OperatorFactory.get(new SelectDesc(Utilities - .makeList(getStringColumn("key"), getStringColumn("value")), + Operator op4 = OperatorFactory.get(new SelectDesc(Arrays.asList(getStringColumn("key"), getStringColumn("value")), outputColumns), op0); addMapWork(mr, src, "a", op4); @@ -444,9 +443,8 @@ private void populateMapRedPlan6(Table src) throws Exception { outputColumns.add("_col" + i); } Operator op1 = OperatorFactory.get(ctx, PlanUtils - .getReduceSinkDesc(Utilities.makeList(getStringColumn("tkey")), - Utilities.makeList(getStringColumn("tkey"), - getStringColumn("tvalue")), outputColumns, false, -1, 1, -1, + .getReduceSinkDesc(Collections.singletonList(getStringColumn("tkey")), + Arrays.asList(getStringColumn("tkey"), getStringColumn("tvalue")), outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST)); Operator op0 = OperatorFactory.get(new ScriptDesc( @@ -456,8 +454,7 @@ private void populateMapRedPlan6(Table src) throws Exception { TextRecordReader.class, TextRecordReader.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key")), op1); - Operator op4 = OperatorFactory.get(new SelectDesc(Utilities - .makeList(getStringColumn("key"), getStringColumn("value")), + Operator op4 = OperatorFactory.get(new SelectDesc(Arrays.asList(getStringColumn("key"), getStringColumn("value")), outputColumns), op0); addMapWork(mr, src, "a", op4);