diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index c3184a8931..9c31f78841 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -354,12 +354,7 @@ public void resetCalciteConfiguration() { @SuppressWarnings("nls") public void analyzeInternal(ASTNode ast) throws SemanticException { if (runCBO) { - super.analyzeInternal(ast, new PlannerContextFactory() { - @Override - public PlannerContext create() { - return new PreCboCtx(); - } - }); + super.analyzeInternal(ast, PreCboCtx::new); } else { super.analyzeInternal(ast); } @@ -1786,8 +1781,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu try { calciteGenPlan = genLogicalPlan(getQB(), true, null, null); // if it is to create view, we do not use table alias - resultSchema = SemanticAnalyzer.convertRowSchemaToResultSetSchema( - relToHiveRR.get(calciteGenPlan), + resultSchema = convertRowSchemaToResultSetSchema(relToHiveRR.get(calciteGenPlan), getQB().isView() || getQB().isMaterializedView() ? false : HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_RESULTSET_USE_UNIQUE_COLUMN_NAMES)); } catch (SemanticException e) { @@ -2916,8 +2910,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc fields.get(i).getFieldName(), TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector()), tableAlias, false); - colInfo.setSkewedCol((SemanticAnalyzer.isSkewedCol(tableAlias, qb, colName)) ? true - : false); + colInfo.setSkewedCol(isSkewedCol(tableAlias, qb, colName)); rr.put(tableAlias, colName, colInfo); cInfoLst.add(colInfo); } @@ -3272,7 +3265,7 @@ private RelNode genLateralViewPlans(ASTNode lateralView, Map al // Output types. They will be the concatenation of the input refs types and // the types of the expressions for the lateral view generated rows // Generate all expressions from lateral view - ExprNodeDesc valuesExpr = genExprNodeDesc(valuesClause, inputRR, false); + ExprNodeDesc valuesExpr = genExprNodeDesc(valuesClause, inputRR, false, false); RexCall convertedOriginalValuesExpr = (RexCall) new RexNodeConverter(this.cluster, inputRel.getRowType(), inputPosMap, 0, false).convert(valuesExpr); RelDataType valuesRowType = ((ArraySqlType) convertedOriginalValuesExpr.getType()).getComponentType(); @@ -3751,7 +3744,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // As we said before, here we use genSelectLogicalPlan to rewrite AllColRef srcRel = genSelectLogicalPlan(qb, srcRel, srcRel, null, null, true).getKey(); RowResolver rr = this.relToHiveRR.get(srcRel); - qbp.setSelExprForClause(detsClauseName, SemanticAnalyzer.genSelectDIAST(rr)); + qbp.setSelExprForClause(detsClauseName, genSelectDIAST(rr)); } } @@ -4599,7 +4592,7 @@ private void setQueryHints(QB qb) throws SemanticException { } else { // 6.3 Get rid of TOK_SELEXPR expr = (ASTNode) child.getChild(0); - String[] colRef = SemanticAnalyzer.getColAlias(child, getAutogenColAliasPrfxLbl(), + String[] colRef = getColAlias(child, getAutogenColAliasPrfxLbl(), inputRR, autogenColAliasPrfxIncludeFuncName(), i); tabAlias = colRef[0]; colAlias = colRef[1]; @@ -4636,7 +4629,7 @@ private void setQueryHints(QB qb) throws SemanticException { } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() - && SemanticAnalyzer.isRegex( + && isRegex( unescapeIdentifier(expr.getChild(0).getText()), conf)) { // In case the expression is a regex COL. // This can only happen without AS clause @@ -4649,7 +4642,7 @@ private void setQueryHints(QB qb) throws SemanticException { .getChild(0).getText().toLowerCase())) && !hasAsClause && !inputRR.getIsExprResolver() - && SemanticAnalyzer.isRegex( + && isRegex( unescapeIdentifier(expr.getChild(1).getText()), conf)) { // In case the expression is TABLE.COL (col can be regex). // This can only happen without AS clause diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 3313766ee4..4a54dae68e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse; +import static java.util.Objects.nonNull; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS; import com.google.common.collect.ArrayListMultimap; @@ -46,10 +47,12 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; +import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; +import com.google.common.collect.Lists; import org.antlr.runtime.ClassicToken; import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; @@ -57,8 +60,6 @@ import org.antlr.runtime.tree.Tree; import org.antlr.runtime.tree.TreeVisitor; import org.antlr.runtime.tree.TreeVisitorAction; -import org.antlr.runtime.tree.TreeWizard; -import org.antlr.runtime.tree.TreeWizard.ContextVisitor; import org.apache.calcite.rel.RelNode; import org.apache.calcite.util.ImmutableBitSet; import org.apache.commons.collections.CollectionUtils; @@ -340,7 +341,7 @@ protected Map prunedPartitions; protected List resultSchema; protected CreateViewDesc createVwDesc; - private MaterializedViewUpdateDesc materializedViewUpdateDesc; + private MaterializedViewUpdateDesc materializedViewUpdateDesc; private List viewsExpanded; protected ASTNode viewSelect; protected final UnparseTranslator unparseTranslator; @@ -398,7 +399,7 @@ Map tabNameToTabObject; // The tokens we should ignore when we are trying to do table masking. - private final Set ignoredTokens = Sets.newHashSet(HiveParser.TOK_GROUPBY, + private static final Set IGNORED_TOKENS = Sets.newHashSet(HiveParser.TOK_GROUPBY, HiveParser.TOK_ORDERBY, HiveParser.TOK_WINDOWSPEC, HiveParser.TOK_CLUSTERBY, HiveParser.TOK_DISTRIBUTEBY, HiveParser.TOK_SORTBY); @@ -552,7 +553,7 @@ public CompilationOpContext getOpContext() { return ctx.getOpContext(); } - static String genPartValueString(String partColType, String partVal) throws SemanticException { + static String genPartValueString(String partColType, String partVal) { String returnVal = partVal; if (partColType.equals(serdeConstants.STRING_TYPE_NAME) || partColType.contains(serdeConstants.VARCHAR_TYPE_NAME) || @@ -689,10 +690,7 @@ private boolean isInsertInto(QBParseInfo qbp, String dest) { return false; } ASTNode destNode = qbp.getDestForClause(dest); - if(destNode != null && destNode.getType() == HiveParser.TOK_TAB) { - return true; - } - return false; + return destNode != null && destNode.getType() == HiveParser.TOK_TAB; } /** @@ -733,7 +731,7 @@ private boolean isValueClause(ASTNode select) { * @return List of default constraints (including NULL if there is no default) * @throws SemanticException */ - private static List getDefaultConstraints(Table tbl, List targetSchema) throws SemanticException{ + private List getDefaultConstraints(Table tbl, List targetSchema) throws SemanticException{ Map colNameToDefaultVal = getColNameToDefaultValueMap(tbl); List defaultConstraints = new ArrayList<>(); if(targetSchema != null) { @@ -749,7 +747,7 @@ private boolean isValueClause(ASTNode select) { return defaultConstraints; } - protected static Map getColNameToDefaultValueMap(Table tbl) throws SemanticException { + protected Map getColNameToDefaultValueMap(Table tbl) throws SemanticException { Map colNameToDefaultVal = null; try { DefaultConstraint dc = Hive.get().getEnabledDefaultConstraints(tbl.getDbName(), tbl.getTableName()); @@ -810,7 +808,8 @@ else if (selectExpr.getChild(0).getChild(0).getText().toLowerCase().equals("defa // replace the node in place selectExpr.replaceChildren(0, 0, newNode); if (LOG.isDebugEnabled()) { - LOG.debug("DEFAULT keyword replacement - Inserted " + newNode.getText() + " for table: " + targetTable.getTableName()); + LOG.debug("DEFAULT keyword replacement - Inserted {} for table: {}", newNode.getText(), + targetTable.getTableName()); } } } @@ -838,9 +837,8 @@ private void replaceDefaultKeyword(ASTNode valueArrClause, Table targetTable, Li ASTNode newNode = getNodeReplacementforDefault(defaultConstraints.get(j-1)); // replace the node in place valueClause.replaceChildren(j, j, newNode); - if (LOG.isDebugEnabled()) { - LOG.debug("DEFAULT keyword replacement - Inserted " + newNode.getText() + " for table: " + targetTable.getTableName()); - } + LOG.debug("DEFAULT keyword replacement - Inserted {} for table: {}", newNode.getText(), + targetTable.getTableName()); } } } @@ -945,8 +943,7 @@ private void doPhase1GetAllAggregations(ASTNode expressionTree, } } - private List doPhase1GetDistinctFuncExprs( - Map aggregationTrees) throws SemanticException { + private List doPhase1GetDistinctFuncExprs(Map aggregationTrees) { List exprs = new ArrayList(); for (Map.Entry entry : aggregationTrees.entrySet()) { ASTNode value = entry.getValue(); @@ -1092,7 +1089,7 @@ private String processTable(QB qb, ASTNode tabref) throws SemanticException { SplitSample sample; if (type.getType() == HiveParser.TOK_PERCENT) { assertCombineInputFormat(numerator, "Percentage"); - Double percent = Double.valueOf(value).doubleValue(); + double percent = Double.valueOf(value); if (percent < 0 || percent > 100) { throw new SemanticException(generateErrorMessage((ASTNode) numerator, "Sampling percentage should be between 0 and 100")); @@ -1334,8 +1331,7 @@ public String toString() { List> cteLeafs = new ArrayList<>(); List> curTopRoots = null; List> curBottomLeafs = null; - for (int i = 0; i < execution.size(); i++) { - CTEClause current = execution.get(i); + for (CTEClause current : execution) { if (current.parents.isEmpty() && curTopRoots != null) { cteRoots.addAll(curTopRoots); cteLeafs.addAll(curBottomLeafs); @@ -1409,7 +1405,7 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException { } table.setMaterializedTable(true); - LOG.info(cteName + " will be materialized into " + location); + LOG.info("{} will be materialized into {}", cteName, location); cte.source = analyzer; ctx.addMaterializedTable(cteName, table); @@ -1419,15 +1415,11 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException { static boolean isJoinToken(ASTNode node) { - if ((node.getToken().getType() == HiveParser.TOK_JOIN) + return (node.getToken().getType() == HiveParser.TOK_JOIN) || (node.getToken().getType() == HiveParser.TOK_CROSSJOIN) || isOuterJoinToken(node) || (node.getToken().getType() == HiveParser.TOK_LEFTSEMIJOIN) - || (node.getToken().getType() == HiveParser.TOK_UNIQUEJOIN)) { - return true; - } - - return false; + || (node.getToken().getType() == HiveParser.TOK_UNIQUEJOIN); } static private boolean isOuterJoinToken(ASTNode node) { @@ -1837,7 +1829,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx) phase1Result = false; skipRecursion = true; LOG.info("Partition already exists so insert into overwrite " + - "skipped for partition : " + parMetaData.toString()); + "skipped for partition : {}", parMetaData); break; } } catch (HiveException e) { @@ -1869,8 +1861,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx) int child_count = ast.getChildCount(); for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) { // Recurse - phase1Result = phase1Result && doPhase1( - (ASTNode)ast.getChild(child_pos), qb, ctx_1, plannerCtx); + phase1Result = doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1, plannerCtx); } } return phase1Result; @@ -1879,9 +1870,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx) private int processQueryHint(ASTNode ast, QBParseInfo qbp, int posn) throws SemanticException{ ParseDriver pd = new ParseDriver(); String queryHintStr = ast.getText(); - if (LOG.isDebugEnabled()) { - LOG.debug("QUERY HINT: "+queryHintStr); - } + LOG.debug("QUERY HINT: {} ", queryHintStr); try { ASTNode hintNode = pd.parseHint(queryHintStr); qbp.setHints(hintNode); @@ -1910,8 +1899,7 @@ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase String fullTableName = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), SessionState.get().getCurrentDatabase()); qbp.setDestSchemaForClause(ctx_1.dest, targetColNames); - Set targetColumns = new HashSet(); - targetColumns.addAll(targetColNames); + Set targetColumns = new HashSet<>(targetColNames); if(targetColNames.size() != targetColumns.size()) { throw new SemanticException(generateErrorMessage(tabColName, "Duplicate column name detected in " + fullTableName + " table schema specification")); @@ -2575,7 +2563,7 @@ private void replaceViewReferenceWithDefinition(QB qb, Table tab, (viewMask.isEnabled() && analyzeRewrite == null)) { viewTree = rewriteASTWithMaskAndFilter(viewMask, viewTree, ctx.getViewTokenRewriteStream(viewFullyQualifiedName), - ctx, db, tabNameToTabObject, ignoredTokens); + ctx, db, tabNameToTabObject); } Dispatcher nodeOriginDispatcher = new Dispatcher() { @Override @@ -2786,7 +2774,7 @@ void parseJoinCondPopulateAlias(QBJoinTree joinTree, ASTNode condn, private void populateAliases(List leftAliases, List rightAliases, ASTNode condn, QBJoinTree joinTree, - List leftSrc) throws SemanticException { + List leftSrc) { if ((leftAliases.size() != 0) && (rightAliases.size() != 0)) { joinTree.addPostJoinFilter(condn); return; @@ -2821,7 +2809,7 @@ void applyEqualityPredicateToQBJoinTree(QBJoinTree joinTree, List leftCondAl1, List leftCondAl2, List rightCondAl1, - List rightCondAl2) throws SemanticException { + List rightCondAl2) { if (leftCondAl1.size() != 0) { if ((rightCondAl1.size() != 0) || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) { @@ -3102,14 +3090,14 @@ private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond, } @SuppressWarnings("rawtypes") - private void extractJoinCondsFromWhereClause(QBJoinTree joinTree, QB qb, String dest, ASTNode predicate, - Map aliasToOpInfo) throws SemanticException { + private void extractJoinCondsFromWhereClause(QBJoinTree joinTree, ASTNode predicate, + Map aliasToOpInfo) { switch (predicate.getType()) { case HiveParser.KW_AND: - extractJoinCondsFromWhereClause(joinTree, qb, dest, + extractJoinCondsFromWhereClause(joinTree, (ASTNode) predicate.getChild(0), aliasToOpInfo); - extractJoinCondsFromWhereClause(joinTree, qb, dest, + extractJoinCondsFromWhereClause(joinTree, (ASTNode) predicate.getChild(1), aliasToOpInfo); break; case HiveParser.EQUAL_NS: @@ -3214,8 +3202,8 @@ private Operator genHavingPlan(String dest, QB qb, Operator input, return output; } - protected static ASTNode rewriteGroupingFunctionAST(final List grpByAstExprs, ASTNode targetNode, - final boolean noneSet) throws SemanticException { + protected ASTNode rewriteGroupingFunctionAST(final List grpByAstExprs, ASTNode targetNode, + final boolean noneSet) { TreeVisitorAction action = new TreeVisitorAction() { @@ -3292,8 +3280,7 @@ private Operator genPlanForSubQueryPredicate( Phase1Ctx ctx_1 = initPhase1Ctx(); doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1, null); getMetaData(qbSQ); - Operator op = genPlan(qbSQ); - return op; + return genPlan(qbSQ); } @SuppressWarnings("nls") @@ -3481,10 +3468,7 @@ private Operator genFilterPlan(QB qb, ASTNode condn, Operator input, boolean use ctx.getPlanMapper().link(condn, output); - if (LOG.isDebugEnabled()) { - LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: " - + inputRR.toString()); - } + LOG.debug("Created Filter Plan for {} row schema: {}", qb.getId(), inputRR.toString()); return output; } @@ -3560,17 +3544,14 @@ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(filterDesc, new RowSchema(inputRR.getColumnInfos()), input), inputRR); - if (LOG.isDebugEnabled()) { - LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: " - + inputRR.toString()); - } + LOG.debug("Created Filter Plan for {} row schema: {}", qb.getId(), inputRR); return output; } @SuppressWarnings("nls") // TODO: make aliases unique, otherwise needless rewriting takes place Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, - List col_list, Set excludeCols, RowResolver input, + List colList, Set excludeCols, RowResolver input, RowResolver colSrcRR, Integer pos, RowResolver output, List aliases, boolean ensureUniqueCols) throws SemanticException { @@ -3641,7 +3622,7 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, if (oColInfo == null) { ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(), name, colInfo.getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isSkewedCol()); - col_list.add(expr); + colList.add(expr); oColInfo = new ColumnInfo(getColumnInternalName(pos), colInfo.getType(), colInfo.getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); inputColsProcessed.put(colInfo, oColInfo); @@ -3655,7 +3636,7 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, } else { output.put(tmp[0], tmp[1], oColInfo); } - pos = Integer.valueOf(pos.intValue() + 1); + pos++; matched++; if (unparseTranslator.isEnabled() || (tableMask.isEnabled() && analyzeRewrite == null)) { @@ -3714,17 +3695,13 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, if (input != colSrcRR) { colInfo = input.get(tabAlias, tmp[1]); if (colInfo == null) { - LOG.error("Cannot find colInfo for " + tabAlias + "." + tmp[1] + ", derived from [" - + colSrcRR + "], in [" + input + "]"); + LOG.error("Cannot find colInfo for {}.{}, derived from [{}], in [{}]", tabAlias, tmp[1], colSrcRR, input); throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY, tmp[1]); } - String oldCol = null; - if (LOG.isDebugEnabled()) { - oldCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1])); - } name = colInfo.getInternalName(); tmp = input.reverseLookup(name); if (LOG.isDebugEnabled()) { + String oldCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1])); String newCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1])); LOG.debug("Translated [" + oldCol + "] to [" + newCol + "]"); } @@ -3734,11 +3711,12 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, if (oColInfo == null) { ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(), name, colInfo.getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isSkewedCol()); - col_list.add(expr); + colList.add(expr); oColInfo = new ColumnInfo(getColumnInternalName(pos), colInfo.getType(), colInfo.getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); inputColsProcessed.put(colInfo, oColInfo); } + assert nonNull(tmp); if (ensureUniqueCols) { if (!output.putWithCheck(tmp[0], tmp[1], null, oColInfo)) { throw new CalciteSemanticException("Cannot add column to RR: " + tmp[0] + "." + tmp[1] @@ -3748,7 +3726,7 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, } else { output.put(tmp[0], tmp[1], oColInfo); } - pos = Integer.valueOf(pos.intValue() + 1); + pos++; matched++; if (unparseTranslator.isEnabled() || tableMask.isEnabled()) { @@ -3788,10 +3766,6 @@ private String getScriptArgs(String cmd) { return (end == -1) ? "" : cmd.substring(end, cmd.length()); } - private static int getPositionFromInternalName(String internalName) { - return HiveConf.getPositionFromInternalName(internalName); - } - private String fetchFilesNotInLocalFilesystem(String cmd) { SessionState ss = SessionState.get(); String progName = getScriptProgName(cmd); @@ -3801,15 +3775,14 @@ private String fetchFilesNotInLocalFilesystem(String cmd) { Path p = new Path(filePath); String fileName = p.getName(); String scriptArgs = getScriptArgs(cmd); - String finalCmd = fileName + scriptArgs; - return finalCmd; + return fileName + scriptArgs; } return cmd; } private TableDesc getTableDescFromSerDe(ASTNode child, String cols, - String colTypes, boolean defaultCols) throws SemanticException { + String colTypes) throws SemanticException { if (child.getType() == HiveParser.TOK_SERDENAME) { String serdeName = unescapeSQLString(child.getChild(0).getText()); Class serdeClass = null; @@ -3822,7 +3795,7 @@ private TableDesc getTableDescFromSerDe(ASTNode child, String cols, } TableDesc tblDesc = PlanUtils.getTableDesc(serdeClass, Integer - .toString(Utilities.tabCode), cols, colTypes, defaultCols); + .toString(Utilities.tabCode), cols, colTypes, false); // copy all the properties if (child.getChildCount() == 2) { ASTNode prop = (ASTNode) ((ASTNode) child.getChild(1)).getChild(0); @@ -3837,7 +3810,7 @@ private TableDesc getTableDescFromSerDe(ASTNode child, String cols, return tblDesc; } else if (child.getType() == HiveParser.TOK_SERDEPROPS) { TableDesc tblDesc = PlanUtils.getDefaultTableDesc(Integer - .toString(Utilities.ctrlaCode), cols, colTypes, defaultCols); + .toString(Utilities.ctrlaCode), cols, colTypes, false); int numChildRowFormat = child.getChildCount(); for (int numC = 0; numC < numChildRowFormat; numC++) { ASTNode rowChild = (ASTNode) child.getChild(numC); @@ -4018,7 +3991,7 @@ private Operator genScriptPlan(ASTNode trfm, QB qb, Operator input) if (trfm.getChild(inputSerDeNum).getChildCount() > 0) { inInfo = getTableDescFromSerDe((ASTNode) (((ASTNode) trfm .getChild(inputSerDeNum))).getChild(0), inpColumns.toString(), - inpColumnTypes.toString(), false); + inpColumnTypes.toString()); } else { inInfo = PlanUtils.getTableDesc(serde, Integer .toString(fieldSeparator), inpColumns.toString(), inpColumnTypes @@ -4028,7 +4001,7 @@ private Operator genScriptPlan(ASTNode trfm, QB qb, Operator input) if (trfm.getChild(outputSerDeNum).getChildCount() > 0) { outInfo = getTableDescFromSerDe((ASTNode) (((ASTNode) trfm .getChild(outputSerDeNum))).getChild(0), columns.toString(), - columnTypes.toString(), false); + columnTypes.toString()); // This is for backward compatibility. If the user did not specify the // output column list, we assume that there are 2 columns: key and value. // However, if the script outputs: col1, col2, col3 seperated by TAB, the @@ -4205,7 +4178,7 @@ public static long setBit(long bitmap, int bitIdx) { return bitmap | (1L << bitIdx); } - private static long unsetBit(long bitmap, int bitIdx) { + private long unsetBit(long bitmap, int bitIdx) { return bitmap & ~(1L << bitIdx); } @@ -4353,7 +4326,7 @@ private boolean isAggregateInSelect(Node node, Collection aggregateFunc * Returns whether the pattern is a regex expression (instead of a normal * string). Normal string is a string with all alphabets/digits and "_". */ - static boolean isRegex(String pattern, HiveConf conf) { + boolean isRegex(String pattern, HiveConf conf) { String qIdSupport = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_QUOTEDID_SUPPORT); if ( "column".equals(qIdSupport)) { return false; @@ -4373,9 +4346,7 @@ static boolean isRegex(String pattern, HiveConf conf) { ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); Operator op = genSelectPlan(dest, selExprList, qb, input, inputForSelectStar, false); - if (LOG.isDebugEnabled()) { - LOG.debug("Created Select Plan for clause: " + dest); - } + LOG.debug("Created Select Plan for clause: {}", dest); return op; } @@ -4384,14 +4355,12 @@ static boolean isRegex(String pattern, HiveConf conf) { private Operator genSelectPlan(String dest, ASTNode selExprList, QB qb, Operator input, Operator inputForSelectStar, boolean outerLV) throws SemanticException { - if (LOG.isDebugEnabled()) { - LOG.debug("tree: " + selExprList.toStringTree()); - } + LOG.debug("tree: {}", selExprList.toStringTree()); - List col_list = new ArrayList(); + List colList = new ArrayList(); RowResolver out_rwsch = new RowResolver(); ASTNode trfm = null; - Integer pos = Integer.valueOf(0); + Integer pos = 0; RowResolver inputRR = opParseCtx.get(input).getRowResolver(); RowResolver starRR = null; if (inputForSelectStar != null && inputForSelectStar != input) { @@ -4442,7 +4411,7 @@ static boolean isRegex(String pattern, HiveConf conf) { } if (isUDTF && (selectStar = udtfExprType == HiveParser.TOK_FUNCTIONSTAR)) { genColListRegex(".*", null, (ASTNode) udtfExpr.getChild(0), - col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + colList, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } } @@ -4478,10 +4447,8 @@ static boolean isRegex(String pattern, HiveConf conf) { assert (false); } } - if (LOG.isDebugEnabled()) { - LOG.debug("UDTF table alias is " + udtfTableAlias); - LOG.debug("UDTF col aliases are " + udtfColAliases); - } + LOG.debug("UDTF table alias is {}", udtfTableAlias); + LOG.debug("UDTF col aliases are {}", udtfColAliases); } // The list of expressions after SELECT or SELECT TRANSFORM. @@ -4494,9 +4461,7 @@ static boolean isRegex(String pattern, HiveConf conf) { exprList = selExprList; } - if (LOG.isDebugEnabled()) { - LOG.debug("genSelectPlan: input = " + inputRR + " starRr = " + starRR); - } + LOG.debug("genSelectPlan: input = {} starRr = {}", inputRR, starRR); // For UDTF's, skip the function name to get the expressions int startPosn = isUDTF ? posn + 1 : posn; @@ -4508,9 +4473,6 @@ static boolean isRegex(String pattern, HiveConf conf) { || !qb.getParseInfo().getDestGroupingSets().isEmpty() || !qb.getParseInfo().getDestCubes().isEmpty()); Set colAliases = new HashSet(); - ASTNode[] exprs = new ASTNode[exprList.getChildCount()]; - String[][] aliases = new String[exprList.getChildCount()][]; - boolean[] hasAsClauses = new boolean[exprList.getChildCount()]; int offset = 0; // Iterate over all expression (either after SELECT, or in SELECT TRANSFORM) for (int i = startPosn; i < exprList.getChildCount(); ++i) { @@ -4551,9 +4513,6 @@ static boolean isRegex(String pattern, HiveConf conf) { .getChild(1)); } } - exprs[i] = expr; - aliases[i] = new String[] {tabAlias, colAlias}; - hasAsClauses[i] = hasAsClause; colAliases.add(colAlias); // The real expression @@ -4561,7 +4520,7 @@ static boolean isRegex(String pattern, HiveConf conf) { int initPos = pos; pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), - expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + expr, colList, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); if (unparseTranslator.isEnabled()) { offset += pos - initPos - 1; } @@ -4573,7 +4532,7 @@ static boolean isRegex(String pattern, HiveConf conf) { // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), - null, expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + null, expr, colList, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0) @@ -4585,7 +4544,7 @@ static boolean isRegex(String pattern, HiveConf conf) { // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), - expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + expr, colList, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR, true, isCBOExecuted()); @@ -4604,12 +4563,12 @@ static boolean isRegex(String pattern, HiveConf conf) { out_rwsch.get(null, recommended) == null) { colAlias = recommended; } - col_list.add(exp); + colList.add(exp); ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(pos), exp.getWritableObjectInspector(), tabAlias, false); - colInfo.setSkewedCol((exp instanceof ExprNodeColumnDesc) ? ((ExprNodeColumnDesc) exp) - .isSkewedCol() : false); + colInfo.setSkewedCol((exp instanceof ExprNodeColumnDesc) && ((ExprNodeColumnDesc) exp) + .isSkewedCol()); out_rwsch.put(tabAlias, colAlias, colInfo); if ( exp instanceof ExprNodeColumnDesc ) { @@ -4620,23 +4579,23 @@ static boolean isRegex(String pattern, HiveConf conf) { } } - pos = Integer.valueOf(pos.intValue() + 1); + pos++; } } selectStar = selectStar && exprList.getChildCount() == posn + 1; - out_rwsch = handleInsertStatementSpec(col_list, dest, out_rwsch, qb, selExprList); + out_rwsch = handleInsertStatementSpec(colList, dest, out_rwsch, qb, selExprList); List columnNames = new ArrayList(); Map colExprMap = new HashMap(); - for (int i = 0; i < col_list.size(); i++) { + for (int i = 0; i < colList.size(); i++) { String outputCol = getColumnInternalName(i); - colExprMap.put(outputCol, col_list.get(i)); + colExprMap.put(outputCol, colList.get(i)); columnNames.add(outputCol); } Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( - new SelectDesc(col_list, columnNames, selectStar), new RowSchema( + new SelectDesc(colList, columnNames, selectStar), new RowSchema( out_rwsch.getColumnInfos()), input), out_rwsch); output.setColumnExprMap(colExprMap); @@ -4648,15 +4607,13 @@ static boolean isRegex(String pattern, HiveConf conf) { output = genUDTFPlan(genericUDTF, udtfTableAlias, udtfColAliases, qb, output, outerLV); } - if (LOG.isDebugEnabled()) { - LOG.debug("Created Select Plan row schema: " + out_rwsch.toString()); - } + LOG.debug("Created Select Plan row schema: {}", out_rwsch); return output; } private RowResolver getColForInsertStmtSpec(Map targetCol2Projection, final Table target, Map targetCol2ColumnInfo, int colListPos, - List targetTableColTypes, List new_col_list, + List targetTableColTypes, List newColList, List targetTableColNames) throws SemanticException { RowResolver newOutputRR = new RowResolver(); @@ -4670,7 +4627,7 @@ private RowResolver getColForInsertStmtSpec(Map targetCol2 String f = targetTableColNames.get(i); if(targetCol2Projection.containsKey(f)) { //put existing column in new list to make sure it is in the right position - new_col_list.add(targetCol2Projection.get(f)); + newColList.add(targetCol2Projection.get(f)); ColumnInfo ci = targetCol2ColumnInfo.get(f); ci.setInternalName(getColumnInternalName(colListPos)); newOutputRR.put(ci.getTabAlias(), ci.getInternalName(), ci); @@ -4691,12 +4648,12 @@ private RowResolver getColForInsertStmtSpec(Map targetCol2 throw new SemanticException("Error while parsing default value: " + defaultValue + ". Error message: " + e.getMessage()); } - LOG.debug("Added default value from metastore: " + exp); + LOG.debug("Added default value from metastore: {}", exp); } else { exp = new ExprNodeConstantDesc(targetTableColTypes.get(i), null); } - new_col_list.add(exp); + newColList.add(exp); final String tableAlias = null;//this column doesn't come from any table ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(colListPos), exp.getWritableObjectInspector(), tableAlias, false); @@ -4755,7 +4712,7 @@ RowResolver handleInsertStatementSpec(List col_list, String dest, throw new SemanticException(generateErrorMessage(selExprList, "No table/partition found in QB metadata for dest='" + dest + "'")); } - List new_col_list = new ArrayList(); + List newColList = new ArrayList(); colListPos = 0; List targetTableCols = target != null ? target.getCols() : partition.getCols(); List targetTableColNames = new ArrayList(); @@ -4780,9 +4737,9 @@ RowResolver handleInsertStatementSpec(List col_list, String dest, //where missing columns are NULL-filled Table tbl = target == null? partition.getTable() : target; RowResolver newOutputRR = getColForInsertStmtSpec(targetCol2Projection, tbl, targetCol2ColumnInfo, colListPos, - targetTableColTypes, new_col_list, targetTableColNames); + targetTableColTypes, newColList, targetTableColNames); col_list.clear(); - col_list.addAll(new_col_list); + col_list.addAll(newColList); return newOutputRR; } @@ -4818,11 +4775,7 @@ boolean autogenColAliasPrfxIncludeFuncName() { * Convert exprNodeDesc array to ObjectInspector array. */ static List getWritableObjectInspector(List exprs) { - List result = new ArrayList(); - for (ExprNodeDesc expr : exprs) { - result.add(expr.getWritableObjectInspector()); - } - return result; + return exprs.stream().map(ExprNodeDesc::getWritableObjectInspector).collect(Collectors.toList()); } /** @@ -4938,7 +4891,7 @@ public static ExprNodeDesc isConstantParameterInAggregationParameters(String int } if (Utilities.ReduceField.VALUE.toString().equals(terms[0])) { - int pos = getPositionFromInternalName(terms[1]); + int pos = HiveConf.getPositionFromInternalName(terms[1]); if (pos >= 0 && pos < reduceValues.size()) { ExprNodeDesc reduceValue = reduceValues.get(pos); if (reduceValue != null) { @@ -4998,8 +4951,7 @@ private Operator genGroupByPlanGroupByOperator(QBParseInfo parseInfo, colExprMap.put(field, groupByKeys.get(groupByKeys.size() - 1)); } // For each aggregation - Map aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); + Map aggregationTrees = parseInfo.getAggregationExprsForClause(dest); assert (aggregationTrees != null); // get the last colName for the reduce KEY // it represents the column name corresponding to distinct aggr, if any @@ -5243,8 +5195,7 @@ private Operator genGroupByPlanGroupByOperator1(QBParseInfo parseInfo, } } - Map aggregationTrees = parseInfo - .getAggregationExprsForClause(dest); + Map aggregationTrees = parseInfo.getAggregationExprsForClause(dest); // get the last colName for the reduce KEY // it represents the column name corresponding to distinct aggr, if any String lastKeyColName = null; @@ -5289,7 +5240,7 @@ private Operator genGroupByPlanGroupByOperator1(QBParseInfo parseInfo, String paraExpression = paraExprInfo.getInternalName(); assert (paraExpression != null); - if (isDistinct && lastKeyColName != null) { + if (lastKeyColName != null) { // if aggr is distinct, the parameter is name is constructed as // KEY.lastKeyColName:._colx paraExpression = Utilities.ReduceField.KEY.name() + "." + @@ -5392,8 +5343,6 @@ private void createNewGroupingKey(List groupByKeys, * (qb.getParseInfo().getXXX(dest)). The new GroupByOperator will be a child * of the inputOperatorInfo. * - * @param mode - * The mode of the aggregation (HASH) * @param genericUDAFEvaluators * If not null, this function will store the mapping from Aggregation * StringTree to the genericUDAFEvaluator in this parameter, so it @@ -5405,7 +5354,6 @@ private Operator genGroupByPlanMapGroupByOperator(QB qb, String dest, List grpByExprs, Operator inputOperatorInfo, - GroupByDesc.Mode mode, Map genericUDAFEvaluators, List groupingSetKeys, boolean groupingSetsPresent) throws SemanticException { @@ -5499,7 +5447,7 @@ private Operator genGroupByPlanMapGroupByOperator(QB qb, boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI; containsDistinctAggr = containsDistinctAggr || isDistinct; boolean isAllColumns = value.getType() == HiveParser.TOK_FUNCTIONSTAR; - Mode amode = groupByDescModeToUDAFMode(mode, isDistinct); + Mode amode = groupByDescModeToUDAFMode(GroupByDesc.Mode.HASH, isDistinct); GenericUDAFEvaluator genericUDAFEvaluator = getGenericUDAFEvaluator( aggName, aggParameters, value, isDistinct, isAllColumns); @@ -5528,7 +5476,7 @@ private Operator genGroupByPlanMapGroupByOperator(QB qb, float minReductionHashAggr = HiveConf .getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRHASHMINREDUCTION); Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( - new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, + new GroupByDesc(GroupByDesc.Mode.HASH, outputColumnNames, groupByKeys, aggregations, false, groupByMemoryUsage, memoryThreshold, minReductionHashAggr, groupingSetKeys, groupingSetsPresent, groupingSetsPosition, containsDistinctAggr), new RowSchema(groupByOutputRowResolver.getColumnInfos()), @@ -5647,8 +5595,7 @@ private ReduceSinkOperator genGroupByPlanReduceSinkOperator(QB qb, List reduceKeys = new ArrayList(); - for (int i = 0; i < grpByExprs.size(); ++i) { - ASTNode grpbyExpr = grpByExprs.get(i); + for (ASTNode grpbyExpr : grpByExprs) { ExprNodeDesc inputExpr = genExprNodeDesc(grpbyExpr, reduceSinkInputRowResolver); ColumnInfo prev = reduceSinkOutputRowResolver.getExpression(grpbyExpr); @@ -5658,7 +5605,7 @@ private ReduceSinkOperator genGroupByPlanReduceSinkOperator(QB qb, } reduceKeys.add(inputExpr); outputKeyColumnNames.add(getColumnInternalName(reduceKeys.size() - 1)); - String field = Utilities.ReduceField.KEY.toString() + "." + String field = ReduceField.KEY.toString() + "." + getColumnInternalName(reduceKeys.size() - 1); ColumnInfo colInfo = new ColumnInfo(field, reduceKeys.get( reduceKeys.size() - 1).getTypeInfo(), null, false); @@ -5730,8 +5677,10 @@ private boolean isConsistentWithinQuery(ExprNodeDesc expr) throws SemanticExcept } private void getReduceValuesForReduceSinkNoMapAgg(QBParseInfo parseInfo, String dest, - RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver, - List outputValueColumnNames, List reduceValues, + RowResolver reduceSinkInputRowResolver, + RowResolver reduceSinkOutputRowResolver, + List outputValueColumnNames, + List reduceValues, Map colExprMap) throws SemanticException { Map aggregationTrees = parseInfo.getAggregationExprsForClause(dest); @@ -5962,8 +5911,6 @@ private Operator genGroupByPlanReduceSinkOperator2MR(QBParseInfo parseInfo, * (parseInfo.getXXX(dest)). The new GroupByOperator will do the second * aggregation based on the partial aggregation results. * - * @param mode - * the mode of aggregation (FINAL) * @param genericUDAFEvaluators * The mapping from Aggregation StringTree to the * genericUDAFEvaluator. @@ -5974,7 +5921,6 @@ private Operator genGroupByPlanReduceSinkOperator2MR(QBParseInfo parseInfo, private Operator genGroupByPlanGroupByOperator2MR(QBParseInfo parseInfo, String dest, Operator reduceSinkOperatorInfo2, - GroupByDesc.Mode mode, Map genericUDAFEvaluators, boolean groupingSetsPresent) throws SemanticException { @@ -6037,7 +5983,7 @@ private Operator genGroupByPlanGroupByOperator2MR(QBParseInfo parseInfo, boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI; containsDistinctAggr = containsDistinctAggr || isDistinct; - Mode amode = groupByDescModeToUDAFMode(mode, isDistinct); + Mode amode = groupByDescModeToUDAFMode(GroupByDesc.Mode.FINAL, isDistinct); GenericUDAFEvaluator genericUDAFEvaluator = genericUDAFEvaluators .get(entry.getKey()); assert (genericUDAFEvaluator != null); @@ -6048,8 +5994,7 @@ private Operator genGroupByPlanGroupByOperator2MR(QBParseInfo parseInfo, aggName.toLowerCase(), udaf.genericUDAFEvaluator, udaf.convertedParameters, - (mode != GroupByDesc.Mode.FINAL && value.getToken().getType() == - HiveParser.TOK_FUNCTIONDI), + false, amode)); String field = getColumnInternalName(groupByKeys.size() + aggregations.size() - 1); @@ -6064,7 +6009,7 @@ private Operator genGroupByPlanGroupByOperator2MR(QBParseInfo parseInfo, .getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRHASHMINREDUCTION); Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( - new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, + new GroupByDesc(GroupByDesc.Mode.FINAL, outputColumnNames, groupByKeys, aggregations, false, groupByMemoryUsage, memoryThreshold, minReductionHashAggr, null, false, groupingSetsPosition, containsDistinctAggr), new RowSchema(groupByOutputRowResolver2.getColumnInfos()), @@ -6187,9 +6132,7 @@ private Operator genGroupByPlan1ReduceMultiGBY(List dests, QB qb, Operat List expressions = new ArrayList(2); expressions.add(current); expressions.add(previous); - ExprNodeDesc orExpr = - new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, or, expressions); - previous = orExpr; + previous = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, or, expressions); } else { // If an expression does not have a where clause, there can be no common filter previous = null; @@ -6338,7 +6281,7 @@ private Operator genGroupByPlan2MR(String dest, QB qb, Operator input) // ////// 4. Generate GroupbyOperator2 Operator groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR(parseInfo, - dest, reduceSinkOperatorInfo2, GroupByDesc.Mode.FINAL, + dest, reduceSinkOperatorInfo2, genericUDAFEvaluators, false); return groupByOperatorInfo2; @@ -6350,15 +6293,10 @@ private boolean optimizeMapAggrGroupBy(String dest, QB qb) throws SemanticExcept return false; } - if (!qb.getParseInfo().getDistinctFuncExprsForClause(dest).isEmpty()) { - return false; - } - - return true; + return qb.getParseInfo().getDistinctFuncExprsForClause(dest).isEmpty(); } - static private void extractColumns(Set colNamesExprs, - ExprNodeDesc exprNode) throws SemanticException { + private void extractColumns(Set colNamesExprs, ExprNodeDesc exprNode) { if (exprNode instanceof ExprNodeColumnDesc) { colNamesExprs.add(((ExprNodeColumnDesc) exprNode).getColumn()); return; @@ -6372,14 +6310,8 @@ static private void extractColumns(Set colNamesExprs, } } - static private boolean hasCommonElement(Set set1, Set set2) { - for (String elem1 : set1) { - if (set2.contains(elem1)) { - return true; - } - } - - return false; + private boolean hasCommonElement(Set set1, Set set2) { + return set1.stream().anyMatch(set2::contains); } void checkExpressionsForGroupingSet(List grpByExprs, @@ -6506,9 +6438,8 @@ private Operator genGroupByPlanMapAggrNoSkew(String dest, QB qb, // Is the grouping sets data consumed in the current in MR job, or // does it need an additional MR job - boolean groupingSetsNeedAdditionalMRJob = - groupingSetsPresent && groupingSets.size() > newMRJobGroupingSetsThreshold ? - true : false; + boolean groupingSetsNeedAdditionalMRJob = groupingSetsPresent && + groupingSets.size() > newMRJobGroupingSetsThreshold; GroupByOperator groupByOperatorInfo = (GroupByOperator) genGroupByPlanMapGroupByOperator( @@ -6516,7 +6447,6 @@ private Operator genGroupByPlanMapAggrNoSkew(String dest, QB qb, dest, grpByExprs, inputOperatorInfo, - GroupByDesc.Mode.HASH, genericUDAFEvaluators, groupingSets, groupingSetsPresent && !groupingSetsNeedAdditionalMRJob); @@ -6587,7 +6517,7 @@ private Operator genGroupByPlanMapAggrNoSkew(String dest, QB qb, // ////// Generate GroupbyOperator3 return genGroupByPlanGroupByOperator2MR(parseInfo, dest, - reduceSinkOperatorInfo2, GroupByDesc.Mode.FINAL, + reduceSinkOperatorInfo2, genericUDAFEvaluators, groupingSetsPresent); } } @@ -6681,7 +6611,7 @@ private Operator genGroupByPlanMapAggr2MR(String dest, QB qb, new LinkedHashMap(); GroupByOperator groupByOperatorInfo = (GroupByOperator) genGroupByPlanMapGroupByOperator( - qb, dest, grpByExprs, inputOperatorInfo, GroupByDesc.Mode.HASH, + qb, dest, grpByExprs, inputOperatorInfo, genericUDAFEvaluators, groupingSets, groupingSetsPresent); groupOpToInputTables.put(groupByOperatorInfo, opParseCtx.get( @@ -6722,7 +6652,7 @@ private Operator genGroupByPlanMapAggr2MR(String dest, QB qb, // ////// Generate GroupbyOperator3 return genGroupByPlanGroupByOperator2MR(parseInfo, dest, - reduceSinkOperatorInfo2, GroupByDesc.Mode.FINAL, + reduceSinkOperatorInfo2, genericUDAFEvaluators, groupingSetsPresent); } else { // If there are no grouping keys, grouping sets cannot be present @@ -6741,7 +6671,7 @@ private Operator genGroupByPlanMapAggr2MR(String dest, QB qb, groupingSetsPresent); return genGroupByPlanGroupByOperator2MR(parseInfo, dest, - reduceSinkOperatorInfo, GroupByDesc.Mode.FINAL, genericUDAFEvaluators, false); + reduceSinkOperatorInfo, genericUDAFEvaluators, false); } } @@ -6863,8 +6793,8 @@ private Operator genBucketingSortingDest(String dest, Operator input, QB qb, if ((dest_tab.getSortCols() != null) && (dest_tab.getSortCols().size() > 0)) { - sortCols = getSortCols(dest, qb, dest_tab, table_desc, input, true); - sortOrders = getSortOrders(dest, qb, dest_tab, input); + sortCols = getSortCols(dest, qb, dest_tab, table_desc, input); + sortOrders = getSortOrders(dest_tab); if (!enforceBucketing) { throw new SemanticException(ErrorMsg.TBL_SORTED_NOT_BUCKETED.getErrorCodedMsg(dest_tab.getCompleteName())); } @@ -6887,7 +6817,7 @@ private Operator genBucketingSortingDest(String dest, Operator input, QB qb, } int numBuckets = dest_tab.getNumBuckets(); if (numBuckets > maxReducers) { - LOG.debug("numBuckets is {}", numBuckets, " and maxReducers is {}", maxReducers); + LOG.debug("numBuckets is {} and maxReducers is {}", numBuckets, maxReducers); multiFileSpray = true; totalFiles = numBuckets; if (totalFiles % maxReducers == 0) { @@ -6940,11 +6870,6 @@ private void genPartnCols(String dest, Operator input, QB qb, if (!enforceBucketing) { throw new SemanticException(ErrorMsg.TBL_SORTED_NOT_BUCKETED.getErrorCodedMsg(dest_tab.getCompleteName())); } - else { - if(!enforceBucketing) { - partnColsNoConvert = getSortCols(dest, qb, dest_tab, table_desc, input, false); - } - } enforceBucketing = true; } @@ -7172,7 +7097,6 @@ private ExprNodeDesc getCheckConstraintExpr(Table tbl, Operator input, RowResolv } private ImmutableBitSet getEnabledNotNullConstraints(Table tbl) throws HiveException{ - List nullConstraints = new ArrayList<>(); final NotNullConstraint nnc = Hive.get().getEnabledNotNullConstraints( tbl.getDbName(), tbl.getTableName()); ImmutableBitSet bitSet = null; @@ -7202,9 +7126,7 @@ private boolean mergeCardinalityViolationBranch(final Operator input) { ExprNodeDesc colExpr = selectOp.getConf().getColList().get(0); if(colExpr instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc)colExpr ; - if(func.getGenericUDF() instanceof GenericUDFCardinalityViolation){ - return true; - } + return func.getGenericUDF() instanceof GenericUDFCardinalityViolation; } } } @@ -7259,11 +7181,9 @@ else if(checkConstraintExpr != null) { if (combinedConstraintExpr != null) { ExprNodeDesc constraintUDF = TypeCheckProcFactory.DefaultExprProcessor. getFuncExprNodeDesc("enforce_constraint", combinedConstraintExpr); - Operator newConstraintFilter = putOpInsertMap(OperatorFactory.getAndMakeChild( + return putOpInsertMap(OperatorFactory.getAndMakeChild( new FilterDesc(constraintUDF, false), new RowSchema( inputRR.getColumnInfos()), input), inputRR); - - return newConstraintFilter; } return input; } @@ -7314,8 +7234,7 @@ private ExprNodeDesc getNotNullConstraintExpr(Table targetTable, Operator input, return currUDF; } - private Path getDestinationFilePath(final String destinationFile, boolean isMmTable) - throws SemanticException { + private Path getDestinationFilePath(final String destinationFile, boolean isMmTable) { if (this.isResultsCacheEnabled() && this.queryTypeCanUseCache()) { assert (!isMmTable); QueryResultsCache instance = QueryResultsCache.getInstance(); @@ -7449,7 +7368,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) if (destTableIsFullAcid) { acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest); //todo: should this be done for MM? is it ok to use CombineHiveInputFormat with MM - checkAcidConstraints(qb, tableDescriptor, destinationTable); + checkAcidConstraints(); } try { if (ctx.getExplainConfig() != null) { @@ -7500,8 +7419,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destinationTable.getFullyQualifiedName(), false, false, true); } - WriteEntity output = generateTableWriteEntity( - dest, destinationTable, partSpec, ltd, dpCtx, isNonNativeTable); + WriteEntity output = generateTableWriteEntity(dest, destinationTable, partSpec, ltd, dpCtx); ctx.getLoadTableOutputMap().put(ltd, output); break; } @@ -7514,7 +7432,6 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) checkExternalTable(destinationTable); - Path tabPath = destinationTable.getPath(); Path partPath = destinationPartition.getDataLocation(); checkImmutableTable(qb, destinationTable, partPath, true); @@ -7573,7 +7490,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) if (destTableIsFullAcid) { acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest); //todo: should this be done for MM? is it ok to use CombineHiveInputFormat with MM? - checkAcidConstraints(qb, tableDescriptor, destinationTable); + checkAcidConstraints(); } try { if (ctx.getExplainConfig() != null) { @@ -7605,9 +7522,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) ltd.setLbCtx(lbCtx); loadTableWork.add(ltd); - if (!outputs.add(new WriteEntity(destinationPartition, - determineWriteType(ltd, destinationTable.isNonNative(), dest)))) { - + if (!outputs.add(new WriteEntity(destinationPartition, determineWriteType(ltd, dest)))) { throw new SemanticException(ErrorMsg.OUTPUT_SPECIFIED_MULTIPLE_TIMES .getMsg(destinationTable.getTableName() + "@" + destinationPartition.getName())); } @@ -7833,7 +7748,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) if (destTableIsFullAcid) { acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest); //todo: should this be done for MM? is it ok to use CombineHiveInputFormat with MM - checkAcidConstraints(qb, tableDescriptor, null); + checkAcidConstraints(); } // isReplace = false in case concurrent operation is executed ltd = new LoadTableDesc(queryTmpdir, tableDescriptor, dpCtx, acidOp, false, writeId); @@ -7852,8 +7767,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) ltd.setLoadFileType(LoadFileType.KEEP_EXISTING); } ltd.setMdTable(destinationTable); - WriteEntity output = generateTableWriteEntity( - dest, destinationTable, dpCtx.getPartSpec(), ltd, dpCtx, isNonNativeTable); + WriteEntity output = generateTableWriteEntity(dest, destinationTable, dpCtx.getPartSpec(), ltd, dpCtx); ctx.getLoadTableOutputMap().put(ltd, output); } else { // Create LFD even for MM CTAS - it's a no-op move, but it still seems to be used for stats. @@ -7890,9 +7804,9 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) .getDeserializer(conf).getObjectInspector(); List fields = rowObjectInspector .getAllStructFieldRefs(); - for (int i = 0; i < fields.size(); i++) { - vecCol.add(new ColumnInfo(fields.get(i).getFieldName(), TypeInfoUtils - .getTypeInfoFromObjectInspector(fields.get(i) + for (StructField field : fields) { + vecCol.add(new ColumnInfo(field.getFieldName(), TypeInfoUtils + .getTypeInfoFromObjectInspector(field .getFieldObjectInspector()), "", false)); } } catch (Exception e) { @@ -7942,10 +7856,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) handleLineage(ltd, output); setWriteIdForSurrogateKeys(ltd, input); - if (LOG.isDebugEnabled()) { - LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: " - + destinationPath + " row schema: " + inputRR.toString()); - } + LOG.debug("Created FileSink Plan for clause: {}dest_path: {} row schema: {}", dest, destinationPath, inputRR); FileSinkOperator fso = (FileSinkOperator) output; fso.getConf().setTable(destinationTable); @@ -7958,16 +7869,16 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) && !destTableIsTemporary && !destTableIsMaterialization && ColumnStatsAutoGatherContext.canRunAutogatherStats(fso)) { if (destType == QBMetaData.DEST_TABLE) { - genAutoColumnStatsGatheringPipeline(qb, destinationTable, partSpec, input, + genAutoColumnStatsGatheringPipeline(destinationTable, partSpec, input, qb.getParseInfo().isInsertIntoTable(destinationTable.getDbName(), destinationTable.getTableName()), false); } else if (destType == QBMetaData.DEST_PARTITION) { - genAutoColumnStatsGatheringPipeline(qb, destinationTable, destinationPartition.getSpec(), input, + genAutoColumnStatsGatheringPipeline(destinationTable, destinationPartition.getSpec(), input, qb.getParseInfo().isInsertIntoTable(destinationTable.getDbName(), destinationTable.getTableName()), false); } else if (destType == QBMetaData.DEST_LOCAL_FILE || destType == QBMetaData.DEST_DFS_FILE) { // CTAS or CMV statement - genAutoColumnStatsGatheringPipeline(qb, destinationTable, null, input, + genAutoColumnStatsGatheringPipeline(destinationTable, null, input, false, true); } } @@ -8009,8 +7920,7 @@ private ColsAndTypes deriveFileSinkColTypes( if (numNonPartitionedCols <= 0) { throw new SemanticException("Too many partition columns declared"); } - for (int i = 0; i < colInfos.size(); i++) { - ColumnInfo colInfo = colInfos.get(i); + for (ColumnInfo colInfo : colInfos) { String[] nm = inputRR.reverseLookup(colInfo.getInternalName()); if (nm[1] != null) { // non-null column alias @@ -8233,7 +8143,7 @@ private void handleLineage(LoadTableDesc ltd, Operator output) } } - private void setWriteIdForSurrogateKeys(LoadTableDesc ltd, Operator input) throws SemanticException { + private void setWriteIdForSurrogateKeys(LoadTableDesc ltd, Operator input) { if (ltd == null) { return; } @@ -8257,7 +8167,7 @@ private void setWriteIdForSurrogateKeys(LoadTableDesc ltd, Operator input) throw private WriteEntity generateTableWriteEntity(String dest, Table dest_tab, Map partSpec, LoadTableDesc ltd, - DynamicPartitionCtx dpCtx, boolean isNonNativeTable) + DynamicPartitionCtx dpCtx) throws SemanticException { WriteEntity output = null; @@ -8265,7 +8175,7 @@ private WriteEntity generateTableWriteEntity(String dest, Table dest_tab, // in the case of DP, we will register WriteEntity in MoveTask when the // list of dynamically created partitions are known. if ((dpCtx == null || dpCtx.getNumDPCols() == 0)) { - output = new WriteEntity(dest_tab, determineWriteType(ltd, isNonNativeTable, dest)); + output = new WriteEntity(dest_tab, determineWriteType(ltd, dest)); if (!outputs.add(output)) { if(!((this instanceof MergeSemanticAnalyzer) && conf.getBoolVar(ConfVars.MERGE_SPLIT_UPDATE))) { @@ -8283,7 +8193,7 @@ private WriteEntity generateTableWriteEntity(String dest, Table dest_tab, if ((dpCtx != null) && (dpCtx.getNumDPCols() >= 0)) { // No static partition specified if (dpCtx.getNumSPCols() == 0) { - output = new WriteEntity(dest_tab, determineWriteType(ltd, isNonNativeTable, dest), false); + output = new WriteEntity(dest_tab, determineWriteType(ltd, dest), false); outputs.add(output); output.setDynamicPartitionWrite(true); } @@ -8374,12 +8284,11 @@ private void createPreInsertDesc(Table table, boolean overwrite) { PreInsertTableDesc preInsertTableDesc = new PreInsertTableDesc(table, overwrite); this.rootTasks .add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), preInsertTableDesc))); - } - private void genAutoColumnStatsGatheringPipeline(QB qb, Table table, Map partSpec, - Operator curr, boolean isInsertInto, boolean useTableValueConstructor) + private void genAutoColumnStatsGatheringPipeline(Table table, Map partSpec, Operator curr, + boolean isInsertInto, boolean useTableValueConstructor) throws SemanticException { LOG.info("Generate an operator pipeline to autogather column stats for table " + table.getTableName() + " in query " + ctx.getCmd()); @@ -8399,8 +8308,7 @@ String fixCtasColumnName(String colName) { return colName; } - private void checkAcidConstraints(QB qb, TableDesc tableDesc, - Table table) throws SemanticException { + private void checkAcidConstraints() { /* LOG.info("Modifying config values for ACID write"); conf.setBoolVar(ConfVars.HIVEOPTREDUCEDEDUPLICATION, true); @@ -8457,8 +8365,6 @@ private Operator genConversionSelectOperator(String dest, QB qb, Operator input, // does the conversion to String by itself. boolean isMetaDataSerDe = table_desc.getDeserializerClass().equals( MetadataTypedColumnsetSerDe.class); - boolean isLazySimpleSerDe = table_desc.getDeserializerClass().equals( - LazySimpleSerDe.class); if (!isMetaDataSerDe && !deleting(dest)) { // If we're updating, add the ROW__ID expression, then make the following column accesses @@ -8536,8 +8442,7 @@ private Operator genConversionSelectOperator(String dest, QB qb, Operator input, } @SuppressWarnings("nls") - private Operator genLimitPlan(String dest, QB qb, Operator input, int offset, int limit) - throws SemanticException { + private Operator genLimitPlan(String dest, Operator input, int offset, int limit) { // A map-only job can be optimized - instead of converting it to a // map-reduce job, we can have another map // job to do the same to avoid the cost of sorting in the map-reduce phase. @@ -8583,10 +8488,7 @@ private Operator genUDTFPlan(GenericUDTF genericUDTF, String outputTableAlias, L throw new SemanticException(ErrorMsg.UDTF_LATERAL_VIEW.getMsg()); } - if (LOG.isDebugEnabled()) { - LOG.debug("Table alias: " + outputTableAlias + " Col aliases: " - + colAliases); - } + LOG.debug("Table alias: {} Col aliases: {}", outputTableAlias, colAliases); // Use the RowResolver from the input operator to generate a input // ObjectInspector that can be used to initialize the UDTF. Then, the @@ -8648,10 +8550,9 @@ private Operator genUDTFPlan(GenericUDTF genericUDTF, String outputTableAlias, L } // Add the UDTFOperator to the operator DAG - Operator udtf = putOpInsertMap(OperatorFactory.getAndMakeChild( + return putOpInsertMap(OperatorFactory.getAndMakeChild( new UDTFDesc(genericUDTF, outerLV), new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch); - return udtf; } @SuppressWarnings("nls") @@ -8663,7 +8564,7 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, // A better approach would be to // write into a local file and then have a map-only job. // Add the limit operator to get the value fields - Operator curr = genLimitPlan(dest, qb, input, offset, limit); + Operator curr = genLimitPlan(dest, input, offset, limit); // the client requested that an extra map-reduce step be performed if (!extraMRStep || !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_GROUPBY_LIMIT_EXTRASTEP)){ @@ -8672,7 +8573,7 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, // Create a reduceSink operator followed by another limit curr = genReduceSinkPlan(dest, qb, curr, 1, false); - return genLimitPlan(dest, qb, curr, offset, limit); + return genLimitPlan(dest, curr, offset, limit); } private List getPartitionColsFromBucketCols(String dest, QB qb, Table tab, TableDesc table_desc, @@ -8695,7 +8596,7 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, } } - return genConvertCol(dest, qb, tab, table_desc, input, posns, convert); + return genConvertCol(dest, qb, table_desc, input, posns, convert); } // We have to set up the bucketing columns differently for update and deletes, @@ -8719,13 +8620,14 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, return rlist; } - private List genConvertCol(String dest, QB qb, Table tab, TableDesc table_desc, Operator input, - List posns, boolean convert) throws SemanticException { + private List genConvertCol(String dest, QB qb, TableDesc tableDesc, Operator input, + List posns, boolean convert) + throws SemanticException { StructObjectInspector oi = null; try { - Deserializer deserializer = table_desc.getDeserializerClass() + Deserializer deserializer = tableDesc.getDeserializerClass() .newInstance(); - SerDeUtils.initializeSerDe(deserializer, conf, table_desc.getProperties(), null); + SerDeUtils.initializeSerDe(deserializer, conf, tableDesc.getProperties(), null); oi = (StructObjectInspector) deserializer.getObjectInspector(); } catch (Exception e) { throw new SemanticException(e); @@ -8767,8 +8669,7 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, return expressions; } - private List getSortCols(String dest, QB qb, Table tab, TableDesc table_desc, - Operator input, boolean convert) + private List getSortCols(String dest, QB qb, Table tab, TableDesc tableDesc, Operator input) throws SemanticException { List tabSortCols = tab.getSortCols(); List tabCols = tab.getCols(); @@ -8786,11 +8687,10 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, } } - return genConvertCol(dest, qb, tab, table_desc, input, posns, convert); + return genConvertCol(dest, qb, tableDesc, input, posns, true); } - private List getSortOrders(String dest, QB qb, Table tab, Operator input) - throws SemanticException { + private List getSortOrders(Table tab) { List tabSortCols = tab.getSortCols(); List tabCols = tab.getCols(); @@ -9066,7 +8966,6 @@ private Operator genJoinOperatorChildren(QBJoinTree join, Operator left, List outputColumnNames = new ArrayList(); // all children are base classes Operator[] rightOps = new Operator[right.length]; - int outputPos = 0; Map reversedExprs = new HashMap(); Map> exprMap = new HashMap>(); @@ -9239,7 +9138,7 @@ private Operator genJoinOperatorChildren(QBJoinTree join, Operator left, } @SuppressWarnings("nls") - private Operator genJoinReduceSinkChild(QB qb, ExprNodeDesc[] joinKeys, + private Operator genJoinReduceSinkChild(ExprNodeDesc[] joinKeys, Operator child, String[] srcs, int tag) throws SemanticException { Operator dummy = Operator.createDummy(); // dummy for backtracking @@ -9258,8 +9157,7 @@ private Operator genJoinReduceSinkChild(QB qb, ExprNodeDesc[] joinKeys, } // Walk over the input row resolver and copy in the output - List reduceValues = new ArrayList(); - List reduceValuesBack = new ArrayList(); + ArrayList reduceValues = new ArrayList(); Map colExprMap = new HashMap(); List columns = inputRR.getColumnInfos(); @@ -9295,7 +9193,6 @@ private Operator genJoinReduceSinkChild(QB qb, ExprNodeDesc[] joinKeys, String outputColName = getColumnInternalName(reduceValues.size()); reduceValues.add(expr); - reduceValuesBack.add(exprBack); ColumnInfo newColInfo = new ColumnInfo(colInfo); newColInfo.setInternalName(Utilities.ReduceField.VALUE + "." + outputColName); @@ -9384,7 +9281,7 @@ private Operator genJoinOperator(QB qb, QBJoinTree joinTree, // generate a groupby operator (HASH mode) for a map-side partial // aggregation for semijoin - srcOps[pos++] = genMapGroupByForSemijoin(qb, fields, srcOp, GroupByDesc.Mode.HASH); + srcOps[pos++] = genMapGroupByForSemijoin(fields, srcOp); } else { srcOps[pos++] = srcOp; } @@ -9402,7 +9299,7 @@ private Operator genJoinOperator(QB qb, QBJoinTree joinTree, if (!isCBOExecuted()) { srcOps[i] = genNotNullFilterForJoinSourcePlan(qb, srcOps[i], joinTree, joinKeys[i]); } - srcOps[i] = genJoinReduceSinkChild(qb, joinKeys[i], srcOps[i], srcs, joinTree.getNextTag()); + srcOps[i] = genJoinReduceSinkChild(joinKeys[i], srcOps[i], srcs, joinTree.getNextTag()); } Operator topOp = genJoinOperatorChildren(joinTree, joinSrcOp, srcOps, omitOpts, joinKeys); @@ -9498,8 +9395,7 @@ private Operator insertSelectForSemijoin(List fields, return output; } - private Operator genMapGroupByForSemijoin(QB qb, List fields, - Operator input, GroupByDesc.Mode mode) + private Operator genMapGroupByForSemijoin(List fields, Operator input) throws SemanticException { RowResolver groupByInputRowResolver = opParseCtx.get(input).getRowResolver(); @@ -9557,7 +9453,7 @@ private Operator genMapGroupByForSemijoin(QB qb, List fields, float minReductionHashAggr = HiveConf .getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRHASHMINREDUCTION); Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild( - new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations, + new GroupByDesc(GroupByDesc.Mode.HASH, outputColumnNames, groupByKeys, aggregations, false, groupByMemoryUsage, memoryThreshold, minReductionHashAggr, null, false, -1, false), new RowSchema(groupByOutputRowResolver.getColumnInfos()), input), groupByOutputRowResolver); @@ -9610,8 +9506,7 @@ private Operator genMapGroupByForSemijoin(QB qb, List fields, private Operator genJoinPlan(QB qb, Map map) throws SemanticException { QBJoinTree joinTree = qb.getQbJoinTree(); - Operator joinOp = genJoinOperator(qb, joinTree, map, null); - return joinOp; + return genJoinOperator(qb, joinTree, map, null); } /** @@ -9881,7 +9776,7 @@ private QBJoinTree genSQJoinTree(QB qb, ISubQueryJoinInfo subQuery, joinTree.getAliasToOpInfo().put( getModifiedAlias(qb, rightalias), aliasToOpInfo.get(rightalias)); // remember rhs table for semijoin - if (joinTree.getNoSemiJoin() == false) { + if (!joinTree.getNoSemiJoin()) { joinTree.addRHSSemijoin(rightalias); } @@ -9920,7 +9815,6 @@ private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, QBJoinTree joinTree = new QBJoinTree(); JoinCond[] condn = new JoinCond[1]; - int joinType = joinParseTree.getToken().getType(); switch (joinParseTree.getToken().getType()) { case HiveParser.TOK_LEFTOUTERJOIN: joinTree.setNoOuterJoin(false); @@ -10013,7 +9907,7 @@ private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, joinTree.getAliasToOpInfo().put( getModifiedAlias(qb, alias), aliasToOpInfo.get(alias)); // remember rhs table for semijoin - if (joinTree.getNoSemiJoin() == false) { + if (!joinTree.getNoSemiJoin()) { joinTree.addRHSSemijoin(alias); } } else { @@ -10077,7 +9971,7 @@ private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, joinTree.setMapAliases(mapAliases); - if ((conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) == false) { + if (!(conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez"))) { parseStreamTables(joinTree, qb); } } @@ -10085,7 +9979,7 @@ private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, return joinTree; } - private static boolean isValidJoinSide(ASTNode right) { + private boolean isValidJoinSide(ASTNode right) { return (right.getToken().getType() == HiveParser.TOK_TABREF) || (right.getToken().getType() == HiveParser.TOK_SUBQUERY) || (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION); @@ -10214,9 +10108,8 @@ private int parseSingleSemiJoinHint(Tree args, int curIdx, Map hints) throws SemanticException { + private boolean disableMapJoinWithHint(List hints) { if (hints == null || hints.size() == 0) { return false; } @@ -10242,11 +10135,10 @@ private boolean disableMapJoinWithHint(List hints) throws SemanticExcep /** * Merges node to target */ - private void mergeJoins(QB qb, QBJoinTree node, QBJoinTree target, int pos, int[] tgtToNodeExprMap) { + private void mergeJoins(QBJoinTree node, QBJoinTree target, int pos, int[] tgtToNodeExprMap) { String[] nodeRightAliases = node.getRightAliases(); String[] trgtRightAliases = target.getRightAliases(); - String[] rightAliases = new String[nodeRightAliases.length - + trgtRightAliases.length]; + String[] rightAliases = new String[nodeRightAliases.length + trgtRightAliases.length]; for (int i = 0; i < trgtRightAliases.length; i++) { rightAliases[i] = trgtRightAliases[i]; @@ -10346,17 +10238,8 @@ private void mergeJoins(QB qb, QBJoinTree node, QBJoinTree target, int pos, int[ } } - if (node.getNoOuterJoin() && target.getNoOuterJoin()) { - target.setNoOuterJoin(true); - } else { - target.setNoOuterJoin(false); - } - - if (node.getNoSemiJoin() && target.getNoSemiJoin()) { - target.setNoSemiJoin(true); - } else { - target.setNoSemiJoin(false); - } + target.setNoOuterJoin(node.getNoOuterJoin() && target.getNoOuterJoin()); + target.setNoSemiJoin(node.getNoSemiJoin() && target.getNoSemiJoin()); target.mergeRHSSemijoin(node); @@ -10396,9 +10279,7 @@ private void mergeJoins(QB qb, QBJoinTree node, QBJoinTree target, int pos, int[ // Safety check: if we are merging join operators and there are post-filtering // conditions, they cannot be outer joins assert node.getNoOuterJoin() ; - if( target.getPostJoinFilters().size() != 0) { - assert target.getNoOuterJoin() ; - } + assert target.getPostJoinFilters().size() == 0 || target.getNoOuterJoin(); for (ASTNode exprPostFilter : node.getPostJoinFilters()) { target.addPostJoinFilter(exprPostFilter); } @@ -10548,7 +10429,7 @@ private void mergeJoinTree(QB qb) { continue; } } - mergeJoins(qb, node, target, pos, mergeDetails.getRight()); + mergeJoins(node, target, pos, mergeDetails.getRight()); trees.set(j, null); mergedQBJTree = true; continue; // continue merging with next alias @@ -10586,8 +10467,7 @@ private void mergeJoinTree(QB qb) { // reconstruct join tree QBJoinTree current = null; - for (int i = 0; i < trees.size(); i++) { - QBJoinTree target = trees.get(i); + for (QBJoinTree target : trees) { if (target == null) { continue; } @@ -10603,23 +10483,17 @@ private void mergeJoinTree(QB qb) { // Join types should be all the same for merging (or returns null) private JoinType getType(JoinCond[] conds) { JoinType type = conds[0].getJoinType(); - for (int k = 1; k < conds.length; k++) { - if (type != conds[k].getJoinType()) { - return null; - } - } - return type; + return Arrays.stream(conds).allMatch(cond -> cond.getJoinType() == type) ? type : null; } - private Operator genSelectAllDesc(Operator input) throws SemanticException { + private Operator genSelectAllDesc(Operator input) { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); List columns = inputRR.getColumnInfos(); List colList = new ArrayList(); List columnNames = new ArrayList(); Map columnExprMap = new HashMap(); - for (int i = 0; i < columns.size(); i++) { - ColumnInfo col = columns.get(i); + for (ColumnInfo col : columns) { colList.add(new ExprNodeColumnDesc(col, true)); columnNames.add(col.getInternalName()); columnExprMap.put(col.getInternalName(), new ExprNodeColumnDesc(col, true)); @@ -10640,8 +10514,7 @@ private Operator genSelectAllDesc(Operator input) throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); - SortedSet ks = new TreeSet(); - ks.addAll(qbp.getClauseNames()); + SortedSet ks = new TreeSet(qbp.getClauseNames()); List> commonGroupByDestGroups = new ArrayList>(); @@ -10686,15 +10559,13 @@ private Operator genSelectAllDesc(Operator input) throws SemanticException { if (currentDistinctKeys.isEmpty()) { // current dest has no distinct keys. - List combinedList = new ArrayList(); - combineExprNodeLists(targetSprayKeys, targetDistinctKeys, combinedList); + List combinedList = combineExprNodeLists(targetSprayKeys, targetDistinctKeys); if (!matchExprLists(combinedList, currentSprayKeys)) { continue; } // else do the common code at the end. } else { if (targetDistinctKeys.isEmpty()) { - List combinedList = new ArrayList(); - combineExprNodeLists(currentSprayKeys, currentDistinctKeys, combinedList); + List combinedList = combineExprNodeLists(currentSprayKeys, currentDistinctKeys); if (!matchExprLists(combinedList, targetSprayKeys)) { continue; } else { @@ -10755,14 +10626,14 @@ private Operator genSelectAllDesc(Operator input) throws SemanticException { return sprayKeys; } - private void combineExprNodeLists(List list, List list2, - List combinedList) { - combinedList.addAll(list); + private List combineExprNodeLists(List list, List list2) { + ArrayList result = new ArrayList<>(list); for (ExprNodeDesc elem : list2) { - if (!combinedList.contains(elem)) { - combinedList.add(elem); + if (!result.contains(elem)) { + result.add(elem); } } + return result; } // Returns whether or not two lists contain the same elements independent of order @@ -10889,7 +10760,7 @@ private Operator genBodyPlan(QB qb, Operator input, Map aliasT if (node.getToken().getType() == HiveParser.TOK_ALLCOLREF) { curr = genSelectPlan(dest, qb, curr, curr); RowResolver rr = opParseCtx.get(curr).getRowResolver(); - qbp.setSelExprForClause(dest, SemanticAnalyzer.genSelectDIAST(rr)); + qbp.setSelExprForClause(dest, genSelectDIAST(rr)); } } if (conf.getBoolVar(HiveConf.ConfVars.HIVEMAPSIDEAGGREGATE)) { @@ -10917,10 +10788,7 @@ private Operator genBodyPlan(QB qb, Operator input, Map aliasT } } - - if (LOG.isDebugEnabled()) { - LOG.debug("Created Body Plan for Query Block " + qb.getId()); - } + LOG.debug("Created Body Plan for Query Block {}", qb.getId()); return curr; } @@ -11174,20 +11042,10 @@ private Operator genUnionPlan(String unionalias, String leftalias, .getColumnInfos())); // set union operator as child of each of leftOp and rightOp - List> child = - new ArrayList>(); - child.add(unionforward); - rightOp.setChildOperators(child); - - child = new ArrayList>(); - child.add(unionforward); - leftOp.setChildOperators(child); + rightOp.setChildOperators(Lists.newArrayList(unionforward)); + leftOp.setChildOperators(Lists.newArrayList(unionforward)); - List> parent = - new ArrayList>(); - parent.add(leftOp); - parent.add(rightOp); - unionforward.setParentOperators(parent); + unionforward.setParentOperators(Lists.newArrayList(leftOp, rightOp)); // create operator info list to return return putOpInsertMap(unionforward, unionoutRR); @@ -11285,9 +11143,6 @@ private Operator genUnionPlan(String unionalias, String leftalias, * The alias used for the table in the row resolver * @param rwsch * The row resolver used to resolve column references - * @param qbm - * The metadata information for the query block which is used to - * resolve unaliased columns * @param planExpr * The plan tree for the expression. If the user specified this, the * parse expressions are not used @@ -11296,8 +11151,7 @@ private Operator genUnionPlan(String unionalias, String leftalias, */ private ExprNodeDesc genSamplePredicate(TableSample ts, List bucketCols, boolean useBucketCols, String alias, - RowResolver rwsch, QBMetaData qbm, ExprNodeDesc planExpr, - int bucketingVersion) + RowResolver rwsch, ExprNodeDesc planExpr, int bucketingVersion) throws SemanticException { ExprNodeDesc numeratorExpr = new ExprNodeConstantDesc( @@ -11387,8 +11241,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { ColumnInfo colInfo = new ColumnInfo(fields.get(i).getFieldName(), TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i) .getFieldObjectInspector()), alias, false); - colInfo.setSkewedCol((isSkewedCol(alias, qb, fields.get(i) - .getFieldName())) ? true : false); + colInfo.setSkewedCol(isSkewedCol(alias, qb, fields.get(i).getFieldName())); rwsch.put(alias, fields.get(i).getFieldName(), colInfo); } } catch (SerDeException e) { @@ -11492,26 +11345,25 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { break; } - if (((ASTNode) sampleExprs.get(i).getChild(0)).getText() - .equalsIgnoreCase(tabBucketCols.get(j))) { + if ((sampleExprs.get(i).getChild(0)).getText().equalsIgnoreCase(tabBucketCols.get(j))) { colFound = true; } } - colsEqual = (colsEqual && colFound); + colsEqual = colFound; } // Check if input can be pruned - ts.setInputPruning((sampleExprs == null || sampleExprs.size() == 0 || colsEqual)); + ts.setInputPruning((sampleExprs.size() == 0 || colsEqual)); // check if input pruning is enough - if ((sampleExprs == null || sampleExprs.size() == 0 || colsEqual) + if ((sampleExprs.size() == 0 || colsEqual) && (num == den || (den % numBuckets == 0 || numBuckets % den == 0))) { // input pruning is enough; add the filter for the optimizer to use it // later LOG.info("No need for sample filter"); ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols, - colsEqual, alias, rwsch, qb.getMetaData(), null, + colsEqual, alias, rwsch, null, tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc( samplePredicate, true, new SampleDesc(ts.getNumerator(), @@ -11524,7 +11376,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // create tableOp to be filterDesc and set as child to 'top' LOG.info("Need sample filter"); ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols, - colsEqual, alias, rwsch, qb.getMetaData(), null, + colsEqual, alias, rwsch, null, tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc(samplePredicate, true); filterDesc.setGenerated(true); @@ -11556,7 +11408,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { tsSample.setInputPruning(true); qb.getParseInfo().setTabSample(alias, tsSample); ExprNodeDesc samplePred = genSamplePredicate(tsSample, tab - .getBucketCols(), true, alias, rwsch, qb.getMetaData(), null, + .getBucketCols(), true, alias, rwsch, null, tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc(samplePred, true, new SampleDesc(tsSample.getNumerator(), tsSample @@ -11576,7 +11428,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { .getFuncExprNodeDesc("rand", new ExprNodeConstantDesc(Integer .valueOf(460476415))); ExprNodeDesc samplePred = genSamplePredicate(tsSample, null, false, - alias, rwsch, qb.getMetaData(), randFunc, tab.getBucketingVersion()); + alias, rwsch, randFunc, tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc(samplePred, true); filterDesc.setGenerated(true); op = OperatorFactory.getAndMakeChild(filterDesc, @@ -11588,22 +11440,14 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { Operator output = putOpInsertMap(op, rwsch); - if (LOG.isDebugEnabled()) { - LOG.debug("Created Table Plan for " + alias + " " + op.toString()); - } + LOG.debug("Created Table Plan for {} {}", alias, op); return output; } - static boolean isSkewedCol(String alias, QB qb, String colName) { - boolean isSkewedCol = false; - List skewedCols = qb.getSkewedColumnNames(alias); - for (String skewedCol : skewedCols) { - if (skewedCol.equalsIgnoreCase(colName)) { - isSkewedCol = true; - } - } - return isSkewedCol; + boolean isSkewedCol(String alias, QB qb, String colName) { + return qb.getSkewedColumnNames(alias).stream() + .anyMatch(skewedCol -> skewedCol.equalsIgnoreCase(colName)); } private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String alias, @@ -11679,7 +11523,6 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String List partitions = qbp.getTableSpec().partitions; if (partitions != null) { for (Partition partn : partitions) { - // inputs.add(new ReadEntity(partn)); // is this needed at all? WriteEntity pwe = new WriteEntity(partn, WriteEntity.WriteType.DDL_NO_LOCK); pwe.setTxnAnalyze(true); outputs.add(pwe); @@ -11810,7 +11653,7 @@ private Operator genPlan(QB qb, boolean skipAmbiguityCheck) String dest = dests.iterator().next(); ASTNode whereClause = qb.getParseInfo().getWhrForClause(dest); if ( whereClause != null ) { - extractJoinCondsFromWhereClause(joinTree, qb, dest, + extractJoinCondsFromWhereClause(joinTree, (ASTNode) whereClause.getChild(0), aliasToOpInfo ); } @@ -11913,7 +11756,6 @@ private Path createDummyFile() throws SemanticException { * @param qb * @throws SemanticException */ - private void genLateralViewPlans(Map aliasToOpInfo, QB qb) throws SemanticException { Map> aliasToLateralViews = qb.getParseInfo().getAliasToLateralViews(); @@ -11933,8 +11775,7 @@ private void genLateralViewPlans(Map aliasToOpInfo, QB qb) // -> LateralViewJoinOperator // - Operator lateralViewJoin = genLateralViewPlan(qb, op, lateralViewTree); - op = lateralViewJoin; + op = genLateralViewPlan(qb, op, lateralViewTree); } e.setValue(op); } @@ -12057,7 +11898,6 @@ private void LVmergeRowResolvers(RowResolver source, RowResolver dest, @SuppressWarnings("nls") Phase1Ctx initPhase1Ctx() { - Phase1Ctx ctx_1 = new Phase1Ctx(); ctx_1.nextNum = 0; ctx_1.dest = "reduce"; @@ -12071,23 +11911,13 @@ public void init(boolean clearPartsCache) { reset(clearPartsCache); // init - QB qb = new QB(null, null, false); - this.qb = qb; + this.qb = new QB(null, null, false); } @Override @SuppressWarnings("nls") public void analyzeInternal(ASTNode ast) throws SemanticException { - analyzeInternal(ast, new PlannerContextFactory() { - @Override - public PlannerContext create() { - return new PlannerContext(); - } - }); - } - - protected static interface PlannerContextFactory { - PlannerContext create(); + analyzeInternal(ast, PlannerContext::new); } /** @@ -12127,8 +11957,7 @@ private Table getTableObjectByName(String tableName) throws HiveException { return getTableObjectByName(tableName, true); } - private static void walkASTAndQualifyNames(ASTNode ast, - Set cteAlias, Context ctx, Hive db, Set ignoredTokens, UnparseTranslator unparseTranslator) + private void walkASTAndQualifyNames(ASTNode ast, Set cteAlias, UnparseTranslator unparseTranslator) throws SemanticException { Queue queue = new LinkedList<>(); queue.add(ast); @@ -12144,7 +11973,7 @@ private static void walkASTAndQualifyNames(ASTNode ast, } } - if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) { + if (astNode.getChildCount() > 0 && !IGNORED_TOKENS.contains(astNode.getToken().getType())) { for (Node child : astNode.getChildren()) { queue.offer(child); } @@ -12181,25 +12010,22 @@ private String rewriteQueryWithQualifiedNames(ASTNode ast, TokenRewriteStream to throw new SemanticException("Duplicate definition of " + alias); } else { cteAlias.add(alias); - walkASTAndQualifyNames(ast, cteAlias, ctx, db, ignoredTokens, unparseTranslator); + walkASTAndQualifyNames(ast, cteAlias, unparseTranslator); } } // walk the other part of ast for (int index = 1; index < ast.getChildCount(); index++) { - walkASTAndQualifyNames(ast, cteAlias, ctx, db, ignoredTokens, unparseTranslator); + walkASTAndQualifyNames(ast, cteAlias, unparseTranslator); } } else { // there is no CTE, walk the whole AST - walkASTAndQualifyNames(ast, cteAlias, ctx, db, ignoredTokens, unparseTranslator); + walkASTAndQualifyNames(ast, cteAlias, unparseTranslator); } unparseTranslator.applyTranslations(tokenRewriteStream); - String rewrittenQuery = tokenRewriteStream.toString( - ast.getTokenStartIndex(), ast.getTokenStopIndex()); - return rewrittenQuery; + return tokenRewriteStream.toString(ast.getTokenStartIndex(), ast.getTokenStopIndex()); } - private void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set cteAlias, - Context ctx, Hive db, Map tabNameToTabObject, Set ignoredTokens) + private void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set cteAlias, Context ctx) throws SemanticException { Queue queue = new LinkedList<>(); queue.add(ast); @@ -12280,16 +12106,14 @@ private void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set cte new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView(), table.isNonNative())); } } - if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) { + if (astNode.getChildCount() > 0 && !IGNORED_TOKENS.contains(astNode.getToken().getType())) { for (Node child : astNode.getChildren()) { queue.offer(child); } } } - List basicPrivObjs = new ArrayList<>(); - basicPrivObjs.addAll(basicInfos.keySet()); - List needRewritePrivObjs = tableMask - .applyRowFilterAndColumnMasking(basicPrivObjs); + List basicPrivObjs = new ArrayList<>(basicInfos.keySet()); + List needRewritePrivObjs = tableMask.applyRowFilterAndColumnMasking(basicPrivObjs); if (needRewritePrivObjs != null && !needRewritePrivObjs.isEmpty()) { for (HivePrivilegeObject privObj : needRewritePrivObjs) { MaskAndFilterInfo info = basicInfos.get(privObj); @@ -12316,7 +12140,7 @@ private void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set cte } } - private static void extractColumnInfos(Table table, List colNames, List colTypes) { + private void extractColumnInfos(Table table, List colNames, List colTypes) { for (FieldSchema col : table.getAllCols()) { colNames.add(col.getName()); colTypes.add(col.getType()); @@ -12329,7 +12153,7 @@ private static void extractColumnInfos(Table table, List colNames, List< // For the replacement, we leverage the methods that are used for // unparseTranslator. private ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, TokenRewriteStream tokenRewriteStream, - Context ctx, Hive db, Map tabNameToTabObject, Set ignoredTokens) + Context ctx, Hive db, Map tabNameToTabObject) throws SemanticException { // 1. collect information about CTE if there is any. // The base table of CTE should be masked. @@ -12353,20 +12177,17 @@ private ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, To throw new SemanticException("Duplicate definition of " + alias); } else { cteAlias.add(alias); - walkASTMarkTABREF(tableMask, subq, cteAlias, - ctx, db, tabNameToTabObject, ignoredTokens); + walkASTMarkTABREF(tableMask, subq, cteAlias, ctx); } } // walk the other part of ast for (int index = 1; index < ast.getChildCount(); index++) { - walkASTMarkTABREF(tableMask, (ASTNode) ast.getChild(index), cteAlias, - ctx, db, tabNameToTabObject, ignoredTokens); + walkASTMarkTABREF(tableMask, (ASTNode) ast.getChild(index), cteAlias, ctx); } } // there is no CTE, walk the whole AST else { - walkASTMarkTABREF(tableMask, ast, cteAlias, - ctx, db, tabNameToTabObject, ignoredTokens); + walkASTMarkTABREF(tableMask, ast, cteAlias, ctx); } // 2. rewrite the AST, replace TABREF with masking/filtering if (tableMask.needsRewrite()) { @@ -12374,9 +12195,6 @@ private ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, To String rewrittenQuery = tokenRewriteStream.toString( ast.getTokenStartIndex(), ast.getTokenStopIndex()); ASTNode rewrittenTree; - // Parse the rewritten query string - // check if we need to ctx.setCmd(rewrittenQuery); - ParseDriver pd = new ParseDriver(); try { rewrittenTree = ParseUtils.parse(rewrittenQuery); } catch (ParseException e) { @@ -12538,21 +12356,22 @@ private void removeOBInSubQuery(QBExpr qbExpr) { } } - private static void removeASTChild(ASTNode node) { - Tree parent = node.getParent(); - if (parent != null) { - parent.deleteChild(node.getChildIndex()); - node.setParent(null); - } + private void removeASTChild(ASTNode node) { + Optional.ofNullable(node.getParent()) + .ifPresent(parent -> { + parent.deleteChild(node.getChildIndex()); + node.setParent(null); + }); } - void analyzeInternal(ASTNode ast, PlannerContextFactory pcf) throws SemanticException { + @SuppressWarnings("checkstyle:methodlength") + void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticException { LOG.info("Starting Semantic Analysis"); // 1. Generate Resolved Parse tree from syntax tree boolean needsTransform = needsTransform(); //change the location of position alias process here processPositionAlias(ast); - PlannerContext plannerCtx = pcf.create(); + PlannerContext plannerCtx = pcf.get(); if (!genResolvedParseTree(ast, plannerCtx)) { return; } @@ -12606,10 +12425,10 @@ void analyzeInternal(ASTNode ast, PlannerContextFactory pcf) throws SemanticExce (tableMask.isEnabled() && analyzeRewrite == null)) { // Here we rewrite the * and also the masking table ASTNode rewrittenAST = rewriteASTWithMaskAndFilter(tableMask, astForMasking, ctx.getTokenRewriteStream(), - ctx, db, tabNameToTabObject, ignoredTokens); + ctx, db, tabNameToTabObject); if (astForMasking != rewrittenAST) { usesMasking = true; - plannerCtx = pcf.create(); + plannerCtx = pcf.get(); ctx.setSkipTableMasking(true); init(true); //change the location of position alias process here @@ -12760,7 +12579,7 @@ void analyzeInternal(ASTNode ast, PlannerContextFactory pcf) throws SemanticExce fetchTask = pCtx.getFetchTask(); } //find all Acid FileSinkOperatorS - QueryPlanPostProcessor qp = new QueryPlanPostProcessor(rootTasks, acidFileSinks, ctx.getExecutionId()); + new QueryPlanPostProcessor(rootTasks, acidFileSinks, ctx.getExecutionId()); // 10. Attach CTAS/Insert-Commit-hooks for Storage Handlers final Optional optionalTezTask = @@ -12792,7 +12611,6 @@ void analyzeInternal(ASTNode ast, PlannerContextFactory pcf) throws SemanticExce // requires SemanticAnalyzer state to be reset. if (checkResultsCache(lookupInfo, true)) { LOG.info("Cached result found on second lookup"); - return; } else { QueryResultsCache.QueryInfo queryInfo = createCacheQueryInfoForQuery(lookupInfo); @@ -12881,8 +12699,7 @@ protected void saveViewDefinition() throws SemanticException { boolean first = true; StringBuilder sb = new StringBuilder(); sb.append("SELECT "); - for (int i = 0; i < derivedSchema.size(); ++i) { - FieldSchema fieldSchema = derivedSchema.get(i); + for (FieldSchema fieldSchema : derivedSchema) { if (!createVwDesc.getPartColNames().contains(fieldSchema.getName())) { if (first) { first = false; @@ -12989,7 +12806,7 @@ protected void saveViewDefinition() throws SemanticException { createVwDesc.setViewExpandedText(expandedText); } - private Set getTablesUsed(ParseContext parseCtx) throws SemanticException { + private Set getTablesUsed(ParseContext parseCtx) { Set tablesUsed = new HashSet<>(); for (TableScanOperator topOp : parseCtx.getTopOps().values()) { Table table = topOp.getConf().getTableMetadata(); @@ -13001,14 +12818,13 @@ protected void saveViewDefinition() throws SemanticException { return tablesUsed; } - private static List convertRowSchemaToViewSchema(RowResolver rr) throws SemanticException { + private List convertRowSchemaToViewSchema(RowResolver rr) throws SemanticException { List fieldSchema = convertRowSchemaToResultSetSchema(rr, false); ParseUtils.validateColumnNameUniqueness(fieldSchema); return fieldSchema; } - static List convertRowSchemaToResultSetSchema(RowResolver rr, - boolean useTabAliasIfAvailable) { + List convertRowSchemaToResultSetSchema(RowResolver rr, boolean useTabAliasIfAvailable) { List fieldSchemas = new ArrayList(); String[] qualifiedColName; String colName; @@ -13050,13 +12866,7 @@ ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, return genExprNodeDesc(expr, input, tcCtx); } - - ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean useCaching) - throws SemanticException { - return genExprNodeDesc(expr, input, useCaching, false); - } - - private ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean useCaching, + ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean useCaching, boolean foldExpr) throws SemanticException { TypeCheckCtx tcCtx = new TypeCheckCtx(input, useCaching, foldExpr); return genExprNodeDesc(expr, input, tcCtx); @@ -13148,10 +12958,7 @@ private ExprNodeDesc getExprNodeDescCached(ASTNode expr, RowResolver input) } else { errMsg = tcCtx.getError(); } - if (errMsg == null) { - errMsg = "Error in parsing "; - } - throw new SemanticException(errMsg); + throw new SemanticException(Optional.ofNullable(errMsg).orElse("Error in parsing ")); } if (desc instanceof ExprNodeColumnListDesc) { throw new SemanticException("TOK_ALLCOLREF is not supported in current context"); @@ -13162,7 +12969,6 @@ private ExprNodeDesc getExprNodeDescCached(ASTNode expr, RowResolver input) return nodeOutputs; } - Map nodeToText = new HashMap<>(); List fieldDescList = new ArrayList<>(); for (Map.Entry entry : nodeOutputs.entrySet()) { @@ -13192,7 +12998,6 @@ private ExprNodeDesc getExprNodeDescCached(ASTNode expr, RowResolver input) replacementText.append(HiveUtils.unparseIdentifier(tmp[0], conf)); replacementText.append("."); replacementText.append(HiveUtils.unparseIdentifier(tmp[1], conf)); - nodeToText.put(columnDesc, replacementText.toString()); unparseTranslator.addTranslation(node, replacementText.toString()); } @@ -13350,12 +13155,8 @@ private void updateDefaultTblProps(Map source, Map tblProp, boolean isExt, StorageFormat storageFormat, String qualifiedTableName, List sortCols, boolean isMaterialization, boolean isTemporaryTable, boolean isTransactional) throws SemanticException { - Map retValue; - if (tblProp == null) { - retValue = new HashMap(); - } else { - retValue = tblProp; - } + Map retValue = Optional.ofNullable(tblProp).orElseGet(HashMap::new); + String paraString = HiveConf.getVar(conf, ConfVars.NEWTABLEDEFAULTPARA); if (paraString != null && !paraString.isEmpty()) { for (String keyValuePair : paraString.split(",")) { @@ -14349,7 +14150,7 @@ public QB getQB() { return qb; } - public void setQB(QB qb) { + void setQB(QB qb) { this.qb = qb; } @@ -14469,9 +14270,8 @@ private void processPTF(QB qb, ASTNode ptf) throws SemanticException{ PartitionedTableFunctionSpec ptfSpec = processPTFChain(qb, ptf); - if ( ptfSpec.getAlias() != null ) { - qb.addAlias(ptfSpec.getAlias()); - } + Optional.ofNullable(ptfSpec.getAlias()) + .ifPresent(qb::addAlias); PTFInvocationSpec spec = new PTFInvocationSpec(); spec.setFunction(ptfSpec); @@ -14481,17 +14281,16 @@ private void processPTF(QB qb, ASTNode ptf) throws SemanticException{ private void handleQueryWindowClauses(QB qb, Phase1Ctx ctx_1, ASTNode node) throws SemanticException { WindowingSpec spec = qb.getWindowingSpec(ctx_1.dest); - for(int i=0; i < node.getChildCount(); i++) { - processQueryWindowClause(spec, (ASTNode) node.getChild(i)); + for(Node child : node.getChildren()) { + processQueryWindowClause(spec, (ASTNode) child); } } private PartitionSpec processPartitionSpec(ASTNode node) { PartitionSpec pSpec = new PartitionSpec(); - int exprCnt = node.getChildCount(); - for(int i=0; i < exprCnt; i++) { + for(Node child : node.getChildren()) { PartitionExpression exprSpec = new PartitionExpression(); - exprSpec.setExpression((ASTNode) node.getChild(i)); + exprSpec.setExpression((ASTNode) child); pSpec.addExpression(exprSpec); } return pSpec; @@ -14526,8 +14325,6 @@ private PartitioningSpec processPTFPartitionSpec(ASTNode pSpecNode) PartitioningSpec partitioning = new PartitioningSpec(); ASTNode firstChild = (ASTNode) pSpecNode.getChild(0); int type = firstChild.getType(); - int exprCnt; - if ( type == HiveParser.TOK_DISTRIBUTEBY || type == HiveParser.TOK_CLUSTERBY ) { @@ -14541,8 +14338,7 @@ private PartitioningSpec processPTFPartitionSpec(ASTNode pSpecNode) } } else if ( type == HiveParser.TOK_SORTBY || type == HiveParser.TOK_ORDERBY ) { - ASTNode sortNode = firstChild; - OrderSpec oSpec = processOrderSpec(sortNode); + OrderSpec oSpec = processOrderSpec(firstChild); partitioning.setOrderSpec(oSpec); } return partitioning; @@ -14572,8 +14368,7 @@ private WindowFunctionSpec processWindowFunction(ASTNode node, ASTNode wsNode) } if ( wsNode != null ) { - WindowSpec ws = processWindowSpec(wsNode); - wfSpec.setWindowSpec(ws); + wfSpec.setWindowSpec(processWindowSpec(wsNode)); } return wfSpec; @@ -14616,11 +14411,6 @@ private void processQueryWindowClause(WindowingSpec spec, ASTNode node) } private WindowSpec processWindowSpec(ASTNode node) throws SemanticException { - String sourceId = null; - PartitionSpec partition = null; - OrderSpec order = null; - WindowFrameSpec windowFrame = null; - boolean hasSrcId = false, hasPartSpec = false, hasWF = false; int srcIdIdx = -1, partIdx = -1, wfIdx = -1; @@ -14667,22 +14457,22 @@ private WindowSpec processWindowSpec(ASTNode node) throws SemanticException { private WindowFrameSpec processWindowFrame(ASTNode node) throws SemanticException { int type = node.getType(); - BoundarySpec start = null, end = null; + BoundarySpec end = null; /* * A WindowFrame may contain just the Start Boundary or in the * between style of expressing a WindowFrame both boundaries * are specified. */ - start = processBoundary(type, (ASTNode) node.getChild(0)); + BoundarySpec start = processBoundary((ASTNode) node.getChild(0)); if ( node.getChildCount() > 1 ) { - end = processBoundary(type, (ASTNode) node.getChild(1)); + end = processBoundary((ASTNode) node.getChild(1)); } // Note: TOK_WINDOWVALUES means RANGE type, TOK_WINDOWRANGE means ROWS type return new WindowFrameSpec(type == HiveParser.TOK_WINDOWVALUES ? WindowType.RANGE : WindowType.ROWS, start, end); } - private BoundarySpec processBoundary(int frameType, ASTNode node) throws SemanticException { + private BoundarySpec processBoundary(ASTNode node) throws SemanticException { BoundarySpec bs = new BoundarySpec(); int type = node.getType(); boolean hasAmt = true; @@ -14699,6 +14489,8 @@ private BoundarySpec processBoundary(int frameType, ASTNode node) throws Semant bs.setDirection(Direction.CURRENT); hasAmt = false; break; + default: + // no-op } if ( hasAmt ) @@ -14722,111 +14514,12 @@ private BoundarySpec processBoundary(int frameType, ASTNode node) throws Semant return bs; } - /* - * check if a Select Expr is a constant. - * - current logic used is to look for HiveParser.TOK_TABLE_OR_COL - * - if there is none then the expression is a constant. - */ - private static class ConstantExprCheck implements ContextVisitor { - boolean isConstant = true; - - @Override - public void visit(Object t, Object parent, int childIndex, Map labels) { - if ( !isConstant ) { - return; - } - ASTNode node = (ASTNode) t; - if (ParseDriver.adaptor.getType(t) == HiveParser.TOK_TABLE_OR_COL ) { - isConstant = false; - } - } - - public void reset() { - isConstant = true; - } - - protected boolean isConstant() { - return isConstant; - } - } - - private static class AggregationExprCheck implements ContextVisitor { - Map destAggrExprs; - boolean isAggr = false; - - public AggregationExprCheck(Map destAggrExprs) { - super(); - this.destAggrExprs = destAggrExprs; - } - - @Override - public void visit(Object t, Object parent, int childIndex, Map labels) { - if ( isAggr ) { - return; - } - if ( destAggrExprs.values().contains(t)) { - isAggr = true; - } - } - - public void reset() { - isAggr = false; - } - - protected boolean isAggr() { - return isAggr; - } - } - - /* - * Returns false if there is a SelectExpr that is not a constant or an aggr. - * - */ - private boolean isValidGroupBySelectList(QB currQB, String clause){ - ConstantExprCheck constantExprCheck = new ConstantExprCheck(); - AggregationExprCheck aggrExprCheck = new AggregationExprCheck( - currQB.getParseInfo().getAggregationExprsForClause(clause)); - - TreeWizard tw = new TreeWizard(ParseDriver.adaptor, HiveParser.tokenNames); - ASTNode selectNode = currQB.getParseInfo().getSelForClause(clause); - - /* - * for Select Distinct Queries we don't move any aggregations. - */ - if ( selectNode != null && selectNode.getType() == HiveParser.TOK_SELECTDI ) { - return true; - } - - for (int i = 0; selectNode != null && i < selectNode.getChildCount(); i++) { - ASTNode selectExpr = (ASTNode) selectNode.getChild(i); - //check for QUERY_HINT expressions on ast - if(selectExpr.getType() != HiveParser.TOK_SELEXPR){ - continue; - } - - constantExprCheck.reset(); - PTFTranslator.visit(selectExpr.getChild(0), constantExprCheck); - - if ( !constantExprCheck.isConstant() ) { - aggrExprCheck.reset(); - PTFTranslator.visit(selectExpr.getChild(0), aggrExprCheck); - if (!aggrExprCheck.isAggr() ) { - return false; - } - } - - } - return true; - } - //--------------------------- PTF handling: PTFInvocationSpec to PTFDesc -------------------------- private PTFDesc translatePTFInvocationSpec(PTFInvocationSpec ptfQSpec, RowResolver inputRR) - throws SemanticException{ - PTFDesc ptfDesc = null; + throws SemanticException { PTFTranslator translator = new PTFTranslator(); - ptfDesc = translator.translate(ptfQSpec, this, conf, inputRR, unparseTranslator); - return ptfDesc; + return translator.translate(ptfQSpec, this, conf, inputRR, unparseTranslator); } private Operator genPTFPlan(PTFInvocationSpec ptfQSpec, Operator input) throws SemanticException { @@ -14834,9 +14527,7 @@ private Operator genPTFPlan(PTFInvocationSpec ptfQSpec, Operator input) throws S for (PTFInvocationSpec ptfSpec : componentQueries) { input = genPTFPlanForComponentQuery(ptfSpec, input); } - if (LOG.isDebugEnabled()) { - LOG.debug("Created PTF Plan "); - } + LOG.debug("Created PTF Plan "); return input; } @@ -14845,15 +14536,12 @@ private Operator genPTFPlan(PTFInvocationSpec ptfQSpec, Operator input) throws S * Construct the data structures containing ExprNodeDesc for partition * columns and order columns. Use the input definition to construct the list * of output columns for the ReduceSinkOperator - * - * @throws SemanticException */ private void buildPTFReduceSinkDetails(PartitionedTableFunctionDef tabDef, - RowResolver inputRR, List partCols, List orderCols, StringBuilder orderString, - StringBuilder nullOrderString) throws SemanticException { + StringBuilder nullOrderString) { List partColList = tabDef.getPartition().getExpressions(); @@ -14875,8 +14563,7 @@ private void buildPTFReduceSinkDetails(PartitionedTableFunctionDef tabDef, * ReduceSinkDesc */ List orderColList = tabDef.getOrder().getExpressions(); - for (int i = 0; i < orderColList.size(); i++) { - OrderExpressionDef colDef = orderColList.get(i); + for (OrderExpressionDef colDef : orderColList) { char orderChar = colDef.getOrder() == PTFInvocationSpec.Order.ASC ? '+' : '-'; char nullOrderChar = colDef.getNullOrder() == PTFInvocationSpec.NullOrder.NULLS_FIRST ? 'a' : 'z'; int index = ExprNodeDescUtils.indexOf(colDef.getExprNode(), orderCols); @@ -14938,7 +14625,7 @@ private Operator genPTFPlanForComponentQuery(PTFInvocationSpec ptfQSpec, Operato * If the parent of ReduceSinkOperator is PTFOperator, use it's * output RR. */ - buildPTFReduceSinkDetails(tabDef, rr, partCols, orderCols, orderString, nullOrderString); + buildPTFReduceSinkDetails(tabDef, partCols, orderCols, orderString, nullOrderString); input = genReduceSinkPlan(input, partCols, orderCols, orderString.toString(), nullOrderString.toString(), -1, Operation.NOT_ACID); } @@ -15119,15 +14806,16 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo, } RowResolver parentRR = opParseCtx.get(parent).getRowResolver(); try { - ColumnInfo pColInfo = parentRR.get(tab_alias, col_alias); - tab_alias = pColInfo == null ? null : pColInfo.getTabAlias(); - } catch(SemanticException se) { + tab_alias = Optional.ofNullable(parentRR.get(null, col_alias)) + .map(ColumnInfo::getTabAlias) + .orElse(null); + } catch (SemanticException se) { } gByRR.put(tab_alias, col_alias, colInfo); } } - private WriteEntity.WriteType determineWriteType(LoadTableDesc ltd, boolean isNonNativeTable, String dest) { + private WriteEntity.WriteType determineWriteType(LoadTableDesc ltd, String dest) { if (ltd == null) { return WriteEntity.WriteType.INSERT_OVERWRITE; @@ -15142,13 +14830,7 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo, (deleting(dest) ? WriteEntity.WriteType.DELETE : WriteEntity.WriteType.INSERT); } private boolean isAcidOutputFormat(Class of) { - Class[] interfaces = of.getInterfaces(); - for (Class iface : interfaces) { - if (iface.equals(AcidOutputFormat.class)) { - return true; - } - } - return false; + return Arrays.asList(of.getInterfaces()).contains(AcidOutputFormat.class); } // Note that this method assumes you have already decided this is an Acid table. It cannot @@ -15185,7 +14867,7 @@ private void checkAcidTxnManager(Table table) throws SemanticException { } } - static ASTNode genSelectDIAST(RowResolver rr) { + ASTNode genSelectDIAST(RowResolver rr) { Map> map = rr.getRslvMap(); ASTNode selectDI = new ASTNode(SELECTDI_TOKEN); // Note: this will determine the order of columns in the result. For now, the columns for each @@ -15198,7 +14880,8 @@ static ASTNode genSelectDIAST(RowResolver rr) { } return selectDI; } - private static ASTNode buildSelExprSubTree(String tableAlias, String col) { + + private ASTNode buildSelExprSubTree(String tableAlias, String col) { tableAlias = StringInternUtils.internIfNotNull(tableAlias); col = StringInternUtils.internIfNotNull(col); ASTNode selexpr = new ASTNode(SELEXPR_TOKEN); @@ -15224,19 +14907,15 @@ private void copyInfoToQueryProperties(QueryProperties queryProperties) { queryProperties.setMaterializedView(qb.getViewDesc() != null); } } + private void warn(String msg) { - SessionState.getConsole().printInfo( - String.format("Warning: %s", msg)); + SessionState.getConsole().printInfo(String.format("Warning: %s", msg)); } public List getLoadFileWork() { return loadFileWork; } - public void setLoadFileWork(List loadFileWork) { - this.loadFileWork = loadFileWork; - } - private String getQueryStringFromAst(ASTNode ast) { StringBuilder sb = new StringBuilder(); int startIdx = ast.getTokenStartIndex(); @@ -15275,12 +14954,10 @@ private String getQueryStringForCache(ASTNode ast) { // Re-using the TokenRewriteStream map for views so we do not overwrite the current TokenRewriteStream String rewriteStreamName = "__qualified_query_string__"; - ASTNode astNode; try { - astNode = ParseUtils.parse(queryString, ctx, rewriteStreamName); + ASTNode astNode = ParseUtils.parse(queryString, ctx, rewriteStreamName); TokenRewriteStream tokenRewriteStream = ctx.getViewTokenRewriteStream(rewriteStreamName); - String fullyQualifiedQuery = rewriteQueryWithQualifiedNames(astNode, tokenRewriteStream); - return fullyQualifiedQuery; + return rewriteQueryWithQualifiedNames(astNode, tokenRewriteStream); } catch (Exception err) { LOG.error("Unexpected error while reparsing the query string [" + queryString + "]", err); // Don't fail the query - just return null (caller should skip cache lookup). @@ -15294,8 +14971,8 @@ private ValidTxnWriteIdList getQueryValidTxnWriteIdList() throws SemanticExcepti // List transactionalTables = tablesFromReadEntities(inputs) .stream() - .filter(table -> AcidUtils.isTransactionalTable(table)) - .map(table -> table.getFullyQualifiedName()) + .filter(AcidUtils::isTransactionalTable) + .map(Table::getFullyQualifiedName) .collect(Collectors.toList()); if (transactionalTables.size() > 0) { try { @@ -15317,7 +14994,7 @@ private ValidTxnWriteIdList getQueryValidTxnWriteIdList() throws SemanticExcepti String queryString = getQueryStringForCache(astNode); if (queryString != null) { ValidTxnWriteIdList writeIdList = getQueryValidTxnWriteIdList(); - lookupInfo = new QueryResultsCache.LookupInfo(queryString, () -> { return writeIdList; }); + lookupInfo = new QueryResultsCache.LookupInfo(queryString, () -> writeIdList); } return lookupInfo; } @@ -15387,11 +15064,7 @@ private boolean queryTypeCanUseCache() { } // HIVE-19096 - disable for explain analyze - if (ctx.getExplainAnalyze() != null) { - return false; - } - - return true; + return ctx.getExplainAnalyze() == null; } private boolean needsTransform() { @@ -15445,10 +15118,10 @@ private boolean queryCanBeCached() { return true; } - private static Set tablesFromReadEntities(Set readEntities) { + private Set
tablesFromReadEntities(Set readEntities) { return readEntities.stream() .filter(entity -> entity.getType() == Entity.Type.TABLE) - .map(entity -> entity.getTable()) + .map(Entity::getTable) .collect(Collectors.toSet()); } @@ -15576,7 +15249,7 @@ protected String getFullTableNameForSQL(ASTNode n) throws SemanticException { } } - protected static IllegalArgumentException raiseWrongType(String expectedTokName, ASTNode n) { + static IllegalArgumentException raiseWrongType(String expectedTokName, ASTNode n) { return new IllegalArgumentException("Expected " + expectedTokName + "; got " + n.getType()); }