diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java index fce5924..4ea877e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java @@ -61,6 +61,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; public class RelOptHiveTable extends RelOptAbstractTable { private final Table hiveTblMetadata; @@ -259,7 +260,7 @@ public void computePartitionList(HiveConf conf, RexNode pruneNode) { } } - private void updateColStats(Set projIndxLst) { + private void updateColStats(Set projIndxLst, boolean allowNullColumnForMissingStats) { List nonPartColNamesThatRqrStats = new ArrayList(); List nonPartColIndxsThatRqrStats = new ArrayList(); List partColNamesThatRqrStats = new ArrayList(); @@ -372,9 +373,13 @@ private void updateColStats(Set projIndxLst) { if (!colNamesFailedStats.isEmpty()) { String logMsg = "No Stats for " + hiveTblMetadata.getCompleteName() + ", Columns: " + getColNamesForLogging(colNamesFailedStats); - LOG.error(logMsg); noColsMissingStats.getAndAdd(colNamesFailedStats.size()); - throw new RuntimeException(logMsg); + if (allowNullColumnForMissingStats) { + LOG.warn(logMsg); + } else { + LOG.error(logMsg); + throw new RuntimeException(logMsg); + } } } @@ -387,10 +392,14 @@ private int getDistinctCount(Set partitions, String partColName) { } public List getColStat(List projIndxLst) { - ImmutableList.Builder colStatsBldr = ImmutableList. builder(); + return getColStat(projIndxLst, false); + } + + public List getColStat(List projIndxLst, boolean allowNullColumnForMissingStats) { + List colStatsBldr = Lists.newArrayList(); if (projIndxLst != null) { - updateColStats(new HashSet(projIndxLst)); + updateColStats(new HashSet(projIndxLst), allowNullColumnForMissingStats); for (Integer i : projIndxLst) { colStatsBldr.add(hiveColStatsMap.get(i)); } @@ -399,13 +408,13 @@ private int getDistinctCount(Set partitions, String partColName) { for (Integer i = 0; i < noOfNonVirtualCols; i++) { pILst.add(i); } - updateColStats(new HashSet(pILst)); + updateColStats(new HashSet(pILst), allowNullColumnForMissingStats); for (Integer pi : pILst) { colStatsBldr.add(hiveColStatsMap.get(pi)); } } - return colStatsBldr.build(); + return colStatsBldr; } /* diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java index b5d0bf1..3224039 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java @@ -17,9 +17,7 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.stats; -import java.util.HashSet; import java.util.List; -import java.util.Set; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; @@ -54,20 +52,19 @@ private HiveRelMdSize() {} public List averageColumnSizes(HiveTableScan scan) { List neededcolsLst = scan.getNeededColIndxsFrmReloptHT(); - Set needColsSet = new HashSet(neededcolsLst); List columnStatistics = ((RelOptHiveTable) scan.getTable()) - .getColStat(neededcolsLst); + .getColStat(neededcolsLst, true); // Obtain list of col stats, or use default if they are not available final ImmutableList.Builder list = ImmutableList.builder(); int indxRqdCol = 0; int nFields = scan.getRowType().getFieldCount(); for (int i = 0; i < nFields; i++) { - if (needColsSet.contains(i)) { + if (neededcolsLst.contains(i)) { ColStatistics columnStatistic = columnStatistics.get(indxRqdCol); indxRqdCol++; if (columnStatistic == null) { - RelDataTypeField field = scan.getPrunedRowType().getFieldList().get(i); + RelDataTypeField field = scan.getRowType().getFieldList().get(i); list.add(averageTypeValueSize(field.getType())); } else { list.add(columnStatistic.getAvgColLen());