diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 9c900af..b855b5c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -37,6 +38,7 @@ import javax.jdo.Transaction; import javax.jdo.datastore.JDOConnection; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -361,7 +363,7 @@ public Database getDatabase(String dbName) throws MetaException{ } return runBatched(partNames, new Batchable() { public List run(List input) throws MetaException { - String filter = "\"PARTITIONS\".\"PART_NAME\" in (" + makeParams(input.size()) + ")"; + String filter = input.size() == 0 ? "" : "\"PARTITIONS\".\"PART_NAME\" in (" + makeParams(input.size()) + ")"; return getPartitionsViaSqlFilterInternal(dbName, tblName, null, filter, input, new ArrayList(), null); } @@ -494,7 +496,7 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException // Get full objects. For Oracle/etc. do it in batches. List result = runBatched(sqlResult, new Batchable() { public List run(List input) throws MetaException { - return getPartitionsFromPartitionIds(dbNameLcase, tblNameLcase, isView, input); + return getPartitionsFromPartitionIds(dbNameLcase, tblNameLcase, isView, input, null, null); } }); @@ -502,19 +504,22 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException return result; } - /** Should be called with the list short enough to not trip up Oracle/etc. */ + /** Should be called with the list short enough to not trip up Oracle/etc. + * @param sqlFilter + * @param params */ private List getPartitionsFromPartitionIds(String dbName, String tblName, - Boolean isView, List partIdList) throws MetaException { + Boolean isView, List partIdList, String sqlFilter, List params) throws MetaException { boolean doTrace = LOG.isDebugEnabled(); - int idStringWidth = (int)Math.ceil(Math.log10(partIdList.size())) + 1; // 1 for comma - int sbCapacity = partIdList.size() * idStringWidth; + int idStringWidth = partIdList == null ? 0: ((int)Math.ceil(Math.log10(partIdList.size())) + 1); // 1 for comma + int sbCapacity = partIdList == null ? 0 : (partIdList.size() * idStringWidth); // Prepare StringBuilder for "PART_ID in (...)" to use in future queries. - StringBuilder partSb = new StringBuilder(sbCapacity); - for (Object partitionId : partIdList) { - partSb.append(extractSqlLong(partitionId)).append(","); + StringBuilder partSb = partIdList == null ? new StringBuilder() : new StringBuilder(sbCapacity); + if (partIdList != null) { + for (Object partitionId : partIdList) { + partSb.append(extractSqlLong(partitionId)).append(","); + } } - String partIds = trimCommaList(partSb); - + String partIds = partIdList == null ? null : trimCommaList(partSb); // Get most of the fields for the IDs provided. // Assume db and table names are the same for all partition, as provided in arguments. String queryText = @@ -526,10 +531,12 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException + "from \"PARTITIONS\"" + " left outer join \"SDS\" on \"PARTITIONS\".\"SD_ID\" = \"SDS\".\"SD_ID\" " + " left outer join \"SERDES\" on \"SDS\".\"SERDE_ID\" = \"SERDES\".\"SERDE_ID\" " - + "where \"PART_ID\" in (" + partIds + ") order by \"PART_NAME\" asc"; + + (partIds == null ? "" : (" where \"PART_ID\" in (" + partIds + ")")) + + (sqlFilter == null ? "" : (" where " + sqlFilter)) + + " order by \"PART_NAME\" asc"; long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - List sqlResult = executeWithArray(query, null, queryText); + List sqlResult = executeWithArray(query, params == null ? null : params.toArray(), queryText); long queryTime = doTrace ? System.nanoTime() : 0; Deadline.checkTimeout(); @@ -539,7 +546,7 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException TreeMap serdes = new TreeMap(); TreeMap> colss = new TreeMap>(); // Keep order by name, consistent with JDO. - ArrayList orderedResult = new ArrayList(partIdList.size()); + ArrayList orderedResult = partIdList != null ? new ArrayList(partIdList.size()) : new ArrayList(); // Prepare StringBuilder-s for "in (...)" lists to use in one-to-many queries. StringBuilder sdSb = new StringBuilder(sbCapacity), serdeSb = new StringBuilder(sbCapacity); @@ -629,7 +636,9 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException // Now get all the one-to-many things. Start with partitions. queryText = "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from \"PARTITION_PARAMS\"" - + " where \"PART_ID\" in (" + partIds + ") and \"PARAM_KEY\" is not null" + + (partIds == null ? " where " : (" where \"PART_ID\" in (" + partIds + ") and ")) + + (sqlFilter == null ? "" : (" where " + sqlFilter)) + + " \"PARAM_KEY\" is not null" + " order by \"PART_ID\" asc"; loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc() { @Override @@ -642,7 +651,9 @@ public void apply(Partition t, Object[] fields) { } queryText = "select \"PART_ID\", \"PART_KEY_VAL\" from \"PARTITION_KEY_VALS\"" - + " where \"PART_ID\" in (" + partIds + ") and \"INTEGER_IDX\" >= 0" + + (partIds == null ? " where " : (" where \"PART_ID\" in (" + partIds + ") and")) + + (sqlFilter == null ? "" : sqlFilter) + + " \"INTEGER_IDX\" >= 0" + " order by \"PART_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc() { @Override @@ -984,7 +995,7 @@ private static String generateSqlFilter(Table table, ExpressionTree tree, List joins, boolean dbHasJoinCastBug, String defaultPartName, DB dbType) throws MetaException { assert table != null; - if (tree.getRoot() == null) { + if (tree == null || tree.getRoot() == null) { return ""; } PartitionFilterGenerator visitor = new PartitionFilterGenerator( @@ -1358,12 +1369,13 @@ private long partsFoundForPartitions(final String dbName, final String tableName // Extrapolation is not needed. if (areAllPartsFound) { queryText = commonPrefix + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" - + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + + (partNames != null ? (" and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")") : "") + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); qResult = executeWithArray(query, prepareParams(dbName, tableName, partNames, colNames), queryText); + if (qResult == null) { query.closeAll(); return Lists.newArrayList(); @@ -1612,17 +1624,18 @@ private ColumnStatisticsObj prepareCSObjWithAdjustedNDV(Object[] row, int i, private Object[] prepareParams(String dbName, String tableName, List partNames, List colNames) throws MetaException { - Object[] params = new Object[colNames.size() + partNames.size() + 2]; + Object[] params = new Object[colNames.size() + (partNames == null ? 0 : partNames.size()) + 2]; int paramI = 0; params[paramI++] = dbName; params[paramI++] = tableName; for (String colName : colNames) { params[paramI++] = colName; } - for (String partName : partNames) { - params[paramI++] = partName; + if (partNames != null) { + for (String partName : partNames) { + params[paramI++] = partName; + } } - return params; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 26e936e..d28cd34 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -237,7 +237,7 @@ private static PrunedPartitionList getAllPartsFromCacheOrServer(Table tab, Strin } catch (HiveException e) { throw new SemanticException(e); } - ppList = new PrunedPartitionList(tab, parts, null, unknownPartitions); + ppList = new PrunedPartitionList(tab, parts, null, unknownPartitions, true); if (partsCache != null) { partsCache.put(key, ppList); } @@ -391,7 +391,7 @@ static private ExprNodeDesc removeNonPartCols(ExprNodeDesc expr, List pa * @param expr Expression. * @return True iff expr contains any non-native user-defined functions. */ - static private boolean hasUserFunctions(ExprNodeDesc expr) { + static public boolean hasUserFunctions(ExprNodeDesc expr) { if (!(expr instanceof ExprNodeGenericFuncDesc)) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java index da2e1e2..921c6eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PrunedPartitionList.java @@ -41,6 +41,9 @@ /** Whether there are partitions in the list that may or may not satisfy the criteria. */ private boolean hasUnknowns; + + /** To determine whether there are all partitions, may have false negatives */ + private boolean hasAllPartitions; public PrunedPartitionList(Table source, Set partitions, List referred, boolean hasUnknowns) { @@ -50,6 +53,23 @@ public PrunedPartitionList(Table source, Set partitions, List this.hasUnknowns = hasUnknowns; } + public PrunedPartitionList(Table source, Set partitions, List referred, + boolean hasUnknowns, boolean hasAllPartitions) { + this.source = source; + this.referred = referred; + this.partitions = partitions; + this.hasUnknowns = hasUnknowns; + this.hasAllPartitions = hasAllPartitions; + } + + public boolean getHasAllPartitions(){ + return hasAllPartitions; + } + + public void setHasAllPartitions(boolean value) { + hasAllPartitions = value; + } + public Table getSourceTable() { return source; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index d8acf94..d946803 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -260,9 +260,12 @@ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList pa stats.setBasicStatsState(State.PARTIAL); } if (fetchColStats) { - List partNames = new ArrayList(partList.getNotDeniedPartns().size()); - for (Partition part : partList.getNotDeniedPartns()) { - partNames.add(part.getName()); + List partNames = null; + if (!partList.getHasAllPartitions()) { + partNames = new ArrayList(partList.getNotDeniedPartns().size()); + for (Partition part : partList.getNotDeniedPartns()) { + partNames.add(part.getName()); + } } neededColumns = processNeededColumns(schema, neededColumns); AggrStats aggrStats = null;