diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index be7ed32..86565a4 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -399,7 +399,12 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String msdb.openTransaction(); oldPart = msdb.getPartition(dbname, name, new_part.getValues()); if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) { - MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext); + // if stats are same, no need to update + if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) { + MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters()); + } else { + MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext); + } } updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part); @@ -627,7 +632,12 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String partValsList.add(tmpPart.getValues()); if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl, environmentContext)) { - MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext); + // Check if stats are same, no need to update + if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) { + MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters()); + } else { + MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext); + } } updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 9a4c61a..9ea6ac4 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -300,16 +300,24 @@ public static boolean requireCalStats(Configuration hiveConf, Partition oldPart, } // requires to calculate stats if new and old have different fast stats + return !isFastStatsSame(oldPart, newPart); + } + + static boolean isFastStatsSame(Partition oldPart, Partition newPart) { + // requires to calculate stats if new and old have different fast stats if ((oldPart != null) && (oldPart.getParameters() != null)) { for (String stat : StatsSetupConst.fastStats) { if (oldPart.getParameters().containsKey(stat)) { Long oldStat = Long.parseLong(oldPart.getParameters().get(stat)); Long newStat = Long.parseLong(newPart.getParameters().get(stat)); if (!oldStat.equals(newStat)) { - return true; + return false; } + } else { + return false; } } + return true; } return false; } @@ -367,14 +375,7 @@ public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionItera FileStatus[] fileStatus = wh.getFileStatusesForLocation(part.getLocation()); populateQuickStats(fileStatus, params); LOG.warn("Updated size to " + params.get(StatsSetupConst.TOTAL_SIZE)); - if (environmentContext != null - && environmentContext.isSetProperties() - && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( - StatsSetupConst.STATS_GENERATED))) { - StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); - } else { - StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); - } + updateBasicState(environmentContext, params); } part.setParameters(params); updated = true; @@ -382,6 +383,20 @@ public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionItera return updated; } + static void updateBasicState(EnvironmentContext environmentContext, Map params) { + if (params == null) { + return; + } + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( + StatsSetupConst.STATS_GENERATED))) { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); + } else { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); + } + } + /** * getDeserializer *