diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 9a4c61a..3f62e11 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -238,14 +238,7 @@ public static boolean updateTableStatsFast(Table tbl, FileStatus[] fileStatus, b LOG.info("Updating table stats fast for " + tbl.getTableName()); populateQuickStats(fileStatus, params); LOG.info("Updated size of table " + tbl.getTableName() +" to "+ params.get(StatsSetupConst.TOTAL_SIZE)); - if (environmentContext != null - && environmentContext.isSetProperties() - && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( - StatsSetupConst.STATS_GENERATED))) { - StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); - } else { - StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); - } + updateStatsState(environmentContext, params); } tbl.setParameters(params); updated = true; @@ -253,6 +246,20 @@ public static boolean updateTableStatsFast(Table tbl, FileStatus[] fileStatus, b return updated; } + static void updateStatsState(EnvironmentContext environmentContext, Map params) { + if (params == null) { + return; + } + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( + StatsSetupConst.STATS_GENERATED))) { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); + } else { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); + } + } + public static void populateQuickStats(FileStatus[] fileStatus, Map params) { int numFiles = 0; long tableSize = 0L; @@ -292,13 +299,6 @@ public static boolean requireCalStats(Configuration hiveConf, Partition oldPart, return true; } - if (environmentContext != null - && environmentContext.isSetProperties() - && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( - StatsSetupConst.STATS_GENERATED))) { - return true; - } - // requires to calculate stats if new and old have different fast stats if ((oldPart != null) && (oldPart.getParameters() != null)) { for (String stat : StatsSetupConst.fastStats) { @@ -310,6 +310,16 @@ public static boolean requireCalStats(Configuration hiveConf, Partition oldPart, } } } + updateStatsState(environmentContext, newPart.getParameters()); + // If fast-stats are same for old & new parts,no need to (re)compute stats + return false; + } + + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( + StatsSetupConst.STATS_GENERATED))) { + return true; } return false; } @@ -367,14 +377,7 @@ public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionItera FileStatus[] fileStatus = wh.getFileStatusesForLocation(part.getLocation()); populateQuickStats(fileStatus, params); LOG.warn("Updated size to " + params.get(StatsSetupConst.TOTAL_SIZE)); - if (environmentContext != null - && environmentContext.isSetProperties() - && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( - StatsSetupConst.STATS_GENERATED))) { - StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); - } else { - StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); - } + updateStatsState(environmentContext, params); } part.setParameters(params); updated = true;