diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 1dc7501291..2fac3f1f1f 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1752,8 +1752,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "The Java class (implementing the StatsPublisher interface) that is used by default if hive.stats.dbclass is custom type."), HIVE_STATS_DEFAULT_AGGREGATOR("hive.stats.default.aggregator", "", "The Java class (implementing the StatsAggregator interface) that is used by default if hive.stats.dbclass is custom type."), - HIVE_STATS_ATOMIC("hive.stats.atomic", false, - "whether to update metastore stats only if all stats are available"), CLIENT_STATS_COUNTERS("hive.client.stats.counters", "", "Subset of counters that should be of interest for hive.client.stats.publishers (when one wants to limit their publishing). \n" + "Non-display names should be used"), diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 476c26196f..a7183ceeae 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -538,9 +538,6 @@ STATSAGGREGATOR_SOURCETASK_NULL(30014, "SourceTask of StatsTask should not be null"), STATSAGGREGATOR_CONNECTION_ERROR(30015, "Stats aggregator of type {0} cannot be connected to", true), - STATSAGGREGATOR_MISSED_SOMESTATS(30016, - "Stats type {0} is missing from stats aggregator. If you don't want the query " + - "to fail because of this, set hive.stats.atomic=false", true), STATS_SKIPPING_BY_ERROR(30017, "Skipping stats aggregation by error {0}", true), INVALID_FILE_FORMAT_IN_LOAD(30019, "The file that you are trying to load does not match the" + diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index 69b076a08a..e85b352754 100644 --- ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -113,13 +113,11 @@ public String getName() { private Partish partish; private FileStatus[] partfileStatus; private BasicStatsWork work; - private boolean atomic; private boolean followedColStats1; public BasicStatsProcessor(Partish partish, BasicStatsWork work, HiveConf conf, boolean followedColStats2) { this.partish = partish; this.work = work; - atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC); followedColStats1 = followedColStats2; } @@ -141,12 +139,7 @@ public Object process(StatsAggregator statsAggregator) throws HiveException, Met if (!work.isExplicitAnalyze() && !followedColStats1) { StatsSetupConst.clearColumnStatsState(parameters); } - // non-partitioned tables: - // XXX: I don't aggree with this logic - // FIXME: deprecate atomic? what's its purpose? - if (!existStats(parameters) && atomic) { - return null; - } + if(partfileStatus == null){ LOG.warn("Partition/partfiles is null for: " + partish.getPartition().getSpec()); return null; @@ -164,7 +157,7 @@ public Object process(StatsAggregator statsAggregator) throws HiveException, Met if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { if (statsAggregator != null) { String prefix = getAggregationPrefix(p.getTable(), p.getPartition()); - updateStats(statsAggregator, parameters, prefix, atomic); + updateStats(statsAggregator, parameters, prefix); } } @@ -172,22 +165,9 @@ public Object process(StatsAggregator statsAggregator) throws HiveException, Met } public void collectFileStatus(Warehouse wh) throws MetaException { - Map parameters = partish.getPartParameters(); - if (!existStats(parameters) && atomic) { - return; - } partfileStatus = wh.getFileStatusesForSD(partish.getPartSd()); } - @Deprecated - private boolean existStats(Map parameters) { - return parameters.containsKey(StatsSetupConst.ROW_COUNT) - || parameters.containsKey(StatsSetupConst.NUM_FILES) - || parameters.containsKey(StatsSetupConst.TOTAL_SIZE) - || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE) - || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS); - } - private void updateQuickStats(Map parameters, FileStatus[] partfileStatus) throws MetaException { MetaStoreUtils.populateQuickStats(partfileStatus, parameters); } @@ -211,7 +191,7 @@ private String getAggregationPrefix0(Table table, Partition partition) throws Me return prefix; } - private void updateStats(StatsAggregator statsAggregator, Map parameters, String aggKey, boolean atomic) throws HiveException { + private void updateStats(StatsAggregator statsAggregator, Map parameters, String aggKey) throws HiveException { for (String statType : StatsSetupConst.statsRequireCompute) { String value = statsAggregator.aggregateStats(aggKey, statType); @@ -225,10 +205,6 @@ private void updateStats(StatsAggregator statsAggregator, Map pa } } parameters.put(statType, String.valueOf(longValue)); - } else { - if (atomic) { - throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); - } } } }