diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d3ea824..2a2b332 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1657,8 +1657,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Whether or not to use a binary search to find the entries in an index table that match the filter, where possible"), // Statistics - HIVESTATSAUTOGATHER("hive.stats.autogather", true, - "A flag to gather statistics (only basic) automatically during the INSERT OVERWRITE command."), HIVESTATSCOLAUTOGATHER("hive.stats.column.autogather", false, "A flag to gather column statistics automatically."), HIVESTATSDBCLASS("hive.stats.dbclass", "fs", new PatternSet("custom", "fs"), diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 557ae5a..248ed8a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1439,8 +1439,7 @@ private void create_table_core(final RawStore ms, final Table tbl, madeDir = true; } } - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && - !MetaStoreUtils.isView(tbl)) { + if (!MetaStoreUtils.isView(tbl)) { MetaStoreUtils.updateTableStatsFast(db, tbl, wh, madeDir, envContext); } @@ -2198,8 +2197,7 @@ private Partition append_partition_common(RawStore ms, String dbName, String tab part.setCreateTime((int) time); part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time)); - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && - !MetaStoreUtils.isView(tbl)) { + if (!MetaStoreUtils.isView(tbl)) { MetaStoreUtils.updatePartitionStatsFast(part, wh, madeDir, envContext); } @@ -2754,8 +2752,7 @@ private void initializeAddedPartition( private void initializeAddedPartition( final Table tbl, final PartitionSpecProxy.PartitionIterator part, boolean madeDir) throws MetaException { - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && - !MetaStoreUtils.isView(tbl)) { + if (!MetaStoreUtils.isView(tbl)) { MetaStoreUtils.updatePartitionStatsFast(part, wh, madeDir, false, null); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 917e565..25c0511 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4522,8 +4522,7 @@ private int createTableLike(Hive db, CreateTableLikeDesc crtTbl) throws Exceptio makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tbl.getTableName(), conf); } - if (crtTbl.getLocation() == null && !tbl.isPartitioned() - && conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (crtTbl.getLocation() == null && !tbl.isPartitioned()) { StatsSetupConst.setBasicStatsStateForCreateTable(tbl.getTTable().getParameters(), StatsSetupConst.TRUE); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index ec36487..f2fd7bb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1681,15 +1681,10 @@ public Partition loadPartition(Path loadPath, Table tbl, skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps); newCreatedTpart.getSd().setSkewedInfo(skewedInfo); } - if (!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE); - } if (oldPart == null) { - newTPart.getTPartition().setParameters(new HashMap()); - if (this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsSetupConst.setBasicStatsStateForCreateTable(newTPart.getParameters(), - StatsSetupConst.TRUE); - } + newTPart.getTPartition().setParameters(new HashMap()); + StatsSetupConst.setBasicStatsStateForCreateTable( + newTPart.getParameters(), StatsSetupConst.TRUE); MetaStoreUtils.populateQuickStats(HiveStatsUtils.getFileStatusRecurse(newPartPath, -1, newPartPath.getFileSystem(conf)), newTPart.getParameters()); try { LOG.debug("Adding new partition " + newTPart.getSpec()); @@ -2024,9 +2019,6 @@ public void loadTable(Path loadPath, String tableName, boolean replace, boolean throw new HiveException("addFiles: filesystem error in check phase", e); } } - if (!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE); - } //column stats will be inaccurate StatsSetupConst.clearColumnStatsState(tbl.getParameters()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 88bf829..e81d8fb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1835,8 +1835,7 @@ public static boolean isMergeRequired(List> mvTasks, HiveConf hco // no need of merging if the move is to a local file system MoveTask mvTask = (MoveTask) GenMapRedUtils.findMoveTask(mvTasks, fsOp); - if (mvTask != null && isInsertTable && hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) - && !fsOp.getConf().isMaterialization()) { + if (mvTask != null && isInsertTable && !fsOp.getConf().isMaterialization()) { // mark the MapredWork and FileSinkOperator for gathering stats fsOp.getConf().setGatherStats(true); fsOp.getConf().setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 0cf9205..b893aec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1063,21 +1063,21 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { truncateTask.addDependentTask(moveTsk); // Recalculate the HDFS stats if auto gather stats is set - if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc; - if (oldTblPartLoc.equals(newTblPartLoc)) { - // If we're merging to the same location, we can avoid some metastore calls - TableSpec tablepart = new TableSpec(this.db, conf, root); - statDesc = new StatsWork(tablepart); - } else { - statDesc = new StatsWork(ltd); - } - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statTask = TaskFactory.get(statDesc, conf); - moveTsk.addDependentTask(statTask); + StatsWork statDesc; + if (oldTblPartLoc.equals(newTblPartLoc)) { + // If we're merging to the same location, we can avoid some metastore + // calls + TableSpec tablepart = new TableSpec(this.db, conf, root); + statDesc = new StatsWork(tablepart); + } else { + statDesc = new StatsWork(ltd); } + statDesc.setNoStatsAggregator(true); + statDesc.setClearAggregatorStats(true); + statDesc.setStatsReliable(conf + .getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + Task statTask = TaskFactory.get(statDesc, conf); + moveTsk.addDependentTask(statTask); } catch (HiveException e) { throw new SemanticException(e); } @@ -1678,21 +1678,21 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, conf); mergeTask.addDependentTask(moveTsk); - if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc; - if (oldTblPartLoc.equals(newTblPartLoc)) { - // If we're merging to the same location, we can avoid some metastore calls - TableSpec tablepart = new TableSpec(db, conf, tableName, partSpec); - statDesc = new StatsWork(tablepart); - } else { - statDesc = new StatsWork(ltd); - } - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statTask = TaskFactory.get(statDesc, conf); - moveTsk.addDependentTask(statTask); + StatsWork statDesc; + if (oldTblPartLoc.equals(newTblPartLoc)) { + // If we're merging to the same location, we can avoid some metastore + // calls + TableSpec tablepart = new TableSpec(db, conf, tableName, partSpec); + statDesc = new StatsWork(tablepart); + } else { + statDesc = new StatsWork(ltd); } + statDesc.setNoStatsAggregator(true); + statDesc.setClearAggregatorStats(true); + statDesc.setStatsReliable(conf + .getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + Task statTask = TaskFactory.get(statDesc, conf); + moveTsk.addDependentTask(statTask); rootTasks.add(mergeTask); } catch (Exception e) { @@ -2884,18 +2884,16 @@ private void analyzeAlterTableAddParts(String[] qualified, CommonTree ast, boole addPartitionDesc.addPartition(currentPart, currentLocation); } - if (this.conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - for (int index = 0; index < addPartitionDesc.getPartitionCount(); index++) { - OnePartitionDesc desc = addPartitionDesc.getPartition(index); - if (desc.getLocation() == null) { - if (desc.getPartParams() == null) { - desc.setPartParams(new HashMap()); - } - StatsSetupConst.setBasicStatsStateForCreateTable(desc.getPartParams(), - StatsSetupConst.TRUE); - } - } - } + for (int index = 0; index < addPartitionDesc.getPartitionCount(); index++) { + OnePartitionDesc desc = addPartitionDesc.getPartition(index); + if (desc.getLocation() == null) { + if (desc.getPartParams() == null) { + desc.setPartParams(new HashMap()); + } + StatsSetupConst.setBasicStatsStateForCreateTable( + desc.getPartParams(), StatsSetupConst.TRUE); + } + } if (addPartitionDesc.getPartitionCount() == 0) { // nothing to do diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index bda94ff..5665c44 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -295,13 +295,12 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // However, some other stats, like number of files, do not require a complete scan // Update the stats which do not require a complete scan. Task statTask = null; - if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc = new StatsWork(loadTableWork); - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - statTask = TaskFactory.get(statDesc, conf); - } + StatsWork statDesc = new StatsWork(loadTableWork); + statDesc.setNoStatsAggregator(true); + statDesc.setClearAggregatorStats(true); + statDesc.setStatsReliable(conf + .getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + statTask = TaskFactory.get(statDesc, conf); // HIVE-3334 has been filed for load file with index auto update if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d39b8bd..dc92317 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2164,10 +2164,8 @@ private void getMetaData(QB qb, ReadEntity parentInput) // This is a partition qb.getMetaData().setDestForAlias(name, ts.partHandle); } - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - // Add the table spec for the destination table. - qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); - } + // Add the table spec for the destination table. + qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); break; } @@ -2209,11 +2207,9 @@ private void getMetaData(QB qb, ReadEntity parentInput) throw new SemanticException(generateErrorMessage(ast, "Error creating temporary folder on: " + location.toString()), e); } - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - TableSpec ts = new TableSpec(db, conf, this.ast); - // Add the table spec for the destination table. - qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); - } + TableSpec ts = new TableSpec(db, conf, this.ast); + // Add the table spec for the destination table. + qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } else { // This is the only place where isQuery is set to true; it defaults to false. qb.setIsQuery(true); @@ -7297,8 +7293,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // the following code is used to collect column stats when // hive.stats.autogather=true // and it is an insert overwrite or insert into table - if (dest_tab != null && conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) - && conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER) + if (dest_tab != null && conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER) && ColumnStatsAutoGatherContext.canRunAutogatherStats(fso)) { if (dest_type.intValue() == QBMetaData.DEST_TABLE) { genAutoColumnStatsGatheringPipeline(qb, table_desc, partSpec, input, qb.getParseInfo() @@ -12632,9 +12627,6 @@ private void validateAnalyzePartialscan(ASTNode tree) throws SemanticException { .getTableName())); } - if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_AUTOGATHER.getMsg()); - } } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java index 4f614a8..c494ce5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java @@ -835,7 +835,7 @@ public Table toTable(HiveConf conf) throws HiveException { } } if (getLocation() == null && !this.isCTAS) { - if (!tbl.isPartitioned() && conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (!tbl.isPartitioned()) { StatsSetupConst.setBasicStatsStateForCreateTable(tbl.getTTable().getParameters(), StatsSetupConst.TRUE); }