diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 94dd72e..c0827ea 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -21,6 +21,7 @@ import com.facebook.fb303.FacebookBase; import com.facebook.fb303.fb_status; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; @@ -3894,19 +3895,12 @@ public String get_config_value(String name, String defaultValue) } } - private List getPartValsFromName(RawStore ms, String dbName, String tblName, - String partName) throws MetaException, InvalidObjectException { + private List getPartValsFromName(Table t, String partName) + throws MetaException, InvalidObjectException { + Preconditions.checkArgument(t != null, "Table can not be null"); // Unescape the partition name LinkedHashMap hm = Warehouse.makeSpecFromName(partName); - // getPartition expects partition values in a list. use info from the - // table to put the partition column values in order - Table t = ms.getTable(dbName, tblName); - if (t == null) { - throw new InvalidObjectException(dbName + "." + tblName - + " table not found"); - } - List partVals = new ArrayList(); for (FieldSchema field : t.getPartitionKeys()) { String key = field.getName(); @@ -3919,6 +3913,16 @@ public String get_config_value(String name, String defaultValue) return partVals; } + private List getPartValsFromName(RawStore ms, String dbName, String tblName, + String partName) throws MetaException, InvalidObjectException { + Table t = ms.getTable(dbName, tblName); + if (t == null) { + throw new InvalidObjectException(dbName + "." + tblName + + " table not found"); + } + return getPartValsFromName(t, partName); + } + private Partition get_partition_by_name_core(final RawStore ms, final String db_name, final String tbl_name, final String part_name) throws MetaException, NoSuchObjectException, TException { @@ -4547,12 +4551,8 @@ public boolean update_table_column_statistics(ColumnStatistics colStats) } } - @Override - public boolean update_partition_column_statistics(ColumnStatistics colStats) - throws NoSuchObjectException,InvalidObjectException,MetaException,TException, - InvalidInputException - { - + private boolean updatePartitonColStats(Table tbl, ColumnStatistics colStats) + throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException { String dbName = null; String tableName = null; String partName = null; @@ -4576,7 +4576,7 @@ public boolean update_partition_column_statistics(ColumnStatistics colStats) colName = statsObj.getColName().toLowerCase(); statsObj.setColName(colName); startFunction("write_partition_column_statistics: db=" + dbName + " table=" + tableName + - " part=" + partName + "column=" + colName); + " part=" + partName + "column=" + colName); } colStats.setStatsDesc(statsDesc); @@ -4585,8 +4585,10 @@ public boolean update_partition_column_statistics(ColumnStatistics colStats) boolean ret = false; try { - List partVals = getPartValsFromName(getMS(), dbName, - tableName, partName); + if (tbl == null) { + tbl = getTable(dbName, tableName); + } + List partVals = getPartValsFromName(tbl, partName); ret = getMS().updatePartitionColumnStatistics(colStats, partVals); return ret; } finally { @@ -4595,6 +4597,13 @@ public boolean update_partition_column_statistics(ColumnStatistics colStats) } @Override + public boolean update_partition_column_statistics(ColumnStatistics colStats) + throws NoSuchObjectException,InvalidObjectException,MetaException,TException, + InvalidInputException { + return updatePartitonColStats(null, colStats); + } + + @Override public boolean delete_partition_column_statistics(String dbName, String tableName, String partName, String colName) throws NoSuchObjectException, MetaException, InvalidObjectException, TException, InvalidInputException @@ -6059,18 +6068,29 @@ public boolean set_aggr_stats_for(SetPartitionsStatsRequest request) map.put(csOld.getStatsDesc().getPartName(), csOld); } } + Table t = getTable(dbName, tableName); for (int index = 0; index < csNews.size(); index++) { ColumnStatistics csNew = csNews.get(index); ColumnStatistics csOld = map.get(csNew.getStatsDesc().getPartName()); if (csOld != null && csOld.getStatsObjSize() != 0) { MetaStoreUtils.mergeColStats(csNew, csOld); } - ret = ret && update_partition_column_statistics(csNew); + ret = ret && updatePartitonColStats(t, csNew); } } return ret; } + private Table getTable(String dbName, String tableName) + throws MetaException, InvalidObjectException { + Table t = getMS().getTable(dbName, tableName); + if (t == null) { + throw new InvalidObjectException(dbName + "." + tableName + + " table not found"); + } + return t; + } + @Override public NotificationEventResponse get_next_notification(NotificationEventRequest rqst) throws TException { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index b6d5276..f98de13 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -54,6 +54,7 @@ import javax.jdo.datastore.DataStoreCache; import javax.jdo.identity.IntIdentity; +import com.google.common.collect.Maps; import org.apache.commons.lang.ArrayUtils; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -6763,8 +6764,9 @@ public UpdateSerdeURIRetVal updateSerdeURI(URI oldLoc, URI newLoc, String serdeP } } - private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj, + MTableColumnStatistics oldStats) throws NoSuchObjectException, MetaException, + InvalidObjectException, InvalidInputException { String dbName = mStatsObj.getDbName(); String tableName = mStatsObj.getTableName(); String colName = mStatsObj.getColName(); @@ -6775,12 +6777,8 @@ private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mSt + " colName=" + colName); validateTableCols(table, Lists.newArrayList(colName)); - List oldStats = - getMTableColumnStatistics(table, Lists.newArrayList(colName), queryWrapper); - - if (!oldStats.isEmpty()) { - assert oldStats.size() == 1; - StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); + if (oldStats != null) { + StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); } else { pm.makePersistent(mStatsObj); } @@ -6790,8 +6788,8 @@ private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mSt } private void writeMPartitionColumnStatistics(Table table, Partition partition, - MPartitionColumnStatistics mStatsObj) throws NoSuchObjectException, - MetaException, InvalidObjectException, InvalidInputException { + MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStats) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { String dbName = mStatsObj.getDbName(); String tableName = mStatsObj.getTableName(); String partName = mStatsObj.getPartitionName(); @@ -6815,17 +6813,39 @@ private void writeMPartitionColumnStatistics(Table table, Partition partition, QueryWrapper queryWrapper = new QueryWrapper(); try { - List oldStats = getMPartitionColumnStatistics( - table, Lists.newArrayList(partName), Lists.newArrayList(colName), queryWrapper); - if (!oldStats.isEmpty()) { - assert oldStats.size() == 1; - StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); - } else { - pm.makePersistent(mStatsObj); + if (oldStats != null) { + StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); + } else { + pm.makePersistent(mStatsObj); + } + } finally { + queryWrapper.close(); } + } + + /** + * Get table's column stats + * + * @param table + * @param colNames + * @return Map of column name and its stats + * @throws NoSuchObjectException + * @throws MetaException + */ + private Map getPartitionColStats(Table table, + List colNames) throws NoSuchObjectException, MetaException { + Map statsMap = Maps.newHashMap(); + QueryWrapper queryWrapper = new QueryWrapper(); + try { + List stats = getMTableColumnStatistics(table, + colNames, queryWrapper); + for(MTableColumnStatistics cStat : stats) { + statsMap.put(cStat.getColName(), cStat); + } } finally { queryWrapper.close(); } + return statsMap; } @Override @@ -6842,11 +6862,16 @@ public boolean updateTableColumnStatistics(ColumnStatistics colStats) // So let's not use them anywhere unless absolutely necessary. Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName()); List colNames = new ArrayList<>(); + for (ColumnStatisticsObj statsObj : statsObjs) { + colNames.add(statsObj.getColName()); + } + Map oldStats = getPartitionColStats(table, colNames); + for (ColumnStatisticsObj statsObj:statsObjs) { // We have to get mtable again because DataNucleus. MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics( ensureGetMTable(statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj); - writeMTableColumnStatistics(table, mStatsObj); + writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName())); colNames.add(statsObj.getColName()); } @@ -6868,40 +6893,70 @@ public boolean updateTableColumnStatistics(ColumnStatistics colStats) } } + /** + * Get partition's column stats + * + * @param table + * @param partitionName + * @param colNames + * @return Map of column name and its stats + * @throws NoSuchObjectException + * @throws MetaException + */ + private Map getPartitionColStats(Table table, + String partitionName, List colNames) throws NoSuchObjectException, MetaException { + Map statsMap = Maps.newHashMap(); + QueryWrapper queryWrapper = new QueryWrapper(); + try { + List stats = getMPartitionColumnStatistics(table, + Lists.newArrayList(partitionName), colNames, queryWrapper); + for(MPartitionColumnStatistics cStat : stats) { + statsMap.put(cStat.getColName(), cStat); + } + } finally { + queryWrapper.close(); + } + return statsMap; + } + @Override public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List partVals) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { boolean committed = false; try { - openTransaction(); - List statsObjs = colStats.getStatsObj(); - ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); - Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName()); - Partition partition = convertToPart(getMPartition( - statsDesc.getDbName(), statsDesc.getTableName(), partVals)); - List colNames = new ArrayList<>(); - for (ColumnStatisticsObj statsObj:statsObjs) { - // We have to get partition again because DataNucleus + openTransaction(); + List statsObjs = colStats.getStatsObj(); + ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); + Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName()); + Partition partition = convertToPart(getMPartition( + statsDesc.getDbName(), statsDesc.getTableName(), partVals)); + List colNames = new ArrayList<>(); + + for(ColumnStatisticsObj statsObj : statsObjs) { + colNames.add(statsObj.getColName()); + } + + Map oldStats = getPartitionColStats(table, statsDesc + .getPartName(), colNames); + MPartition mPartition = getMPartition( statsDesc.getDbName(), statsDesc.getTableName(), partVals); if (partition == null) { throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist."); } - MPartitionColumnStatistics mStatsObj = - StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, statsDesc, statsObj); - writeMPartitionColumnStatistics(table, partition, mStatsObj); - colNames.add(statsObj.getColName()); - } - // Set the partition properties - // No need to check again if it exists. - MPartition mPartition = getMPartition( - statsDesc.getDbName(), statsDesc.getTableName(), partVals); - Map parameters = mPartition.getParameters(); - StatsSetupConst.setColumnStatsState(parameters, colNames); - mPartition.setParameters(parameters); - committed = commitTransaction(); - return committed; + + for (ColumnStatisticsObj statsObj : statsObjs) { + MPartitionColumnStatistics mStatsObj = + StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, statsDesc, statsObj); + writeMPartitionColumnStatistics(table, partition, mStatsObj, + oldStats.get(statsObj.getColName())); + } + Map parameters = mPartition.getParameters(); + StatsSetupConst.setColumnStatsState(parameters, colNames); + mPartition.setParameters(parameters); + committed = commitTransaction(); + return committed; } finally { if (!committed) { rollbackTransaction();