diff --git a/metastore/bin/.gitignore b/metastore/bin/.gitignore deleted file mode 100644 index 0dd9890..0000000 --- a/metastore/bin/.gitignore +++ /dev/null @@ -1 +0,0 @@ -# Dummy file to make Git recognize this empty directory diff --git a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote old mode 100644 new mode 100755 diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 9e3481a..9ae6d7a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -3719,6 +3719,19 @@ public boolean update_partition_column_statistics(ColumnStatistics colStats) } finally { endFunction("write_partition_column_statistics: ", ret != false, null, tableName); } + } + public boolean update_partition_column_statistics( + SetPartitionsStatsRequest request) throws NoSuchObjectException, + InvalidObjectException, MetaException, TException, + InvalidInputException { + boolean ret = false; + try { + ret = getMS().updatePartitionColumnStatistics(request); + return ret; + } finally { + endFunction("write_partition_column_statistics: ", ret != false, null, + null); + } } @Override @@ -5046,15 +5059,11 @@ public AggrStats get_aggr_stats_for(PartitionsStatsRequest request) public boolean set_aggr_stats_for(SetPartitionsStatsRequest request) throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException { - boolean ret = true; - for (ColumnStatistics colStats : request.getColStats()) { - ret = ret && update_partition_column_statistics(colStats); - } - return ret; + return update_partition_column_statistics(request); } - } + public static IHMSHandler newHMSHandler(String name, HiveConf hiveConf) throws MetaException { return newHMSHandler(name, hiveConf, false); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index e8f16a5..9071826 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -23,6 +23,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -88,6 +89,7 @@ import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; @@ -5704,6 +5706,19 @@ private void writeMPartitionColumnStatistics(Table table, Partition partition, pm.makePersistent(mStatsObj); } } + + private void writeMPartitionColumnStatistics(MPartitionColumnStatistics mStatsObj) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + List oldStats = getMPartitionColumnStatistics( + mStatsObj.getTableName(), mStatsObj.getDbName(), + Arrays.asList(mStatsObj.getPartitionName()), Arrays.asList(mStatsObj.getColName())); + if (!oldStats.isEmpty()) { + assert oldStats.size() == 1; + StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); + } else { + pm.makePersistent(mStatsObj); + } + } @Override public boolean updateTableColumnStatistics(ColumnStatistics colStats) @@ -5765,6 +5780,34 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List statsObjs = colStats.getStatsObj(); + for (ColumnStatisticsObj statsObj : statsObjs) { + statsObj.setColName(statsObj.getColName().toLowerCase()); + statsObj.setColType(statsObj.getColType().toLowerCase()); + MPartitionColumnStatistics mStatsObj = StatObjectConverter + .convertToMPartitionColumnStatistics(null, statsDesc, statsObj); + writeMPartitionColumnStatistics(mStatsObj); + } + } + committed = commitTransaction(); + return committed; + } finally { + if (!committed) { + rollbackTransaction(); + } + } + } + private List getMTableColumnStatistics( Table table, List colNames) throws MetaException { boolean committed = false; @@ -5986,6 +6029,58 @@ protected String describeResult() { } } } + + private List getMPartitionColumnStatistics( + String tableName, String dbName, List partNames, List colNames) + throws NoSuchObjectException, MetaException { + boolean committed = false; + MPartitionColumnStatistics mStatsObj = null; + try { + openTransaction(); + Query query = pm.newQuery(MPartitionColumnStatistics.class); + String paramStr = "java.lang.String t1, java.lang.String t2"; + String filter = "tableName == t1 && dbName == t2 && ("; + Object[] params = new Object[colNames.size() + partNames.size() + 2]; + int i = 0; + params[i++] = tableName; + params[i++] = dbName; + int firstI = i; + for (String s : partNames) { + filter += ((i == firstI) ? "" : " || ") + "partitionName == p" + i; + paramStr += ", java.lang.String p" + i; + params[i++] = s; + } + filter += ") && ("; + firstI = i; + for (String s : colNames) { + filter += ((i == firstI) ? "" : " || ") + "colName == c" + i; + paramStr += ", java.lang.String c" + i; + params[i++] = s; + } + filter += ")"; + query.setFilter(filter); + query.declareParameters(paramStr); + query.setOrdering("partitionName ascending"); + @SuppressWarnings("unchecked") + List result = + (List) query.executeWithArray(params); + pm.retrieveAll(result); + committed = commitTransaction(); + return result; + } catch (Exception ex) { + LOG.error("Error retrieving statistics via jdo", ex); + if (ex instanceof MetaException) { + throw (MetaException)ex; + } + throw new MetaException(ex.getMessage()); + } finally { + if (!committed) { + rollbackTransaction(); + return Lists.newArrayList(); + } + } + } + private void dropPartitionColumnStatisticsNoTxn( String dbName, String tableName, List partNames) throws MetaException { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java index e435d69..0fdafa2 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; @@ -551,4 +552,8 @@ public void dropFunction(String dbName, String funcName) public AggrStats get_aggr_stats_for(String dbName, String tblName, List partNames, List colNames) throws MetaException, NoSuchObjectException; + + boolean updatePartitionColumnStatistics( + SetPartitionsStatsRequest request) throws NoSuchObjectException, + MetaException, InvalidObjectException, InvalidInputException; } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 3847d99..6b5e79d 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; @@ -719,5 +720,10 @@ public AggrStats get_aggr_stats_for(String dbName, return null; } + @Override + public boolean updatePartitionColumnStatistics(SetPartitionsStatsRequest request) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + return objectStore.updatePartitionColumnStatistics(request); + } } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 981fa1a..981c5ff 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; @@ -735,6 +736,12 @@ public AggrStats get_aggr_stats_for(String dbName, throws MetaException { return null; } + + @Override + public boolean updatePartitionColumnStatistics(SetPartitionsStatsRequest request) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + return false; + } } diff --git a/ql/.gitignore b/ql/.gitignore deleted file mode 100644 index 916e17c..0000000 --- a/ql/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dependency-reduced-pom.xml diff --git a/service/src/gen/thrift/gen-py/TCLIService/TCLIService-remote b/service/src/gen/thrift/gen-py/TCLIService/TCLIService-remote old mode 100644 new mode 100755 diff --git a/service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote b/service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote old mode 100644 new mode 100755