diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e511260..96697c6 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -135,6 +135,7 @@ HiveConf.ConfVars.METASTORE_CACHE_LEVEL2_TYPE, HiveConf.ConfVars.METASTORE_IDENTIFIER_FACTORY, HiveConf.ConfVars.METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK, + HiveConf.ConfVars.METASTORE_ALLOW_NON_SELECT_QUERIES, HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS, HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX, HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, @@ -450,8 +451,9 @@ METASTORE_USE_LEGACY_VALUE_STRATEGY("datanucleus.rdbms.useLegacyNativeValueStrategy", true, ""), METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK("datanucleus.plugin.pluginRegistryBundleCheck", "LOG", "Defines what happens when plugin bundles are found and are duplicated [EXCEPTION|LOG|NONE]"), - METASTORE_BATCH_RETRIEVE_MAX("hive.metastore.batch.retrieve.max", 300, - "Maximum number of objects (tables/partitions) can be retrieved from metastore in one batch. \n" + + METASTORE_ALLOW_NON_SELECT_QUERIES("datanucleus.query.sql.allowAll", true, + "Pass the property datanucleus.query.sql.allowAll as true when creating the PersistenceManagerFactory.."), + METASTORE_BATCH_RETRIEVE_MAX("hive.metastore.batch.retrieve.max", 300, "Maximum number of objects (tables/partitions) can be retrieved from metastore in one batch. \n" + "The higher the number, the less the number of round trips is needed to the Hive metastore server, \n" + "but it may also cause higher memory requirement at the client side."), METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX( diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 59d5244..4ec1c6a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -1208,6 +1208,21 @@ private ColumnStatisticsObj prepareCSObj (Object[] row, int i) throws MetaExcept return result; } + public void deletePartitionStats(String dbName, String tableName, + List partNames, List colNames) throws MetaException { + boolean doTrace = LOG.isDebugEnabled(); + long start = doTrace ? System.nanoTime() : 0; + String queryText = "delete from \"PART_COL_STATS\"" + + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in (" + + makeParams(colNames.size()) + ") AND \"PARTITION_NAME\" in (" + + makeParams(partNames.size()) + ")"; + Query query = pm.newQuery("javax.jdo.query.SQL", queryText); + Object qResult = query.executeWithArray(prepareParams(dbName, tableName, partNames, colNames)); + long queryTime = doTrace ? System.nanoTime() : 0; + timingTrace(doTrace, queryText, start, queryTime); + query.closeAll(); + } + /** The common query part for table and partition stats */ private static final String STATS_COLLIST = "\"COLUMN_NAME\", \"COLUMN_TYPE\", \"LONG_LOW_VALUE\", \"LONG_HIGH_VALUE\", " diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index a16d1c2..5acb240 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -5782,21 +5782,35 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List partNames = new ArrayList(); + List colNames = new ArrayList(); + String dbName = null; + String tblName = null; + List stats = new ArrayList(); + for (ColumnStatistics colStats : request.getColStats()) { + ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); + dbName = statsDesc.getDbName().toLowerCase(); + tblName = statsDesc.getTableName().toLowerCase(); + statsDesc.setDbName(dbName); + statsDesc.setTableName(tblName); + partNames.add(statsDesc.getPartName()); + List statsObjs = colStats.getStatsObj(); + for (ColumnStatisticsObj statsObj : statsObjs) { + String colName = statsObj.getColName().toLowerCase(); + colNames.add(colName); + String colType = statsObj.getColType().toLowerCase(); + statsObj.setColName(colName); + statsObj.setColType(colType); + stats.add(StatObjectConverter + .convertToMPartitionColumnStatistics(null, statsDesc, statsObj)); + } + } boolean committed = false; try { openTransaction(); - for (ColumnStatistics colStats : request.getColStats()) { - ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); - statsDesc.setDbName(statsDesc.getDbName().toLowerCase()); - statsDesc.setTableName(statsDesc.getTableName().toLowerCase()); - List statsObjs = colStats.getStatsObj(); - for (ColumnStatisticsObj statsObj : statsObjs) { - statsObj.setColName(statsObj.getColName().toLowerCase()); - statsObj.setColType(statsObj.getColType().toLowerCase()); - MPartitionColumnStatistics mStatsObj = StatObjectConverter - .convertToMPartitionColumnStatistics(null, statsDesc, statsObj); - pm.makePersistent(mStatsObj); - } + directSql.deletePartitionStats(dbName, tblName, partNames, colNames); + for (MPartitionColumnStatistics mStatsObj : stats) { + pm.makePersistent(mStatsObj); } committed = commitTransaction(); return committed;