diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java index 1ca18b9..d7d6806 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java @@ -88,7 +88,7 @@ public void testEndFunctionListener() throws Exception { Exception e = context.getException(); assertTrue((e!=null)); assertTrue((e instanceof NoSuchObjectException)); - assertEquals(context.getInputTableName(), null); + assertEquals(context.getMetaInfo(), null); driver.run("use " + dbName); driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName)); @@ -106,7 +106,7 @@ public void testEndFunctionListener() throws Exception { e = context.getException(); assertTrue((e!=null)); assertTrue((e instanceof NoSuchObjectException)); - assertEquals(context.getInputTableName(), tableName); + assertEquals(context.getMetaInfo(), tableName); try { msc.getPartition("hive3524", tblName, "b=2012"); @@ -121,7 +121,7 @@ public void testEndFunctionListener() throws Exception { e = context.getException(); assertTrue((e!=null)); assertTrue((e instanceof NoSuchObjectException)); - assertEquals(context.getInputTableName(), tblName); + assertEquals(context.getMetaInfo(), tblName); try { driver.run("drop table Unknown"); } @@ -135,7 +135,7 @@ public void testEndFunctionListener() throws Exception { e = context.getException(); assertTrue((e!=null)); assertTrue((e instanceof NoSuchObjectException)); - assertEquals(context.getInputTableName(), "Unknown"); + assertEquals(context.getMetaInfo(), "Unknown"); } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index f1b58c5..0b75ea5 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -824,7 +824,7 @@ private static void logInfo(String m) { private String startFunction(String function, String extraLogInfo) { incrementCounter(function); - logInfo((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") + + logInfo("START: " + (getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") + function + extraLogInfo); com.codahale.metrics.Timer timer = Metrics.getOrCreateTimer(MetricsConstants.API_PREFIX + function); @@ -864,8 +864,8 @@ private void endFunction(String function, boolean successful, Exception e) { endFunction(function, successful, e, null); } private void endFunction(String function, boolean successful, Exception e, - String inputTableName) { - endFunction(function, new MetaStoreEndFunctionContext(successful, e, inputTableName)); + String metaInfo) { + endFunction(function, new MetaStoreEndFunctionContext(successful, e, metaInfo)); } private void endFunction(String function, MetaStoreEndFunctionContext context) { @@ -879,6 +879,10 @@ private void endFunction(String function, MetaStoreEndFunctionContext context) { for (MetaStoreEndFunctionListener listener : endFunctionListeners) { listener.onEndFunction(function, context); } + + logInfo("END: " + (getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") + + function + " metaInfo=" + context.getMetaInfo() + " success=" + context.isSuccess() + " exceptionMsg=" + + (context.getException() == null ? "null" : context.getException().getMessage())); } @Override @@ -998,7 +1002,8 @@ public void create_database(final Database db) throw newMetaException(e); } } finally { - endFunction("create_database", success, ex); + String metaInfo = " db=" + db.getName(); + endFunction("create_database", success, ex, metaInfo); } } @@ -1014,7 +1019,8 @@ public Database get_database(final String name) throws NoSuchObjectException, Me ex = e; throw e; } finally { - endFunction("get_database", db != null, ex); + String metaInfo = " db=" + name; + endFunction("get_database", db != null, ex, metaInfo); } return db; } @@ -1081,7 +1087,8 @@ public void alter_database(final String dbName, final Database newDB) throws TEx null, transactionalListenersResponses, ms); } - endFunction("alter_database", success, ex); + String metaInfo = " db=" + dbName; + endFunction("alter_database", success, ex, metaInfo); } } @@ -1235,7 +1242,8 @@ public void drop_database(final String dbName, final boolean deleteData, final b startFunction("drop_database", ": " + dbName); if (DEFAULT_DATABASE_NAME.equalsIgnoreCase(dbName)) { - endFunction("drop_database", false, null); + String metaInfo = " db=" + dbName; + endFunction("drop_database", false, null, metaInfo); throw new MetaException("Can not drop default database"); } @@ -1259,7 +1267,8 @@ public void drop_database(final String dbName, final boolean deleteData, final b throw newMetaException(e); } } finally { - endFunction("drop_database", success, ex); + String metaInfo = " db=" + dbName; + endFunction("drop_database", success, ex, metaInfo); } } @@ -1279,7 +1288,8 @@ public void drop_database(final String dbName, final boolean deleteData, final b throw newMetaException(e); } } finally { - endFunction("get_databases", ret != null, ex); + String metaInfo = " pattern=" + pattern; + endFunction("get_databases", ret != null, ex, metaInfo); } return ret; } @@ -1347,7 +1357,8 @@ public boolean create_type(final Type type) throws AlreadyExistsException, throw newMetaException(e); } } finally { - endFunction("create_type", success, ex); + String metaInfo = " type=" + type.toString(); + endFunction("create_type", success, ex, metaInfo); } return success; @@ -1368,7 +1379,8 @@ public Type get_type(final String name) throws MetaException, NoSuchObjectExcept ex = e; throwMetaException(e); } finally { - endFunction("get_type", ret != null, ex); + String metaInfo = " type=" + name; + endFunction("get_type", ret != null, ex, metaInfo); } return ret; } @@ -1391,7 +1403,8 @@ public boolean drop_type(final String name) throws MetaException, NoSuchObjectEx ex = e; throwMetaException(e); } finally { - endFunction("drop_type", success, ex); + String metaInfo = " db=" + name; + endFunction("drop_type", success, ex, metaInfo); } return success; } @@ -1400,7 +1413,8 @@ public boolean drop_type(final String name) throws MetaException, NoSuchObjectEx public Map get_type_all(String name) throws MetaException { // TODO Auto-generated method stub startFunction("get_type_all", ": " + name); - endFunction("get_type_all", false, null); + String metaInfo = " type=" + name; + endFunction("get_type_all", false, null, metaInfo); throw new MetaException("Not yet implemented"); } @@ -1626,7 +1640,11 @@ public void create_table_with_environment_context(final Table tbl, throw newMetaException(e); } } finally { - endFunction("create_table", success, ex, tbl.getTableName()); + String metaInfo = " db="+ tbl.getDbName() + ", table=" + tbl.getTableName(); + endFunction("create_table", success, ex, metaInfo); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } } @@ -1658,7 +1676,11 @@ public void create_table_with_constraints(final Table tbl, throw newMetaException(e); } } finally { - endFunction("create_table", success, ex, tbl.getTableName()); + String metaInfo = " db="+ tbl.getDbName() + ", table=" + tbl.getTableName(); + endFunction("create_table", success, ex, metaInfo); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } } @@ -1703,7 +1725,10 @@ public void drop_constraint(DropConstraintRequest req) listener.onDropConstraint(dropConstraintEvent); } } - endFunction("drop_constraint", success, ex, constraintName); + endFunction("drop_constraint", success, ex, " constraintName=" + constraintName); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } } @@ -1755,7 +1780,10 @@ public void add_primary_key(AddPrimaryKeyRequest req) listener.onAddPrimaryKey(addPrimaryKeyEvent); } } - endFunction("add_primary_key", success, ex, constraintName); + endFunction("add_primary_key", success, ex, " constraintName=" + constraintName); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } } @@ -1807,7 +1835,10 @@ public void add_foreign_key(AddForeignKeyRequest req) listener.onAddForeignKey(addForeignKeyEvent); } } - endFunction("add_foreign_key", success, ex, constraintName); + endFunction("add_foreign_key", success, ex, " constraintName=" + constraintName); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } } @@ -1859,7 +1890,10 @@ public void add_unique_constraint(AddUniqueConstraintRequest req) listener.onAddUniqueConstraint(addUniqueConstraintEvent); } } - endFunction("add_unique_constraint", success, ex, constraintName); + endFunction("add_unique_constraint", success, ex, " constraintName=" + constraintName); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } } @@ -1911,7 +1945,10 @@ public void add_not_null_constraint(AddNotNullConstraintRequest req) listener.onAddNotNullConstraint(addNotNullConstraintEvent); } } - endFunction("add_not_null_constraint", success, ex, constraintName); + endFunction("add_not_null_constraint", success, ex, " constraintName=" + constraintName); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } } @@ -2173,9 +2210,12 @@ public void drop_table_with_environment_context(final String dbname, final Strin ex = e; throwMetaException(e); } finally { - endFunction("drop_table", success, ex, name); + String metaInfo = " db="+ dbname + ", table=" + name; + endFunction("drop_table", success, ex, metaInfo); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } - } private void updateStatsForTruncate(Map props, EnvironmentContext environmentContext) { @@ -2378,7 +2418,8 @@ private Table getTableInternal(String dbname, String name, ex = e; throw e; } finally { - endFunction("get_table", t != null, ex, name); + String metaInfo = " db="+ dbname + ", table=" + name; + endFunction("get_table", t != null, ex, metaInfo); } return t; } @@ -2396,7 +2437,8 @@ private Table getTableInternal(String dbname, String name, ex = e; throw newMetaException(e); } finally { - endFunction("get_table_metas", t != null, ex); + String metaInfo = " db=" + dbnames + ", table=" + tblNames; + endFunction("get_table_metas", t != null, ex, metaInfo); } return t; } @@ -2506,7 +2548,8 @@ public GetTablesResult get_table_objects_by_name_req(GetTablesRequest req) throw throw newMetaException(e); } } finally { - endFunction("get_multi_table", tables != null, ex, join(tableNames, ",")); + String metaInfo = " db="+ dbName + ", tables=" + tableNames; + endFunction("get_multi_table", tables != null, ex, metaInfo); } return tables; } @@ -2555,7 +2598,8 @@ private boolean doesClientHaveCapability(ClientCapabilities client, ClientCapabi throw newMetaException(e); } } finally { - endFunction("get_table_names_by_filter", tables != null, ex, join(tables, ",")); + String metaInfo = " db="+ dbName + ", tables=" + tables; + endFunction("get_table_names_by_filter", tables != null, ex, metaInfo); } return tables; } @@ -2700,7 +2744,8 @@ public Partition append_partition_with_environment_context(final String dbName, throw newMetaException(e); } } finally { - endFunction("append_partition", ret != null, ex, tableName); + String metaInfo = " db="+ dbName + ", table=" + tableName + ", partitions=" + part_vals; + endFunction("append_partition", ret != null, ex, metaInfo); } return ret; } @@ -2976,7 +3021,13 @@ public int add_partitions(final List parts) throws MetaException, } } finally { String tableName = parts.get(0).getTableName(); - endFunction("add_partition", ret != null, ex, tableName); + String dbName = parts.get(0).getDbName(); + List part_vals = new ArrayList<>(); + for (Partition partition : parts) { + part_vals.addAll(partition.getValues()); + } + String metaInfo = " db="+ dbName + ", table=" + tableName + ", partitions=" + part_vals; + endFunction("add_partition", ret != null, ex, metaInfo); } return ret; } @@ -3297,7 +3348,12 @@ public Partition add_partition_with_environment_context( throw newMetaException(e); } } finally { - endFunction("add_partition", ret != null, ex, part != null ? part.getTableName(): null); + String tableName = part.getTableName(); + String dbName = part.getDbName(); + List part_vals = new ArrayList<>(); + part_vals.addAll(part.getValues()); + String metaInfo = " db="+ dbName + ", table=" + tableName + ", partitions=" + part_vals; + endFunction("add_partition", ret != null, ex, metaInfo); } return ret; } @@ -3750,7 +3806,8 @@ public boolean drop_partition_with_environment_context(final String db_name, ex = e; rethrowException(e); } finally { - endFunction("drop_partition", ret, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partitions=" + part_vals; + endFunction("drop_partition", ret, ex, metaInfo); } return ret; @@ -3770,7 +3827,8 @@ public Partition get_partition(final String db_name, final String tbl_name, ex = e; throwMetaException(e); } finally { - endFunction("get_partition", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partitions=" + part_vals; + endFunction("get_partition", ret != null, ex, metaInfo); } return ret; } @@ -3817,7 +3875,8 @@ public Partition get_partition_with_auth(final String db_name, ex = e; rethrowException(e); } finally { - endFunction("get_partition_with_auth", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partitions=" + part_vals; + endFunction("get_partition_with_auth", ret != null, ex, metaInfo); } return ret; } @@ -3836,7 +3895,8 @@ public Partition get_partition_with_auth(final String db_name, ex = e; throwMetaException(e); } finally { - endFunction("get_partitions", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name; + endFunction("get_partitions", ret != null, ex, metaInfo); } return ret; @@ -3861,7 +3921,8 @@ public Partition get_partition_with_auth(final String db_name, ex = e; rethrowException(e); } finally { - endFunction("get_partitions_with_auth", ret != null, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_partitions_with_auth", ret != null, ex, metaInfo); } return ret; @@ -3925,7 +3986,8 @@ private void checkLimitNumberOfPartitions(String tblName, int numPartitions, int return partitionSpecs; } finally { - endFunction("get_partitions_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tbl_name); + String metaInfo = " db="+ dbName + ", table=" + tableName; + endFunction("get_partitions_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, metaInfo); } } @@ -4068,7 +4130,8 @@ private static boolean is_partition_spec_grouping_enabled(Table table) { throw newMetaException(e); } } finally { - endFunction("get_partition_names", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name; + endFunction("get_partition_names", ret != null, ex, metaInfo); } return ret; } @@ -4170,7 +4233,8 @@ private void rename_partition(final String db_name, final String tbl_name, throw newMetaException(e); } } finally { - endFunction("alter_partition", oldPart != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name; + endFunction("alter_partition", oldPart != null, ex, metaInfo); } } @@ -4241,7 +4305,8 @@ public void alter_partitions_with_environment_context(final String db_name, fina throw newMetaException(e); } } finally { - endFunction("alter_partition", oldParts != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name; + endFunction("alter_partition", oldParts != null, ex, metaInfo); } } @@ -4288,7 +4353,8 @@ public void alter_index(final String dbname, final String base_table_name, ms.rollbackTransaction(); } - endFunction("alter_index", success, ex, base_table_name); + String metaInfo = " db="+ dbname + ", table=" + base_table_name; + endFunction("alter_index", success, ex, metaInfo); if (!listeners.isEmpty()) { MetaStoreListenerNotifier.notifyEvent(listeners, @@ -4298,6 +4364,9 @@ public void alter_index(final String dbname, final String base_table_name, transactionalListenerResponses, ms); } } + if (!success) { + throw new MetaException("Commit transaction failed"); + } } @Override @@ -4402,7 +4471,11 @@ private void alter_table_core(final String dbname, final String name, final Tabl throw newMetaException(e); } } finally { - endFunction("alter_table", success, ex, name); + String metaInfo = " db="+ dbname + ", table=" + name; + endFunction("alter_table", success, ex, metaInfo); + } + if (!success) { + throw new MetaException("Commit transaction failed"); } } @@ -4423,7 +4496,8 @@ private void alter_table_core(final String dbname, final String name, final Tabl throw newMetaException(e); } } finally { - endFunction("get_tables", ret != null, ex); + String metaInfo = " db=" + dbname + ", pattern=" + pattern; + endFunction("get_tables", ret != null, ex, metaInfo); } return ret; } @@ -4445,7 +4519,8 @@ private void alter_table_core(final String dbname, final String name, final Tabl throw newMetaException(e); } } finally { - endFunction("get_tables_by_type", ret != null, ex); + String metaInfo = " db=" + dbname + ", pattern=" + pattern + ", type=" + tableType; + endFunction("get_tables_by_type", ret != null, ex, metaInfo); } return ret; } @@ -4466,7 +4541,8 @@ private void alter_table_core(final String dbname, final String name, final Tabl throw newMetaException(e); } } finally { - endFunction("get_all_tables", ret != null, ex); + String metaInfo = " db=" + dbname; + endFunction("get_all_tables", ret != null, ex, metaInfo); } return ret; } @@ -4517,7 +4593,8 @@ private void alter_table_core(final String dbname, final String name, final Tabl if (orgHiveLoader != null) { conf.setClassLoader(orgHiveLoader); } - endFunction("get_fields_with_environment_context", ret != null, ex, tableName); + String metaInfo = " db="+ db + ", table=" + tableName; + endFunction("get_fields_with_environment_context", ret != null, ex, metaInfo); } return ret; @@ -4618,7 +4695,8 @@ private StorageSchemaReader getStorageSchemaReader() throws MetaException { throw me; } } finally { - endFunction("get_schema_with_environment_context", success, ex, tableName); + String metaInfo = " db="+ db + ", table=" + tableName; + endFunction("get_schema_with_environment_context", success, ex, metaInfo); } } @@ -4671,7 +4749,8 @@ public String get_config_value(String name, String defaultValue) throw new TException(e); } } finally { - endFunction("get_config_value", success, ex); + String metaInfo = " name=" + name + ", defaultValue=" + defaultValue; + endFunction("get_config_value", success, ex, metaInfo); } } @@ -4736,7 +4815,8 @@ public Partition get_partition_by_name(final String db_name, final String tbl_na ex = e; rethrowException(e); } finally { - endFunction("get_partition_by_name", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partition=" + part_name; + endFunction("get_partition_by_name", ret != null, ex, metaInfo); } return ret; } @@ -4772,7 +4852,8 @@ public Partition append_partition_by_name_with_environment_context(final String throw newMetaException(e); } } finally { - endFunction("append_partition_by_name", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partition=" + part_name; + endFunction("append_partition_by_name", ret != null, ex, metaInfo); } return ret; } @@ -4817,7 +4898,8 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n ex = e; rethrowException(e); } finally { - endFunction("drop_partition_by_name", ret, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partition=" + part_name; + endFunction("drop_partition_by_name", ret, ex, metaInfo); } return ret; @@ -4838,7 +4920,8 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n ex = e; rethrowException(e); } finally { - endFunction("get_partitions_ps", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partitions=" + part_vals; + endFunction("get_partitions_ps", ret != null, ex, metaInfo); } return ret; @@ -4864,7 +4947,8 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n ex = e; rethrowException(e); } finally { - endFunction("get_partitions_ps_with_auth", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partitions=" + part_vals; + endFunction("get_partitions_ps_with_auth", ret != null, ex, metaInfo); } return ret; } @@ -4883,7 +4967,8 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n ex = e; rethrowException(e); } finally { - endFunction("get_partitions_names_ps", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partitions=" + part_vals; + endFunction("get_partitions_names_ps", ret != null, ex, metaInfo); } return ret; } @@ -4927,7 +5012,9 @@ public Index add_index(final Index newIndex, final Table indexTable) throws TExc throw newMetaException(e); } } finally { - endFunction("add_index", ret != null, ex, tableName); + String metaInfo = " db="+ indexTable.getDbName() + ", table=" + indexTable.getTableName() + ", index=" + + newIndex.getIndexName(); + endFunction("add_index", ret != null, ex, metaInfo); } return ret; } @@ -5024,7 +5111,8 @@ public boolean drop_index_by_name(final String dbName, final String tblName, ex = e; rethrowException(e); } finally { - endFunction("drop_index_by_name", ret, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName + ", index=" + indexName; + endFunction("drop_index_by_name", ret, ex, metaInfo); } return ret; @@ -5113,7 +5201,8 @@ public Index get_index_by_name(final String dbName, final String tblName, ex = e; rethrowException(e); } finally { - endFunction("get_index_by_name", ret != null, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName + ", index=" + indexName; + endFunction("get_index_by_name", ret != null, ex, metaInfo); } return ret; } @@ -5146,7 +5235,8 @@ private Index get_index_by_name_core(final RawStore ms, final String db_name, throw newMetaException(e); } } finally { - endFunction("get_index_names", ret != null, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_index_names", ret != null, ex, metaInfo); } return ret; } @@ -5164,7 +5254,8 @@ private Index get_index_by_name_core(final RawStore ms, final String db_name, ex = e; rethrowException(e); } finally { - endFunction("get_indexes", ret != null, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_indexes", ret != null, ex, metaInfo); } return ret; } @@ -5205,7 +5296,8 @@ public ColumnStatistics get_table_column_statistics(String dbName, String tableN } return statsObj; } finally { - endFunction("get_column_statistics_by_table", statsObj != null, null, tableName); + String metaInfo = " db="+ dbName + ", table=" + tableName + ", column=" + colName; + endFunction("get_column_statistics_by_table", statsObj != null, null, metaInfo); } } @@ -5224,7 +5316,8 @@ public TableStatsResult get_table_statistics_req(TableStatsRequest request) thro result = new TableStatsResult((cs == null || cs.getStatsObj() == null) ? Lists.newArrayList() : cs.getStatsObj()); } finally { - endFunction("get_table_statistics_req", result == null, null, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_table_statistics_req", result == null, null, metaInfo); } return result; } @@ -5250,7 +5343,8 @@ public ColumnStatistics get_partition_column_statistics(String dbName, String ta } statsObj = list.get(0); } finally { - endFunction("get_column_statistics_by_partition", statsObj != null, null, tableName); + String metaInfo = " db="+ dbName + ", table=" + tableName + ", partition=" + convertedPartName + ", column=" + colName; + endFunction("get_column_statistics_by_partition", statsObj != null, null, metaInfo); } return statsObj; } @@ -5280,7 +5374,8 @@ public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsReques } result = new PartitionsStatsResult(map); } finally { - endFunction("get_partitions_statistics_req", result == null, null, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_partitions_statistics_req", result == null, null, metaInfo); } return result; } @@ -5289,7 +5384,7 @@ public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsReques public boolean update_table_column_statistics(ColumnStatistics colStats) throws TException { String dbName; String tableName; - String colName; + String colName = null; ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); dbName = statsDesc.getDbName().toLowerCase(); tableName = statsDesc.getTableName().toLowerCase(); @@ -5318,7 +5413,8 @@ public boolean update_table_column_statistics(ColumnStatistics colStats) throws ret = getMS().updateTableColumnStatistics(colStats); return ret; } finally { - endFunction("write_column_statistics", ret != false, null, tableName); + String metaInfo = " db="+ dbName + ", table=" + tableName + ", column=" + colName; + endFunction("write_column_statistics", ret != false, null, metaInfo); } } @@ -5327,7 +5423,7 @@ private boolean updatePartitonColStats(Table tbl, ColumnStatistics colStats) String dbName; String tableName; String partName; - String colName; + String colName = null; ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); dbName = statsDesc.getDbName().toLowerCase(); @@ -5365,7 +5461,8 @@ private boolean updatePartitonColStats(Table tbl, ColumnStatistics colStats) ret = getMS().updatePartitionColumnStatistics(colStats, partVals); return ret; } finally { - endFunction("write_partition_column_statistics", ret != false, null, tableName); + String metaInfo = " db="+ dbName + ", table=" + tableName + ", partition=" + partName + ", column=" + colName; + endFunction("write_partition_column_statistics", ret != false, null, metaInfo); } } @@ -5393,7 +5490,8 @@ public boolean delete_partition_column_statistics(String dbName, String tableNam ret = getMS().deletePartitionColumnStatistics(dbName, tableName, convertedPartName, partVals, colName); } finally { - endFunction("delete_column_statistics_by_partition", ret != false, null, tableName); + String metaInfo = " db="+ dbName + ", table=" + tableName + ", partition=" + convertedPartName + ", column=" + colName; + endFunction("delete_column_statistics_by_partition", ret != false, null, metaInfo); } return ret; } @@ -5414,7 +5512,8 @@ public boolean delete_table_column_statistics(String dbName, String tableName, S try { ret = getMS().deleteTableColumnStatistics(dbName, tableName, colName); } finally { - endFunction("delete_column_statistics_by_table", ret != false, null, tableName); + String metaInfo = " db="+ dbName + ", table=" + tableName + ", column=" + colName; + endFunction("delete_column_statistics_by_table", ret != false, null, metaInfo); } return ret; } @@ -5434,7 +5533,8 @@ public boolean delete_table_column_statistics(String dbName, String tableName, S ex = e; rethrowException(e); } finally { - endFunction("get_partitions_by_filter", ret != null, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_partitions_by_filter", ret != null, ex, metaInfo); } return ret; } @@ -5466,7 +5566,8 @@ public boolean delete_table_column_statistics(String dbName, String tableName, S return partitionSpecs; } finally { - endFunction("get_partitions_by_filter_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_partitions_by_filter_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, metaInfo); } } @@ -5488,7 +5589,8 @@ public PartitionsByExprResult get_partitions_by_expr( ex = e; rethrowException(e); } finally { - endFunction("get_partitions_by_expr", ret != null, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_partitions_by_expr", ret != null, ex, metaInfo); } return ret; } @@ -5519,7 +5621,8 @@ public int get_num_partitions_by_filter(final String dbName, ex = e; rethrowException(e); } finally { - endFunction("get_num_partitions_by_filter", ret != -1, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_num_partitions_by_filter", ret != -1, ex, metaInfo); } return ret; } @@ -5537,7 +5640,8 @@ int get_num_partitions_by_expr(final String dbName, ex = e; rethrowException(e); } finally { - endFunction("get_num_partitions_by_expr", ret != -1, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_num_partitions_by_expr", ret != -1, ex, metaInfo); } return ret; } @@ -5556,7 +5660,8 @@ int get_num_partitions_by_expr(final String dbName, ex = e; rethrowException(e); } finally { - endFunction("get_partitions_by_names", ret != null, ex, tblName); + String metaInfo = " db="+ dbName + ", table=" + tblName; + endFunction("get_partitions_by_names", ret != null, ex, metaInfo); } return ret; } @@ -6163,7 +6268,7 @@ public boolean add_token(String token_identifier, String delegation_token) throw throw newMetaException(e); } } finally { - endFunction("add_token", ret == true, ex); + endFunction("add_token", ret == true, ex, " tokenId=" + token_identifier); } return ret; } @@ -6183,7 +6288,7 @@ public boolean remove_token(String token_identifier) throws TException { throw newMetaException(e); } } finally { - endFunction("remove_token", ret == true, ex); + endFunction("remove_token", ret == true, ex, " tokenId=" + token_identifier); } return ret; } @@ -6203,7 +6308,7 @@ public String get_token(String token_identifier) throws TException { throw newMetaException(e); } } finally { - endFunction("get_token", ret != null, ex); + endFunction("get_token", ret != null, ex, " tokenId=" + token_identifier); } //Thrift cannot return null result return ret == null ? "" : ret; @@ -6353,8 +6458,8 @@ public void markPartitionForEvent(final String db_name, final String tbl_name, if (!success) { ms.rollbackTransaction(); } - - endFunction("markPartitionForEvent", tbl != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partition=" + partName; + endFunction("markPartitionForEvent", tbl != null, ex, metaInfo); } } @@ -6382,7 +6487,8 @@ public boolean isPartitionMarkedForEvent(final String db_name, final String tbl_ throw newMetaException(original); } } finally { - endFunction("isPartitionMarkedForEvent", ret != null, ex, tbl_name); + String metaInfo = " db="+ db_name + ", table=" + tbl_name + ", partition=" + partName; + endFunction("isPartitionMarkedForEvent", ret != null, ex, metaInfo); } return ret; @@ -6566,7 +6672,8 @@ public void alter_function(String dbName, String funcName, Function newFunc) thr ex = e; throw newMetaException(e); } finally { - endFunction("get_functions", funcNames != null, ex); + String metaInfo = " db="+ dbName + ", pattern=" + pattern; + endFunction("get_functions", funcNames != null, ex, metaInfo); } return funcNames; @@ -6613,7 +6720,8 @@ public Function get_function(String dbName, String funcName) throws TException { ex = e; throw newMetaException(e); } finally { - endFunction("get_function", func != null, ex); + String metaInfo = " db="+ dbName + ", function=" + funcName; + endFunction("get_function", func != null, ex, metaInfo); } return func; @@ -6772,7 +6880,8 @@ public AggrStats get_aggr_stats_for(PartitionsStatsRequest request) throws TExce lowerCaseColNames)); return aggrStats; } finally { - endFunction("get_aggr_stats_for", aggrStats == null, null, request.getTblName()); + String metaInfo = " db="+ request.getDbName() + ", table=" + request.getTblName(); + endFunction("get_aggr_stats_for", aggrStats == null, null, metaInfo); } } @@ -7192,7 +7301,8 @@ public PrimaryKeysResponse get_primary_keys(PrimaryKeysRequest request) throws T ex = e; throwMetaException(e); } finally { - endFunction("get_primary_keys", ret != null, ex, tbl_name); + String metaInfo = " db=" + db_name + ", table=" + tbl_name; + endFunction("get_primary_keys", ret != null, ex, metaInfo); } return new PrimaryKeysResponse(ret); } @@ -7215,7 +7325,9 @@ public ForeignKeysResponse get_foreign_keys(ForeignKeysRequest request) throws T ex = e; throwMetaException(e); } finally { - endFunction("get_foreign_keys", ret != null, ex, foreign_tbl_name); + String metaInfo = " parentdb=" + parent_db_name + ", parenttbl=" + parent_tbl_name + + ", foreigndb=" + foreign_db_name + ", foreigntbl=" + foreign_tbl_name; + endFunction("get_foreign_keys", ret != null, ex, metaInfo); } return new ForeignKeysResponse(ret); } @@ -7249,7 +7361,8 @@ public UniqueConstraintsResponse get_unique_constraints(UniqueConstraintsRequest throw newMetaException(e); } } finally { - endFunction("get_unique_constraints", ret != null, ex, tbl_name); + String metaInfo = " db=" + db_name + ", table=" + tbl_name; + endFunction("get_unique_constraints", ret != null, ex, metaInfo); } return new UniqueConstraintsResponse(ret); } @@ -7272,7 +7385,8 @@ public NotNullConstraintsResponse get_not_null_constraints(NotNullConstraintsReq throw newMetaException(e); } } finally { - endFunction("get_not_null_constraints", ret != null, ex, tbl_name); + String metaInfo = " db=" + db_name + ", table=" + tbl_name; + endFunction("get_not_null_constraints", ret != null, ex, metaInfo); } return new NotNullConstraintsResponse(ret); } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java index 998531f..4fe03f0 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java @@ -29,12 +29,12 @@ */ private final boolean success; private final Exception e; - private final String inputTableName; + private final String metaInfo; - public MetaStoreEndFunctionContext(boolean success, Exception e, String inputTableName) { + public MetaStoreEndFunctionContext(boolean success, Exception e, String metaInfo) { this.success = success; this.e = e; - this.inputTableName = inputTableName; + this.metaInfo = metaInfo; } public MetaStoreEndFunctionContext(boolean success) { @@ -52,8 +52,8 @@ public Exception getException() { return e; } - public String getInputTableName() { - return inputTableName; + public String getMetaInfo() { + return metaInfo; } } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 2e43dc8..e6dae82 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -349,6 +349,9 @@ public static ConfVars getMetaConf(String name) { CONNECTION_DRIVER("javax.jdo.option.ConnectionDriverName", "javax.jdo.option.ConnectionDriverName", "org.apache.derby.jdbc.EmbeddedDriver", "Driver class name for a JDBC metastore"), + CONNECTION_POOLING_CONNECTION_TIMEOUT("datanucleus.connectionPool.connectionTimeout", + "datanucleus.connectionPool.connectionTimeout", 30, TimeUnit.SECONDS, + "When connection pool is exhausted, wait until this time before throwing exception (default: 30s)"), CONNECTION_POOLING_MAX_CONNECTIONS("datanucleus.connectionPool.maxPoolSize", "datanucleus.connectionPool.maxPoolSize", 10, "Specify the maximum number of connections in the connection pool. Note: The configured size will be used by\n" + diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java index 4ff2bb7..04e6977 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java @@ -27,6 +27,7 @@ import javax.sql.DataSource; import java.sql.SQLException; import java.util.Properties; +import java.util.concurrent.TimeUnit; /** * DataSourceProvider for the BoneCP connection pool. @@ -51,7 +52,8 @@ public DataSource create(Configuration hdpConfig) throws SQLException { MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS); Properties properties = DataSourceProvider.getPrefixedProperties(hdpConfig, BONECP); - long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L); + long connectionTimeout = MetastoreConf.getTimeVar(hdpConfig, + MetastoreConf.ConfVars.CONNECTION_POOLING_CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS); String partitionCount = properties.getProperty(PARTITION_COUNT_PROPERTY, "1"); BoneCPConfig config = null; diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java index 6ffc24a..237f509 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java @@ -27,6 +27,7 @@ import javax.sql.DataSource; import java.sql.SQLException; import java.util.Properties; +import java.util.concurrent.TimeUnit; /** * DataSourceProvider for the HikariCP connection pool. @@ -36,7 +37,6 @@ private static final Logger LOG = LoggerFactory.getLogger(HikariCPDataSourceProvider.class); public static final String HIKARI = "hikari"; - private static final String CONNECTION_TIMEOUT_PROPERTY= "hikari.connectionTimeout"; @Override public DataSource create(Configuration hdpConfig) throws SQLException { @@ -51,7 +51,8 @@ public DataSource create(Configuration hdpConfig) throws SQLException { Properties properties = replacePrefix( DataSourceProvider.getPrefixedProperties(hdpConfig, HIKARI)); - long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L); + long connectionTimeout = MetastoreConf.getTimeVar(hdpConfig, + MetastoreConf.ConfVars.CONNECTION_POOLING_CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS); HikariConfig config = null; try { config = new HikariConfig(properties); @@ -64,6 +65,9 @@ public DataSource create(Configuration hdpConfig) throws SQLException { config.setPassword(passwd); //https://github.com/brettwooldridge/HikariCP config.setConnectionTimeout(connectionTimeout); + if (LOG.isDebugEnabled()) { + config.setLeakDetectionThreshold(connectionTimeout); + } return new HikariDataSource(config); }