From a26dcd55c24253a190653b2ada55cc17ead65016 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Wed, 13 Jul 2016 23:35:00 -0700 Subject: [PATCH] HIVE-14237 : Move stats collection logic out of transaction for create table --- .../hadoop/hive/metastore/HiveAlterHandler.java | 3 +- .../hadoop/hive/metastore/HiveMetaStore.java | 47 +++++++++++----------- .../hadoop/hive/metastore/MetaStoreUtils.java | 9 +---- .../apache/hadoop/hive/metastore/Warehouse.java | 2 +- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 7b84595..a1c1e24 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -233,10 +233,9 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, } } else if (MetaStoreUtils.requireCalStats(hiveConf, null, null, newt, environmentContext) && (newt.getPartitionKeysSize() == 0)) { - Database db = msdb.getDatabase(newt.getDbName()); // Update table stats. For partitioned table, we update stats in // alterPartition() - MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true, environmentContext); + MetaStoreUtils.updateTableStatsFast(newt, wh, false, true, environmentContext); } alterTableUpdateTableColumnStats(msdb, oldt, newt); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index c6c1e11..59be8b0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -492,7 +492,7 @@ public Object getValue() { } expressionProxy = PartFilterExprUtil.createExpressionProxy(hiveConf); - fileMetadataManager = new FileMetadataManager((ThreadLocalRawStore)this, hiveConf); + fileMetadataManager = new FileMetadataManager(this, hiveConf); } private String addPrefix(String s) { @@ -1366,6 +1366,26 @@ private void create_table_core(final RawStore ms, final Table tbl, } Path tblPath = null; + if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) { + if (tbl.getSd().getLocation() == null + || tbl.getSd().getLocation().isEmpty()) { + tblPath = wh.getTablePath( + ms.getDatabase(tbl.getDbName()), tbl.getTableName()); + } else { + if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) { + LOG.warn("Location: " + tbl.getSd().getLocation() + + " specified for non-external table:" + tbl.getTableName()); + } + tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation())); + } + tbl.getSd().setLocation(tblPath.toString()); + } + + if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && + !MetaStoreUtils.isView(tbl)) { + MetaStoreUtils.updateTableStatsFast(tbl, wh, false, false, envContext); + } + boolean success = false, madeDir = false; try { firePreEvent(new PreCreateTableEvent(tbl, this)); @@ -1383,21 +1403,6 @@ private void create_table_core(final RawStore ms, final Table tbl, + " already exists"); } - if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) { - if (tbl.getSd().getLocation() == null - || tbl.getSd().getLocation().isEmpty()) { - tblPath = wh.getTablePath( - ms.getDatabase(tbl.getDbName()), tbl.getTableName()); - } else { - if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) { - LOG.warn("Location: " + tbl.getSd().getLocation() - + " specified for non-external table:" + tbl.getTableName()); - } - tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation())); - } - tbl.getSd().setLocation(tblPath.toString()); - } - if (tblPath != null) { if (!wh.isDir(tblPath)) { if (!wh.mkdirs(tblPath, true)) { @@ -1407,10 +1412,6 @@ private void create_table_core(final RawStore ms, final Table tbl, madeDir = true; } } - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && - !MetaStoreUtils.isView(tbl)) { - MetaStoreUtils.updateTableStatsFast(db, tbl, wh, madeDir, envContext); - } // set create time long time = System.currentTimeMillis() / 1000; @@ -4832,6 +4833,7 @@ private void rethrowException(Exception e) } } + @Override public int get_num_partitions_by_filter(final String dbName, final String tblName, final String filter) throws TException { @@ -5943,9 +5945,6 @@ public Function get_function(String dbName, String funcName) throw new NoSuchObjectException( "Function " + dbName + "." + funcName + " does not exist"); } - } catch (NoSuchObjectException e) { - ex = e; - rethrowException(e); } catch (Exception e) { ex = e; throw newMetaException(e); @@ -6792,7 +6791,7 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, LOG.info("Starting DB backed MetaStore Server"); } } - + TServerTransport serverTransport = tcpKeepAlive ? new TServerSocketKeepAlive(port) : new TServerSocket(port); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 2f1e460..c8ae77e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -185,16 +185,11 @@ public static boolean containsAllFastStats(Map partParams) { return true; } - public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, - boolean madeDir, EnvironmentContext environmentContext) throws MetaException { - return updateTableStatsFast(db, tbl, wh, madeDir, false, environmentContext); - } - - public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + public static boolean updateTableStatsFast(Table tbl, Warehouse wh, boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { if (tbl.getPartitionKeysSize() == 0) { // Update stats only when unpartitioned - FileStatus[] fileStatuses = wh.getFileStatusesForUnpartitionedTable(db, tbl); + FileStatus[] fileStatuses = wh.getFileStatusesForUnpartitionedTable(tbl); return updateTableStatsFast(tbl, fileStatuses, madeDir, forceRecompute, environmentContext); } else { return false; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java index d624d1b..1eca499 100755 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -511,7 +511,7 @@ public static String makePartName(List partCols, * @return array of FileStatus objects corresponding to the files making up the passed * unpartitioned table */ - public FileStatus[] getFileStatusesForUnpartitionedTable(Database db, Table table) + public FileStatus[] getFileStatusesForUnpartitionedTable(Table table) throws MetaException { Path tablePath = getDnsPath(new Path(table.getSd().getLocation())); try { -- 1.7.12.4 (Apple Git-37)