From 07c3ba27b848d1e6c02af11e26a7bf1a639a2615 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Thu, 21 Jan 2016 16:56:30 -0800 Subject: [PATCH] HIVE-12907 : Improve dynamic partition loading - II --- .../apache/hadoop/hive/common/StatsSetupConst.java | 6 ++-- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 32 ++++++++++------------ .../apache/hadoop/hive/ql/metadata/Partition.java | 3 +- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java index 029d415..c9ef647 100644 --- a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java +++ b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java @@ -192,7 +192,7 @@ public static boolean areColumnStatsUptoDate(Map params, String // note that set basic stats false will wipe out column stats too. public static void setBasicStatsState(Map params, String setting) { if (setting.equals(FALSE)) { - if (params.containsKey(COLUMN_STATS_ACCURATE)) { + if (params != null && params.containsKey(COLUMN_STATS_ACCURATE)) { params.remove(COLUMN_STATS_ACCURATE); } } else { @@ -299,8 +299,8 @@ public static void setColumnStatsState(Map params, List } public static void clearColumnStatsState(Map params) { - String statsAcc = params.get(COLUMN_STATS_ACCURATE); - if (statsAcc != null) { + String statsAcc; + if (params != null && (statsAcc = params.get(COLUMN_STATS_ACCURATE)) != null) { // statsAcc may not be jason format, which will throw exception JSONObject stats; try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index efb50b2..a453d4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -84,6 +84,7 @@ import org.apache.hadoop.hive.metastore.api.HiveObjectType; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -1427,14 +1428,13 @@ public void loadPartition(Path loadPath, String tableName, * @param isSrcLocal * If the source directory is LOCAL * @param isAcid true if this is an ACID operation - * @throws JSONException + * @throws JSONException */ public Partition loadPartition(Path loadPath, Table tbl, Map partSpec, boolean replace, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, boolean isSrcLocal, boolean isAcid) throws HiveException { Path tblDataLocationPath = tbl.getDataLocation(); - Partition newTPart = null; try { /** * Move files before creating the partition since down stream processes @@ -1475,18 +1475,14 @@ public Partition loadPartition(Path loadPath, Table tbl, newPartPath = oldPartPath; } - List newFiles = null; if (replace) { Hive.replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(), isSrcLocal); } else { - newFiles = new ArrayList(); FileSystem fs = tbl.getDataLocation().getFileSystem(conf); - Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles); + Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, null); } - - newTPart = getPartition(tbl, partSpec, true, newPartPath.toString(), - inheritTableSpecs, newFiles); + Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath); //column stats will be inaccurate StatsSetupConst.clearColumnStatsState(newTPart.getParameters()); @@ -1500,18 +1496,16 @@ public Partition loadPartition(Path loadPath, Table tbl, /* Add list bucketing location mappings. */ skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps); newCreatedTpart.getSd().setSkewedInfo(skewedInfo); - if(!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE); - } - alterPartition(tbl.getDbName(), tbl.getTableName(), new Partition(tbl, newCreatedTpart)); - newTPart = getPartition(tbl, partSpec, true, newPartPath.toString(), inheritTableSpecs, - newFiles); - return new Partition(tbl, newCreatedTpart); } if(!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE); } - alterPartition(tbl.getDbName(), tbl.getTableName(), new Partition(tbl, newTPart.getTPartition())); + if (oldPart == null) { + getMSC().add_partition(newTPart.getTPartition()); + } else { + alterPartition(tbl.getDbName(), tbl.getTableName(), new Partition(tbl, newTPart.getTPartition())); + } + return newTPart; } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); throw new HiveException(e); @@ -1521,8 +1515,10 @@ public Partition loadPartition(Path loadPath, Table tbl, } catch (InvalidOperationException e) { LOG.error(StringUtils.stringifyException(e)); throw new HiveException(e); + } catch (TException e) { + LOG.error(StringUtils.stringifyException(e)); + throw new HiveException(e); } - return newTPart; } /** @@ -1622,7 +1618,7 @@ private void constructOneLBLocationMap(FileStatus fSta, * @param txnId txnId, can be 0 unless isAcid == true * @return partition map details (PartitionSpec and Partition) * @throws HiveException - * @throws JSONException + * @throws JSONException */ public Map, Partition> loadDynamicPartitions(Path loadPath, String tableName, Map partSpec, boolean replace, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index c8895c2..c0edde9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -164,12 +164,11 @@ protected void initialize(Table table, if (table.isPartitioned()) { try { - String partName = Warehouse.makePartName(table.getPartCols(), tPartition.getValues()); if (tPartition.getSd().getLocation() == null) { // set default if location is not set and this is a physical // table partition (not a view partition) if (table.getDataLocation() != null) { - Path partPath = new Path(table.getDataLocation(), partName); + Path partPath = new Path(table.getDataLocation(), Warehouse.makePartName(table.getPartCols(), tPartition.getValues())); tPartition.getSd().setLocation(partPath.toString()); } } -- 1.7.12.4 (Apple Git-37)