diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java index d616946..71ef989 100755 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -349,12 +349,15 @@ public static String makePartName(Map spec, */ public static String makeDynamicPartName(Map spec) { StringBuilder suffixBuf = new StringBuilder(); + int index = 0; for (Entry e : spec.entrySet()) { if (e.getValue() != null && e.getValue().length() > 0) { + if (index++ > 0) { + suffixBuf.append(Path.SEPARATOR); + } suffixBuf.append(escapePathName(e.getKey())); suffixBuf.append('='); suffixBuf.append(escapePathName(e.getValue())); - suffixBuf.append(Path.SEPARATOR); } else { // stop once we see a dynamic partition break; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 7459bba..b35de00 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -1166,6 +1166,10 @@ private void publishStats() throws HiveException { // not be retrieved from staging table and hence not aggregated. To avoid this issue // we will remove the taskId from the key which is redundant anyway. fspKey = fspKey.split(taskID)[0]; + if (fspKey != null && !fspKey.isEmpty()) { + // fspKey is ended with Path.SEPARATOR + fspKey = fspKey.substring(0, fspKey.length() - 1); + } } // split[0] = DP, split[1] = LB diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java index c50d5b6..61847f1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java @@ -254,7 +254,7 @@ private String getAggregationPrefix(Table table, Partition partition) // prefix is of the form dbName.tblName String prefix = table.getDbName()+"."+table.getTableName(); if (partition != null) { - return Utilities.join(prefix, Warehouse.makePartPath(partition.getSpec())); + return Utilities.join(prefix, Warehouse.makePartName(partition.getSpec(), false)); } return prefix; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 02adf0c..df0ac32 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -2883,19 +2883,20 @@ public static String getHashedStatsPrefix(String statsPrefix, int maxPrefixLengt throw new RuntimeException(e); } } - return statsPrefix.endsWith(Path.SEPARATOR) ? statsPrefix : statsPrefix + Path.SEPARATOR; + return statsPrefix; } public static String join(String... elements) { StringBuilder builder = new StringBuilder(); + int index = 0; for (String element : elements) { if (element == null || element.isEmpty()) { continue; } - builder.append(element); - if (!element.endsWith(Path.SEPARATOR)) { + if (index++ > 0) { builder.append(Path.SEPARATOR); } + builder.append(element); } return builder.toString(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java index 336d490..6ef926c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java @@ -278,7 +278,8 @@ private void printConfigInfo() throws IOException { prefixs.add(Utilities.getHashedStatsPrefix(tablePrefix, maxPrefixLength)); } else { for (Map partitionSpec : partitionSpecs) { - String prefixWithPartition = Utilities.join(tablePrefix, Warehouse.makePartPath(partitionSpec)); + String prefixWithPartition = Utilities.join(tablePrefix, + Warehouse.makePartName(partitionSpec, false)); prefixs.add(Utilities.getHashedStatsPrefix(prefixWithPartition, maxPrefixLength)); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 05dfc4b..ec1b917 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1857,7 +1857,7 @@ public static Path createMoveTask(Task currTask, boolean Partition part = tableScanOp.getConf().getTableMetadata() .getTableSpec().partHandle; try { - aggregationKey.append(Warehouse.makePartPath(part.getSpec())); + aggregationKey.append(Warehouse.makePartName(part.getSpec(), false)); } catch (MetaException e) { throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_AGGKEY.getMsg( part.getDataLocation().toString() + e.getMessage())); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java index 9acc7b7..7dd45c6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java @@ -180,8 +180,7 @@ private void replaceTableScanProcess(TableScanOperator scanOperator) throws Sema TableScanDesc indexTableScanDesc = new TableScanDesc(indexTableHandle); indexTableScanDesc.setGatherStats(false); - String k = indexTableName + Path.SEPARATOR; - indexTableScanDesc.setStatsAggPrefix(k); + indexTableScanDesc.setStatsAggPrefix(indexTableName); scanOperator.setConf(indexTableScanDesc); // Construct the new RowResolver for the new TableScanOperator diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 3ac2feb..a22a894 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6303,7 +6303,6 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) else { try { String ppath = dpCtx.getSPPath(); - ppath = ppath.substring(0, ppath.length() - 1); DummyPartition p = new DummyPartition(dest_tab, dest_tab.getDbName() + "@" + dest_tab.getTableName() + "@" + ppath, @@ -6606,7 +6605,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) if (dest_part != null) { try { - String staticSpec = Warehouse.makePartPath(dest_part.getSpec()); + String staticSpec = Warehouse.makePartName(dest_part.getSpec(), false); fileSinkDesc.setStaticSpec(staticSpec); } catch (MetaException e) { throw new SemanticException(e); @@ -9536,8 +9535,11 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String // db_name.table_name + partitionSec // as the prefix for easy of read during explain and debugging. // Currently, partition spec can only be static partition. - String k = tblName + Path.SEPARATOR; - tsDesc.setStatsAggPrefix(tab.getDbName()+"."+k); + String prefix = tblName; + if (partSpec != null) { + prefix += Path.SEPARATOR; + } + tsDesc.setStatsAggPrefix(tab.getDbName() + "." + prefix); // set up WriteEntity for replication outputs.add(new WriteEntity(tab, WriteEntity.WriteType.DDL_SHARED)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java index 95d5635..0dc0485 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java @@ -158,10 +158,6 @@ public int getNumSPCols() { return this.numSPCols; } - public void setSPPath(String sp) { - this.spPath = sp; - } - public String getSPPath() { return this.spPath; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java index 9d6318a..2f356c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java @@ -341,17 +341,8 @@ public String getStatsAggPrefix() { return statsKeyPref; } - /** - * Set the stats aggregation key. If the input string is not terminated by Path.SEPARATOR - * aggregation key will add one to make it as a directory name. - * @param k input directory name. - */ public void setStatsAggPrefix(String k) { - if (k.endsWith(Path.SEPARATOR)) { - statsKeyPref = k; - } else { - statsKeyPref = k + Path.SEPARATOR; - } + statsKeyPref = k; } public boolean isLinkedFileSink() { diff --git a/ql/src/test/queries/clientpositive/special_character_in_tabnames_1.q b/ql/src/test/queries/clientpositive/special_character_in_tabnames_1.q index 7540d27..799a66b 100644 --- a/ql/src/test/queries/clientpositive/special_character_in_tabnames_1.q +++ b/ql/src/test/queries/clientpositive/special_character_in_tabnames_1.q @@ -1072,4 +1072,9 @@ insert overwrite table `src/_/cbo` select * from src; select * from `src/_/cbo` limit 1; - +drop table `t//`; +create table `t//` (col string); +insert into `t//` values(1); +insert into `t//` values(null); +analyze table `t//` compute statistics; +explain select * from `t//`; diff --git a/ql/src/test/results/clientpositive/special_character_in_tabnames_1.q.out b/ql/src/test/results/clientpositive/special_character_in_tabnames_1.q.out index bd0088a..cb949e4 100644 --- a/ql/src/test/results/clientpositive/special_character_in_tabnames_1.q.out +++ b/ql/src/test/results/clientpositive/special_character_in_tabnames_1.q.out @@ -19548,3 +19548,62 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src/_/cbo #### A masked pattern was here #### 238 val_238 +PREHOOK: query: drop table `t//` +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table `t//` +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table `t//` (col string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t// +POSTHOOK: query: create table `t//` (col string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t// +PREHOOK: query: insert into `t//` values(1) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@t// +POSTHOOK: query: insert into `t//` values(1) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@t// +POSTHOOK: Lineage: t//.col SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: insert into `t//` values(null) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@t// +POSTHOOK: query: insert into `t//` values(null) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@t// +POSTHOOK: Lineage: t//.col SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: analyze table `t//` compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@t// +PREHOOK: Output: default@t// +POSTHOOK: query: analyze table `t//` compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t// +POSTHOOK: Output: default@t// +PREHOOK: query: explain select * from `t//` +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from `t//` +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: t// + Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE + ListSink +