diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index dcdb0fa56e..e1897efb11 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -434,6 +434,7 @@ minillap.query.files=acid_bucket_pruning.q,\ minillaplocal.query.files=\ dp_counter_non_mm.q,\ dp_counter_mm.q,\ + acid_export.q,\ acid_no_buckets.q, \ acid_globallimit.q,\ acid_vectorization_missing_cols.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 7cb17241db..9aff0069fd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -162,7 +162,7 @@ private StringBuilder genPartitionClause(Map partSpec) throws Se } else { whereClause.append(" and "); } - whereClause.append("`").append(partKey).append("` = ").append(genPartValueString(partKey, value)); + whereClause.append("`").append(partKey).append("` = ").append(genPartValueString(getColTypeOf(partKey), value)); } } @@ -179,33 +179,7 @@ private StringBuilder genPartitionClause(Map partSpec) throws Se return predPresent ? whereClause.append(groupByClause) : groupByClause; } - private String genPartValueString(String partKey, String partVal) throws SemanticException { - String returnVal = partVal; - String partColType = getColTypeOf(partKey); - if (partColType.equals(serdeConstants.STRING_TYPE_NAME) || - partColType.contains(serdeConstants.VARCHAR_TYPE_NAME) || - partColType.contains(serdeConstants.CHAR_TYPE_NAME)) { - returnVal = "'" + escapeSQLString(partVal) + "'"; - } else if (partColType.equals(serdeConstants.TINYINT_TYPE_NAME)) { - returnVal = partVal + "Y"; - } else if (partColType.equals(serdeConstants.SMALLINT_TYPE_NAME)) { - returnVal = partVal + "S"; - } else if (partColType.equals(serdeConstants.INT_TYPE_NAME)) { - returnVal = partVal; - } else if (partColType.equals(serdeConstants.BIGINT_TYPE_NAME)) { - returnVal = partVal + "L"; - } else if (partColType.contains(serdeConstants.DECIMAL_TYPE_NAME)) { - returnVal = partVal + "BD"; - } else if (partColType.equals(serdeConstants.DATE_TYPE_NAME) || - partColType.equals(serdeConstants.TIMESTAMP_TYPE_NAME)) { - returnVal = partColType + " '" + escapeSQLString(partVal) + "'"; - } else { - //for other usually not used types, just quote the value - returnVal = "'" + escapeSQLString(partVal) + "'"; - } - return returnVal; - } private String getColTypeOf(String partKey) throws SemanticException{ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7ff7e189ea..bf78ec1dcd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -532,6 +532,33 @@ public CompilationOpContext getOpContext() { return ctx.getOpContext(); } + public String genPartValueString(String partColType, String partVal) throws SemanticException { + String returnVal = partVal; + if (partColType.equals(serdeConstants.STRING_TYPE_NAME) || + partColType.contains(serdeConstants.VARCHAR_TYPE_NAME) || + partColType.contains(serdeConstants.CHAR_TYPE_NAME)) { + returnVal = "'" + escapeSQLString(partVal) + "'"; + } else if (partColType.equals(serdeConstants.TINYINT_TYPE_NAME)) { + returnVal = partVal + "Y"; + } else if (partColType.equals(serdeConstants.SMALLINT_TYPE_NAME)) { + returnVal = partVal + "S"; + } else if (partColType.equals(serdeConstants.INT_TYPE_NAME)) { + returnVal = partVal; + } else if (partColType.equals(serdeConstants.BIGINT_TYPE_NAME)) { + returnVal = partVal + "L"; + } else if (partColType.contains(serdeConstants.DECIMAL_TYPE_NAME)) { + returnVal = partVal + "BD"; + } else if (partColType.equals(serdeConstants.DATE_TYPE_NAME) || + partColType.equals(serdeConstants.TIMESTAMP_TYPE_NAME)) { + returnVal = partColType + " '" + escapeSQLString(partVal) + "'"; + } else { + //for other usually not used types, just quote the value + returnVal = "'" + escapeSQLString(partVal) + "'"; + } + + return returnVal; + } + public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) throws SemanticException { doPhase1QBExpr(ast, qbexpr, id, alias, false); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 512f1ff3da..8e5118b94e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -277,7 +277,9 @@ private StringBuilder generateExportQuery(List partCols, TableSpec exportTableSpec = new TableSpec(db, conf, tableTree, false, true); if(exportTableSpec.getPartSpec() != null) { StringBuilder whereClause = null; + int partColsIdx = -1; //keep track of corresponding col in partCols for(Map.Entry ent : exportTableSpec.getPartSpec().entrySet()) { + partColsIdx++; if(ent.getValue() == null) { continue; //partial spec } @@ -288,7 +290,7 @@ private StringBuilder generateExportQuery(List partCols, whereClause.append(" AND "); } whereClause.append(HiveUtils.unparseIdentifier(ent.getKey(), conf)) - .append(" = ").append(ent.getValue()); + .append(" = ").append(genPartValueString(partCols.get(partColsIdx).getType(), ent.getValue())); } if(whereClause != null) { rewrittenQueryStr.append(whereClause); diff --git a/ql/src/test/queries/clientpositive/acid_export.q b/ql/src/test/queries/clientpositive/acid_export.q new file mode 100644 index 0000000000..efe86a7c93 --- /dev/null +++ b/ql/src/test/queries/clientpositive/acid_export.q @@ -0,0 +1,18 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create table exportPartitionTable(id int, name string) partitioned by(country string) + clustered by (id) into 2 buckets stored as orc tblproperties ("transactional"="true"); + +export table exportPartitionTable PARTITION (country='india') to '${system:test.tmp.dir}/exportDataStore'; + +drop table exportPartitionTable; + + +--non-partitioned table +create table exportTable(id int, name string) + clustered by (id) into 2 buckets stored as orc tblproperties ("transactional"="true"); + +export table exportTable to '${system:test.tmp.dir}/exportTableStore'; +drop table exportTable; + diff --git a/ql/src/test/results/clientpositive/llap/acid_export.q.out b/ql/src/test/results/clientpositive/llap/acid_export.q.out new file mode 100644 index 0000000000..8ebb1d9b68 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/acid_export.q.out @@ -0,0 +1,59 @@ +PREHOOK: query: create table exportPartitionTable(id int, name string) partitioned by(country string) + clustered by (id) into 2 buckets stored as orc tblproperties ("transactional"="true") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@exportPartitionTable +POSTHOOK: query: create table exportPartitionTable(id int, name string) partitioned by(country string) + clustered by (id) into 2 buckets stored as orc tblproperties ("transactional"="true") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@exportPartitionTable +#### A masked pattern was here #### +PREHOOK: type: QUERY +PREHOOK: Input: database:default +PREHOOK: Input: default@exportpartitiontable +PREHOOK: Output: default@exportpartitiontable_9bbbd040_695d_4552_8234_7e9523647a79 +#### A masked pattern was here #### +POSTHOOK: type: QUERY +POSTHOOK: Input: database:default +POSTHOOK: Input: default@exportpartitiontable +#### A masked pattern was here #### +PREHOOK: query: drop table exportPartitionTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@exportpartitiontable +PREHOOK: Output: default@exportpartitiontable +POSTHOOK: query: drop table exportPartitionTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@exportpartitiontable +POSTHOOK: Output: default@exportpartitiontable +PREHOOK: query: create table exportTable(id int, name string) + clustered by (id) into 2 buckets stored as orc tblproperties ("transactional"="true") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@exportTable +POSTHOOK: query: create table exportTable(id int, name string) + clustered by (id) into 2 buckets stored as orc tblproperties ("transactional"="true") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@exportTable +#### A masked pattern was here #### +PREHOOK: type: QUERY +PREHOOK: Input: database:default +PREHOOK: Input: default@exporttable +PREHOOK: Output: default@exporttable_b98fa75b_ec89_4322_9349_83259f0b0921 +#### A masked pattern was here #### +POSTHOOK: type: QUERY +POSTHOOK: Input: database:default +POSTHOOK: Input: default@exporttable +POSTHOOK: Output: default@exporttable_b98fa75b_ec89_4322_9349_83259f0b0921 +#### A masked pattern was here #### +POSTHOOK: Lineage: exporttable_b98fa75b_ec89_4322_9349_83259f0b0921.id SIMPLE [(exporttable)exporttable.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: exporttable_b98fa75b_ec89_4322_9349_83259f0b0921.name SIMPLE [(exporttable)exporttable.FieldSchema(name:name, type:string, comment:null), ] +PREHOOK: query: drop table exportTable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@exporttable +PREHOOK: Output: default@exporttable +POSTHOOK: query: drop table exportTable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@exporttable +POSTHOOK: Output: default@exporttable