diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 2e583da..07aee0a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -2322,7 +2322,7 @@ destination @after { popMsg(state); } : KW_LOCAL KW_DIRECTORY StringLiteral tableRowFormat? tableFileFormat? -> ^(TOK_LOCAL_DIR StringLiteral tableRowFormat? tableFileFormat?) - | KW_DIRECTORY StringLiteral -> ^(TOK_DIR StringLiteral) + | KW_DIRECTORY StringLiteral tableRowFormat? tableFileFormat? -> ^(TOK_DIR StringLiteral tableRowFormat? tableFileFormat?) | KW_TABLE tableOrPartition -> tableOrPartition ; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index cf6941c..7f4d0ff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -54,7 +54,7 @@ private boolean isQuery; private boolean isAnalyzeRewrite; private CreateTableDesc tblDesc = null; // table descriptor of the final - private CreateTableDesc localDirectoryDesc = null ; + private CreateTableDesc directoryDesc = null ; // used by PTFs /* @@ -285,12 +285,12 @@ public void setTableDesc(CreateTableDesc desc) { tblDesc = desc; } - public CreateTableDesc getLLocalDirectoryDesc() { - return localDirectoryDesc; + public CreateTableDesc getDirectoryDesc() { + return directoryDesc; } - public void setLocalDirectoryDesc(CreateTableDesc localDirectoryDesc) { - this.localDirectoryDesc = localDirectoryDesc; + public void setDirectoryDesc(CreateTableDesc directoryDesc) { + this.directoryDesc = directoryDesc; } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index bc3b8d6..adfecf3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1794,40 +1794,40 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException qb.getMetaData().setDestForAlias(name, fname, (ast.getToken().getType() == HiveParser.TOK_DIR)); - CreateTableDesc localDirectoryDesc = new CreateTableDesc(); - boolean localDirectoryDescIsSet = false; + CreateTableDesc directoryDesc = new CreateTableDesc(); + boolean directoryDescIsSet = false; int numCh = ast.getChildCount(); for (int num = 1; num < numCh ; num++){ ASTNode child = (ASTNode) ast.getChild(num); if (child != null) { if (storageFormat.fillStorageFormat(child)) { - localDirectoryDesc.setOutputFormat(storageFormat.getOutputFormat()); - localDirectoryDesc.setSerName(storageFormat.getSerde()); - localDirectoryDescIsSet = true; + directoryDesc.setOutputFormat(storageFormat.getOutputFormat()); + directoryDesc.setSerName(storageFormat.getSerde()); + directoryDescIsSet = true; continue; } switch (child.getToken().getType()) { case HiveParser.TOK_TABLEROWFORMAT: rowFormatParams.analyzeRowFormat(child); - localDirectoryDesc.setFieldDelim(rowFormatParams.fieldDelim); - localDirectoryDesc.setLineDelim(rowFormatParams.lineDelim); - localDirectoryDesc.setCollItemDelim(rowFormatParams.collItemDelim); - localDirectoryDesc.setMapKeyDelim(rowFormatParams.mapKeyDelim); - localDirectoryDesc.setFieldEscape(rowFormatParams.fieldEscape); - localDirectoryDesc.setNullFormat(rowFormatParams.nullFormat); - localDirectoryDescIsSet=true; + directoryDesc.setFieldDelim(rowFormatParams.fieldDelim); + directoryDesc.setLineDelim(rowFormatParams.lineDelim); + directoryDesc.setCollItemDelim(rowFormatParams.collItemDelim); + directoryDesc.setMapKeyDelim(rowFormatParams.mapKeyDelim); + directoryDesc.setFieldEscape(rowFormatParams.fieldEscape); + directoryDesc.setNullFormat(rowFormatParams.nullFormat); + directoryDescIsSet=true; break; case HiveParser.TOK_TABLESERIALIZER: ASTNode serdeChild = (ASTNode) child.getChild(0); storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText())); - localDirectoryDesc.setSerName(storageFormat.getSerde()); - localDirectoryDescIsSet=true; + directoryDesc.setSerName(storageFormat.getSerde()); + directoryDescIsSet=true; break; } } } - if (localDirectoryDescIsSet){ - qb.setLocalDirectoryDesc(localDirectoryDesc); + if (directoryDescIsSet){ + qb.setDirectoryDesc(directoryDesc); } break; } @@ -6500,7 +6500,7 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) String fileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT); table_desc = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat); } else { - table_desc = PlanUtils.getDefaultTableDesc(qb.getLLocalDirectoryDesc(), cols, colTypes); + table_desc = PlanUtils.getDefaultTableDesc(qb.getDirectoryDesc(), cols, colTypes); } } else { table_desc = PlanUtils.getTableDesc(tblDesc, cols, colTypes); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 24226bd..2b0eb6d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -107,52 +107,52 @@ public static MapredWork getMapRedWork() { } } - public static TableDesc getDefaultTableDesc(CreateTableDesc localDirectoryDesc, + public static TableDesc getDefaultTableDesc(CreateTableDesc directoryDesc, String cols, String colTypes ) { TableDesc ret = getDefaultTableDesc(Integer.toString(Utilities.ctrlaCode), cols, colTypes, false);; - if (localDirectoryDesc == null) { + if (directoryDesc == null) { return ret; } try { Properties properties = ret.getProperties(); - if (localDirectoryDesc.getFieldDelim() != null) { + if (directoryDesc.getFieldDelim() != null) { properties.setProperty( - serdeConstants.FIELD_DELIM, localDirectoryDesc.getFieldDelim()); + serdeConstants.FIELD_DELIM, directoryDesc.getFieldDelim()); properties.setProperty( - serdeConstants.SERIALIZATION_FORMAT, localDirectoryDesc.getFieldDelim()); + serdeConstants.SERIALIZATION_FORMAT, directoryDesc.getFieldDelim()); } - if (localDirectoryDesc.getLineDelim() != null) { + if (directoryDesc.getLineDelim() != null) { properties.setProperty( - serdeConstants.LINE_DELIM, localDirectoryDesc.getLineDelim()); + serdeConstants.LINE_DELIM, directoryDesc.getLineDelim()); } - if (localDirectoryDesc.getCollItemDelim() != null) { + if (directoryDesc.getCollItemDelim() != null) { properties.setProperty( - serdeConstants.COLLECTION_DELIM, localDirectoryDesc.getCollItemDelim()); + serdeConstants.COLLECTION_DELIM, directoryDesc.getCollItemDelim()); } - if (localDirectoryDesc.getMapKeyDelim() != null) { + if (directoryDesc.getMapKeyDelim() != null) { properties.setProperty( - serdeConstants.MAPKEY_DELIM, localDirectoryDesc.getMapKeyDelim()); + serdeConstants.MAPKEY_DELIM, directoryDesc.getMapKeyDelim()); } - if (localDirectoryDesc.getFieldEscape() !=null) { + if (directoryDesc.getFieldEscape() !=null) { properties.setProperty( - serdeConstants.ESCAPE_CHAR, localDirectoryDesc.getFieldEscape()); + serdeConstants.ESCAPE_CHAR, directoryDesc.getFieldEscape()); } - if (localDirectoryDesc.getSerName() != null) { + if (directoryDesc.getSerName() != null) { properties.setProperty( - serdeConstants.SERIALIZATION_LIB, localDirectoryDesc.getSerName()); + serdeConstants.SERIALIZATION_LIB, directoryDesc.getSerName()); } - if (localDirectoryDesc.getOutputFormat() != null){ - ret.setOutputFileFormatClass(JavaUtils.loadClass(localDirectoryDesc.getOutputFormat())); + if (directoryDesc.getOutputFormat() != null){ + ret.setOutputFileFormatClass(JavaUtils.loadClass(directoryDesc.getOutputFormat())); } - if (localDirectoryDesc.getNullFormat() != null) { + if (directoryDesc.getNullFormat() != null) { properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, - localDirectoryDesc.getNullFormat()); + directoryDesc.getNullFormat()); } - if (localDirectoryDesc.getTblProps() != null) { - properties.putAll(localDirectoryDesc.getTblProps()); + if (directoryDesc.getTblProps() != null) { + properties.putAll(directoryDesc.getTblProps()); } } catch (ClassNotFoundException e) { diff --git a/ql/src/test/queries/clientpositive/insert_overwrite_directory.q b/ql/src/test/queries/clientpositive/insert_overwrite_directory.q index e69de29..2e24951 100644 --- a/ql/src/test/queries/clientpositive/insert_overwrite_directory.q +++ b/ql/src/test/queries/clientpositive/insert_overwrite_directory.q @@ -0,0 +1,141 @@ +insert overwrite directory '../../data/files/src_table_1' +select * from src ; +dfs -cat ../../data/files/src_table_1/000000_0; + +insert overwrite directory '../../data/files/src_table_2' +row format delimited +FIELDS TERMINATED BY ':' +select * from src ; + +dfs -cat ../../data/files/src_table_2/000000_0; + +create table array_table (a array, b array) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\t' +COLLECTION ITEMS TERMINATED BY ','; + +load data local inpath "../../data/files/array_table.txt" overwrite into table array_table; + +insert overwrite directory '../../data/files/array_table_1' +select * from array_table; +dfs -cat ../../data/files/array_table_1/000000_0; + +insert overwrite directory '../../data/files/array_table_2' +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ':' +COLLECTION ITEMS TERMINATED BY '#' +select * from array_table; + +dfs -cat ../../data/files/array_table_2/000000_0; + +insert overwrite directory '../../data/files/array_table_2_withfields' +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ':' +COLLECTION ITEMS TERMINATED BY '#' +select b,a from array_table; + +dfs -cat ../../data/files/array_table_2_withfields/000000_0; + + +create table map_table (foo STRING , bar MAP) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\t' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +STORED AS TEXTFILE; + +load data local inpath "../../data/files/map_table.txt" overwrite into table map_table; + +insert overwrite directory '../../data/files/map_table_1' +select * from map_table; +dfs -cat ../../data/files/map_table_1/000000_0; + +insert overwrite directory '../../data/files/map_table_2' +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ':' +COLLECTION ITEMS TERMINATED BY '#' +MAP KEYS TERMINATED BY '=' +select * from map_table; + +dfs -cat ../../data/files/map_table_2/000000_0; + +insert overwrite directory '../../data/files/map_table_2_withfields' +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ':' +COLLECTION ITEMS TERMINATED BY '#' +MAP KEYS TERMINATED BY '=' +select bar,foo from map_table; + +dfs -cat ../../data/files/map_table_2_withfields/000000_0; + +insert overwrite directory '../../data/files/array_table_3' +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.DelimitedJSONSerDe' +STORED AS TEXTFILE +select * from array_table; + +dfs -cat ../../data/files/array_table_3/000000_0; + + +insert overwrite directory '../../data/files/array_table_4' +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( +'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol', +'quote.delim'= '("|\\[|\\])', 'field.delim'=', ', +'serialization.null.format'='-' ) STORED AS TEXTFILE +select a, null, b from array_table; + +dfs -cat ../../data/files/array_table_4/000000_0; + +insert overwrite directory '../../data/files/map_table_3' +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.DelimitedJSONSerDe' +STORED AS TEXTFILE +select * from map_table; + +dfs -cat ../../data/files/map_table_3/000000_0; + +insert overwrite directory '../../data/files/map_table_4' +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( +'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol', +'quote.delim'= '("|\\[|\\])', 'field.delim'=', ', +'serialization.null.format'='-' ) STORED AS TEXTFILE +select foo, null, bar from map_table; + +dfs -cat ../../data/files/map_table_4/000000_0; + +insert overwrite directory '../../data/files/rctable' +STORED AS RCFILE +select value,key from src; + +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/rctable/temp; +dfs -rmr ${system:test.tmp.dir}/rctable; +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/rctable; +dfs -put ../../data/files/rctable/000000_0 ${system:test.tmp.dir}/rctable/000000_0; + +create external table rctable(value string, key string) +STORED AS RCFILE +LOCATION '${system:test.tmp.dir}/rctable'; + +insert overwrite directory '../../data/files/rctable_out' +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\t' +select key,value from rctable; + +dfs -cat ../../data/files/rctable_out/000000_0; + +drop table rctable; +drop table array_table; +drop table map_table; +dfs -rmr ${system:test.tmp.dir}/rctable; +dfs -rmr ../../data/files/array_table_1; +dfs -rmr ../../data/files/array_table_2; +dfs -rmr ../../data/files/array_table_3; +dfs -rmr ../../data/files/array_table_4; +dfs -rmr ../../data/files/map_table_1; +dfs -rmr ../../data/files/map_table_2; +dfs -rmr ../../data/files/map_table_3; +dfs -rmr ../../data/files/map_table_4; +dfs -rmr ../../data/files/rctable; +dfs -rmr ../../data/files/rctable_out; +dfs -rmr ../../data/files/src_table_1; +dfs -rmr ../../data/files/src_table_2;