diff --git hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java index d706e07..234d32f 100644 --- hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java +++ hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java @@ -19,6 +19,7 @@ package org.apache.hive.hcatalog.api; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,6 +38,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; @@ -72,6 +74,7 @@ private String outputformat; private String serde; private String storageHandler; + private Map serdeParams; private HCatCreateTableDesc(String dbName, String tableName, List columns) { this.dbName = dbName; @@ -146,6 +149,11 @@ Table toHiveTable(HiveConf conf) throws HCatException { } } newTable.setSd(sd); + if(serdeParams != null) { + for(Map.Entry param : serdeParams.entrySet()) { + sd.getSerdeInfo().putToParameters(param.getKey(), param.getValue()); + } + } if (this.partCols != null) { ArrayList hivePtnCols = new ArrayList(); for (HCatFieldSchema fs : this.partCols) { @@ -296,6 +304,12 @@ public String getFileFormat() { public String getDatabaseName() { return this.dbName; } + /** + * Gets the SerDe parameters; for example see {@link org.apache.hive.hcatalog.api.HCatCreateTableDesc.Builder#fieldsTerminatedBy(char)} + */ + public Map getSerdeParams() { + return serdeParams; + } @Override public String toString() { @@ -325,7 +339,9 @@ public String toString() { : "outputformat=null") + (serde != null ? "serde=" + serde + ", " : "serde=null") + (storageHandler != null ? "storageHandler=" + storageHandler - : "storageHandler=null") + "]"; + : "storageHandler=null") + + ",serdeParams=" + (serdeParams == null ? "null" : serdeParams) + + "]"; } public static class Builder { @@ -344,6 +360,7 @@ public String toString() { private Map tblProps; private boolean ifNotExists; private String dbName; + private Map serdeParams; private Builder(String dbName, String tableName, List columns) { @@ -466,7 +483,52 @@ public Builder fileFormat(String format) { this.fileFormat = format; return this; } - + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public Builder fieldsTerminatedBy(char delimiter) { + return serdeParam(serdeConstants.FIELD_DELIM, Character.toString(delimiter)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public Builder escapeChar(char escapeChar) { + return serdeParam(serdeConstants.ESCAPE_CHAR, Character.toString(escapeChar)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public Builder collectionItemsTerminatedBy(char delimiter) { + return serdeParam(serdeConstants.COLLECTION_DELIM, Character.toString(delimiter)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public Builder mapKeysTerminatedBy(char delimiter) { + return serdeParam(serdeConstants.MAPKEY_DELIM, Character.toString(delimiter)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public Builder linesTerminatedBy(char delimiter) { + return serdeParam(serdeConstants.LINE_DELIM, Character.toString(delimiter)); + } + /** + * See row_format element of CREATE_TABLE DDL for Hive. + */ + public Builder nullDefinedAs(char nullChar) { + return serdeParam(serdeConstants.SERIALIZATION_NULL_FORMAT, Character.toString(nullChar)); + } + /** + * used for setting arbitrary SerDe parameter + */ + public Builder serdeParam(String paramName, String value) { + if(serdeParams == null) { + serdeParams = new HashMap(); + } + serdeParams.put(paramName, value); + return this; + } /** * Builds the HCatCreateTableDesc. * @@ -514,6 +576,7 @@ public HCatCreateTableDesc build() throws HCatException { .getName(); LOG.info("Table output format:" + desc.outputformat); } + desc.serdeParams = this.serdeParams; return desc; } } diff --git hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java index 47c1470..be254c3 100644 --- hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java +++ hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java @@ -48,6 +48,7 @@ private String dbName; private String serde; private String location; + private Map serdeParams; HCatTable(Table hiveTable) throws HCatException { this.tableName = hiveTable.getTableName(); @@ -73,6 +74,7 @@ tblProps = hiveTable.getParameters(); serde = hiveTable.getSd().getSerdeInfo().getSerializationLib(); location = hiveTable.getSd().getLocation(); + serdeParams = hiveTable.getSd().getSerdeInfo().getParameters(); } /** @@ -200,6 +202,12 @@ public String getSerdeLib() { public String getLocation() { return location; } + /** + * Returns parameters such as field delimiter,etc. + */ + public Map getSerdeParams() { + return serdeParams; + } @Override public String toString() { @@ -222,6 +230,8 @@ public String toString() { + ", " : "storageHandler=null") + (tblProps != null ? "tblProps=" + tblProps + ", " : "tblProps=null") + (serde != null ? "serde=" + serde + ", " : "serde=") - + (location != null ? "location=" + location : "location=") + "]"; + + (location != null ? "location=" + location : "location=") + + ",serdeParams=" + (serdeParams == null ? "null" : serdeParams) + + "]"; } } diff --git hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index 360acd7..7886478 100644 --- hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; @@ -161,13 +162,28 @@ public void testBasicDDLCommands() throws Exception { client.dropTable(db, tableOne, true); HCatCreateTableDesc tableDesc2 = HCatCreateTableDesc.create(db, - tableTwo, cols).build(); + tableTwo, cols).fieldsTerminatedBy('\001').escapeChar('\002').linesTerminatedBy('\003'). + mapKeysTerminatedBy('\004').collectionItemsTerminatedBy('\005').nullDefinedAs('\006').build(); client.createTable(tableDesc2); HCatTable table2 = client.getTable(db, tableTwo); assertTrue(table2.getInputFileFormat().equalsIgnoreCase( TextInputFormat.class.getName())); assertTrue(table2.getOutputFileFormat().equalsIgnoreCase( IgnoreKeyTextOutputFormat.class.getName())); + assertTrue("SerdeParams not found", table2.getSerdeParams() != null); + assertEquals("checking " + serdeConstants.FIELD_DELIM, Character.toString('\001'), + table2.getSerdeParams().get(serdeConstants.FIELD_DELIM)); + assertEquals("checking " + serdeConstants.ESCAPE_CHAR, Character.toString('\002'), + table2.getSerdeParams().get(serdeConstants.ESCAPE_CHAR)); + assertEquals("checking " + serdeConstants.LINE_DELIM, Character.toString('\003'), + table2.getSerdeParams().get(serdeConstants.LINE_DELIM)); + assertEquals("checking " + serdeConstants.MAPKEY_DELIM, Character.toString('\004'), + table2.getSerdeParams().get(serdeConstants.MAPKEY_DELIM)); + assertEquals("checking " + serdeConstants.COLLECTION_DELIM, Character.toString('\005'), + table2.getSerdeParams().get(serdeConstants.COLLECTION_DELIM)); + assertEquals("checking " + serdeConstants.SERIALIZATION_NULL_FORMAT, Character.toString('\006'), + table2.getSerdeParams().get(serdeConstants.SERIALIZATION_NULL_FORMAT)); + assertEquals((expectedDir + "/" + db + ".db/" + tableTwo).toLowerCase(), table2.getLocation().toLowerCase()); client.close(); }