diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 22bb22d..2f274b2 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -770,6 +770,12 @@ HIVE_DDL_OUTPUT_FORMAT("hive.ddl.output.format", null), HIVE_ENTITY_SEPARATOR("hive.entity.separator", "@"), + //Whether to do space padding, indentation for human readability + //controls 'describe table' and 'show columns' output. But formatting + // keywords in query can override this. + //Automatically set by hive-server2 + HIVE_HUMAN_FRIENDLY_FORMAT("hive.human.friendly.format", true), + HIVE_SERVER2_MAX_START_ATTEMPTS("hive.server2.max.start.attempts", 30L, new LongRangeValidator(0L, Long.MAX_VALUE)), diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index 74fd53f..68d5a48 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -1301,17 +1301,35 @@ public void testDescribeTable() throws SQLException { ResultSet res = stmt.executeQuery("describe " + tableName); res.next(); - assertEquals("Column name 'under_col' not found", "under_col", res.getString(1).trim()); + assertEquals("Column name 'under_col' not found", "under_col", res.getString(1)); assertEquals("Column type 'under_col' for column under_col not found", "int", res - .getString(2).trim()); + .getString(2)); res.next(); - assertEquals("Column name 'value' not found", "value", res.getString(1).trim()); + assertEquals("Column name 'value' not found", "value", res.getString(1)); assertEquals("Column type 'string' for column key not found", "string", res - .getString(2).trim()); + .getString(2)); assertFalse("More results found than expected", res.next()); } + + public void testShowColumns() throws SQLException { + Statement stmt = con.createStatement(); + assertNotNull("Statement is null", stmt); + + ResultSet res = stmt.executeQuery("show columns in " + tableName); + res.next(); + assertEquals("Column name 'under_col' not found", + "under_col", res.getString(1)); + + res.next(); + assertEquals("Column name 'value' not found", + "value", res.getString(1)); + assertFalse("More results found than expected", res.next()); + } + + + @Test public void testDatabaseMetaData() throws SQLException { DatabaseMetaData meta = con.getMetaData(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index a926f1e..2505fcb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -656,7 +656,7 @@ private int showGrantsV2(ShowGrantDesc showGrantDesc) throws HiveException { //only grantInfo is used HiveObjectPrivilege thriftObjectPriv = new HiveObjectPrivilege(new HiveObjectRef( AuthorizationUtils.getThriftHiveObjType(privObj.getType()),privObj.getDbname(), - privObj.getTableviewname(),null,null), principal.getName(), + privObj.getTableviewname(),null,null), principal.getName(), AuthorizationUtils.getThriftPrincipalType(principal.getType()), grantInfo); privList.add(thriftObjectPriv); } @@ -2398,8 +2398,10 @@ public int showColumns(Hive db, ShowColumnsDesc showCols) List cols = table.getCols(); cols.addAll(table.getPartCols()); + boolean humanFriendly = + db.getConf().getBoolVar(ConfVars.HIVE_HUMAN_FRIENDLY_FORMAT); outStream.writeBytes( - MetaDataFormatUtils.getAllColumnsInformation(cols, false)); + MetaDataFormatUtils.getAllColumnsInformation(cols, false, humanFriendly)); outStream.close(); outStream = null; } catch (IOException e) { @@ -3027,9 +3029,13 @@ private int describeTable(Hive db, DescTableDesc descTbl) throws HiveException { } fixDecimalColumnTypeName(cols); - + boolean humanFriendly = db.getConf().getBoolVar(ConfVars.HIVE_HUMAN_FRIENDLY_FORMAT); formatter.describeTable(outStream, colPath, tableName, tbl, part, cols, - descTbl.isFormatted(), descTbl.isExt(), descTbl.isPretty()); + descTbl.isFormatted(), + descTbl.isExt(), + descTbl.isPretty(), + humanFriendly + ); LOG.info("DDLTask: written data for " + tbl.getTableName()); outStream.close(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java index 54b02cb..cdecf0e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java @@ -105,7 +105,7 @@ public void describeTable(DataOutputStream out, String colPath, String tableName, Table tbl, Partition part, List cols, boolean isFormatted, boolean isExt, - boolean isPretty) + boolean isPretty, boolean humanFriendly) throws HiveException { MapBuilder builder = MapBuilder.create(); @@ -113,10 +113,11 @@ public void describeTable(DataOutputStream out, builder.put("columns", makeColsUnformatted(cols)); if (isExt) { - if (part != null) - builder.put("partitionInfo", part.getTPartition()); - else - builder.put("tableInfo", tbl.getTTable()); + if (part != null) { + builder.put("partitionInfo", part.getTPartition()); + } else { + builder.put("tableInfo", tbl.getTTable()); + } } asJson(out, builder.build()); @@ -124,8 +125,9 @@ public void describeTable(DataOutputStream out, private List> makeColsUnformatted(List cols) { ArrayList> res = new ArrayList>(); - for (FieldSchema col : cols) - res.add(makeOneColUnformatted(col)); + for (FieldSchema col : cols) { + res.add(makeOneColUnformatted(col)); + } return res; } @@ -162,8 +164,9 @@ public void showTableStatus(DataOutputStream out, { try { ArrayList> res = new ArrayList>(); - for (Table tbl : tbls) - res.add(makeOneTableStatus(tbl, db, conf, part, par)); + for (Table tbl : tbls) { + res.add(makeOneTableStatus(tbl, db, conf, part, par)); + } return res; } catch(IOException e) { throw new HiveException(e); @@ -206,8 +209,9 @@ public void showTableStatus(DataOutputStream out, builder.put("columns", makeColsUnformatted(tbl.getCols())); builder.put("partitioned", tbl.isPartitioned()); - if (tbl.isPartitioned()) - builder.put("partitionColumns", makeColsUnformatted(tbl.getPartCols())); + if (tbl.isPartitioned()) { + builder.put("partitionColumns", makeColsUnformatted(tbl.getPartCols())); + } putFileSystemsStats(builder, makeTableStatusLocations(tbl, db, par), conf, tbl.getPath()); @@ -340,8 +344,9 @@ public void showTablePartitons(DataOutputStream out, List parts) { try { ArrayList> res = new ArrayList>(); - for (String part : parts) - res.add(makeOneTablePartition(part)); + for (String part : parts) { + res.add(makeOneTablePartition(part)); + } return res; } catch (UnsupportedEncodingException e) { throw new HiveException(e); @@ -361,13 +366,15 @@ public void showTablePartitons(DataOutputStream out, List parts) String[] kv = StringUtils.split(part, "=", 2); if (kv != null) { name = kv[0]; - if (kv.length > 1) - val = URLDecoder.decode(kv[1], "UTF-8"); + if (kv.length > 1) { + val = URLDecoder.decode(kv[1], "UTF-8"); + } + } + if (val != null) { + names.add(name + "='" + val + "'"); + } else { + names.add(name); } - if (val != null) - names.add(name + "='" + val + "'"); - else - names.add(name); res.add(MapBuilder.create() .put("columnName", name) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java index de788f7..b726730 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java @@ -63,40 +63,80 @@ private static void formatColumnsHeader(StringBuilder columnInformation) { columnInformation.append(LINE_DELIM); } + /** + * Write formatted information about the given columns to a string + * @param cols - list of columns + * @param printHeader - if header should be included + * @param humanFriendly - make it more human readable by setting indentation + * with spaces. Turned off for use by HS2 + * @return string with formatted column information + */ public static String getAllColumnsInformation(List cols, - boolean printHeader) { + boolean printHeader, boolean humanFriendly) { StringBuilder columnInformation = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); if(printHeader){ formatColumnsHeader(columnInformation); } - formatAllFields(columnInformation, cols); + formatAllFields(columnInformation, cols, humanFriendly); return columnInformation.toString(); } - public static String getAllColumnsInformation(List cols, List partCols, - boolean printHeader) { + /** + * Write formatted information about the given columns, including partition + * columns to a string + * @param cols - list of columns + * @param partCols - list of partition columns + * @param printHeader - if header should be included + * @param humanFriendly - make it more human readable by setting indentation + * with spaces. Turned off for use by HS2 + * @return string with formatted column information + */ + public static String getAllColumnsInformation(List cols, + List partCols, boolean printHeader, boolean humanFriendly) { StringBuilder columnInformation = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); if(printHeader){ formatColumnsHeader(columnInformation); } - formatAllFields(columnInformation, cols); + formatAllFields(columnInformation, cols, humanFriendly); if ((partCols != null) && (!partCols.isEmpty())) { columnInformation.append(LINE_DELIM).append("# Partition Information") .append(LINE_DELIM); formatColumnsHeader(columnInformation); - formatAllFields(columnInformation, partCols); + formatAllFields(columnInformation, partCols, humanFriendly); } return columnInformation.toString(); } - private static void formatAllFields(StringBuilder tableInfo, List cols) { + /** + * Write formatted column information into given StringBuilder + * @param tableInfo - StringBuilder to append column information into + * @param cols - list of columns + * @param humanFriendly - make it more human readable by setting indentation + * with spaces. Turned off for use by HS2 + */ + private static void formatAllFields(StringBuilder tableInfo, + List cols, boolean humanFriendly) { for (FieldSchema col : cols) { - formatOutput(col.getName(), col.getType(), getComment(col), tableInfo); + if(humanFriendly){ + formatWithIndentation(col.getName(), col.getType(), getComment(col), tableInfo); + }else { + formatWithoutIndentation(col.getName(), col.getType(), col.getComment(), tableInfo); + } } } + private static void formatWithoutIndentation(String name, String type, String comment, + StringBuilder colBuffer) { + colBuffer.append(name); + colBuffer.append(FIELD_DELIM); + colBuffer.append(type); + colBuffer.append(FIELD_DELIM); + colBuffer.append(comment == null ? "" : comment); + colBuffer.append(LINE_DELIM); + } + public static String getAllColumnsInformation(Index index) { StringBuilder indexInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); @@ -298,7 +338,7 @@ private static void formatOutput(String name, String value, tableInfo.append(String.format("%-" + ALIGNMENT + "s", value)).append(LINE_DELIM); } - private static void formatOutput(String colName, String colType, String colComment, + private static void formatWithIndentation(String colName, String colType, String colComment, StringBuilder tableInfo) { tableInfo.append(String.format("%-" + ALIGNMENT + "s", colName)).append(FIELD_DELIM); tableInfo.append(String.format("%-" + ALIGNMENT + "s", colType)).append(FIELD_DELIM); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java index 4c40034..61661df 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java @@ -23,13 +23,13 @@ import java.util.List; import java.util.Map; import java.util.Set; + import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; /** * Interface to format table and index information. We can format it @@ -59,11 +59,23 @@ public void showTables(DataOutputStream out, Set tables) /** * Describe table. + * @param out + * @param colPath + * @param tableName + * @param tbl + * @param part + * @param cols + * @param isFormatted - describe with formatted keyword + * @param isExt + * @param isPretty + * @param humanFriendly - if true, add spacing and indentation + * @throws HiveException */ public void describeTable(DataOutputStream out, String colPath, String tableName, Table tbl, Partition part, List cols, - boolean isFormatted, boolean isExt, boolean isPretty) + boolean isFormatted, boolean isExt, + boolean isPretty, boolean humanFriendly) throws HiveException; /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java index 9b0d482..2673520 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java @@ -114,7 +114,8 @@ public void showTables(DataOutputStream out, Set tables) public void describeTable(DataOutputStream outStream, String colPath, String tableName, Table tbl, Partition part, List cols, - boolean isFormatted, boolean isExt, boolean isPretty) + boolean isFormatted, boolean isExt, + boolean isPretty, boolean humanFriendly) throws HiveException { try { String output; @@ -124,9 +125,11 @@ public void describeTable(DataOutputStream outStream, MetaDataPrettyFormatUtils.getAllColumnsInformation( cols, partCols, prettyOutputNumCols) : - MetaDataFormatUtils.getAllColumnsInformation(cols, partCols, isFormatted); + MetaDataFormatUtils.getAllColumnsInformation(cols, partCols, + isFormatted, humanFriendly); } else { - output = MetaDataFormatUtils.getAllColumnsInformation(cols, isFormatted); + output = MetaDataFormatUtils.getAllColumnsInformation(cols, + isFormatted, humanFriendly); } outStream.write(output.getBytes("UTF-8")); diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java index 445c858..a2f8633 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java @@ -86,6 +86,10 @@ public HiveSessionImpl(TProtocolVersion protocol, String username, String passwo this.password = password; this.sessionHandle = new SessionHandle(protocol); + //set hs2 default config + setHiveServer2Configs(); + + //set conf properties specified by user from client side if (sessionConf != null) { for (Map.Entry entry : sessionConf.entrySet()) { hiveConf.set(entry.getKey(), entry.getValue()); @@ -102,6 +106,18 @@ public HiveSessionImpl(TProtocolVersion protocol, String username, String passwo SessionState.start(sessionState); } + /** + * Set configurations recommended for hive-server2 + */ + private void setHiveServer2Configs() { + //as the results are meant to be consumed by java code, turn off + //human friendly format, so that additional indentation and space padding + // is not done + hiveConf.setBoolVar(ConfVars.HIVE_HUMAN_FRIENDLY_FORMAT, false); + } + + + public TProtocolVersion getProtocolVersion() { return sessionHandle.getProtocolVersion(); }