Index: conf/hive-default.xml.template =================================================================== --- conf/hive-default.xml.template (revision 1435342) +++ conf/hive-default.xml.template (working copy) @@ -71,6 +71,14 @@ + hive.cli.pretty.output.num.cols + -1 + The number of columns to use when formatting output generated + by the DESCRIBE PRETTY table_name command. If the value of this property + is -1, then hive will use the auto-detected terminal width. + + + hive.exec.scratchdir /tmp/hive-${user.name} Scratch space for Hive jobs Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1435342) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -351,6 +351,7 @@ CLIIGNOREERRORS("hive.cli.errors.ignore", false), CLIPRINTCURRENTDB("hive.cli.print.current.db", false), CLIPROMPT("hive.cli.prompt", "hive"), + CLIPRETTYOUTPUTNUMCOLS("hive.cli.pretty.output.num.cols", -1), HIVE_METASTORE_FS_HANDLER_CLS("hive.metastore.fs.handler.class", "org.apache.hadoop.hive.metastore.HiveMetaStoreFsImpl"), Index: ql/ivy.xml =================================================================== --- ql/ivy.xml (revision 1435342) +++ ql/ivy.xml (working copy) @@ -79,6 +79,8 @@ transitive="false"/> + + Index: ql/src/test/results/clientpositive/describe_pretty.q.out =================================================================== --- ql/src/test/results/clientpositive/describe_pretty.q.out (revision 0) +++ ql/src/test/results/clientpositive/describe_pretty.q.out (revision 0) @@ -0,0 +1,332 @@ +PREHOOK: query: -- test comment indent processing for multi-line comments + +CREATE TABLE test_table( + col1 INT COMMENT 'col1 one line comment', + col2 STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines', + col5 STRING COMMENT 'col5 very long multi-line comment where each line is very long by itself and is likely to spill +into multiple lines. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin in dolor nisl, sodales +adipiscing tortor. Integer venenatis', + col6 STRING COMMENT 'This comment has a very long single word ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvzxyz123 which will not fit in a line by itself for small column widths.', + col7_NoComment STRING) +COMMENT 'table comment +two lines' +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- test comment indent processing for multi-line comments + +CREATE TABLE test_table( + col1 INT COMMENT 'col1 one line comment', + col2 STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines', + col5 STRING COMMENT 'col5 very long multi-line comment where each line is very long by itself and is likely to spill +into multiple lines. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin in dolor nisl, sodales +adipiscing tortor. Integer venenatis', + col6 STRING COMMENT 'This comment has a very long single word ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvzxyz123 which will not fit in a line by itself for small column widths.', + col7_NoComment STRING) +COMMENT 'table comment +two lines' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@test_table +PREHOOK: query: -- There will be an extra tab at the end of each comment line in the output. +-- This is because DESCRIBE command separates the column, type and +-- comment field using a \t. DESCRIBE PRETTY uses spaces instead +-- of \t to separate columns. Hive gets confused when it parses the string +-- table description constructed in MetaDataPrettyFormatUtils, and adds a tab +-- at the end of each line. +-- There are three ways to address this: +-- 1. Pad each row to the full terminal width with extra spaces. +-- 2. Assume a maximum tab width of 8, and subtract 2 * 8 spaces from the +-- available line width. This approach wastes upto 2 * 8 - 2 columns. +-- 3. Since the pretty output is meant only for human consumption, do nothing. +-- Just add a comment to the unit test file explaining what is happening. +-- This is the approach chosen. + +DESCRIBE PRETTY test_table +PREHOOK: type: DESCTABLE +POSTHOOK: query: -- There will be an extra tab at the end of each comment line in the output. +-- This is because DESCRIBE command separates the column, type and +-- comment field using a \t. DESCRIBE PRETTY uses spaces instead +-- of \t to separate columns. Hive gets confused when it parses the string +-- table description constructed in MetaDataPrettyFormatUtils, and adds a tab +-- at the end of each line. +-- There are three ways to address this: +-- 1. Pad each row to the full terminal width with extra spaces. +-- 2. Assume a maximum tab width of 8, and subtract 2 * 8 spaces from the +-- available line width. This approach wastes upto 2 * 8 - 2 columns. +-- 3. Since the pretty output is meant only for human consumption, do nothing. +-- Just add a comment to the unit test file explaining what is happening. +-- This is the approach chosen. + +DESCRIBE PRETTY test_table +POSTHOOK: type: DESCTABLE +col_name data_type comment + +col1 int col1 one line comment +col2 string col2 + two lines comment +col3 string col3 + three lines + comment +col4 string col4 very long comment that is greater than 80 + chars and is likely to spill into multiple + lines +col5 string col5 very long multi-line comment where each + line is very long by itself and is likely to + spill + into multiple lines. Lorem ipsum dolor sit + amet, consectetur adipiscing elit. Proin in + dolor nisl, sodales + adipiscing tortor. Integer venenatis +col6 string This comment has a very long single word ABCDEF + GHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvzxyz12 + 3 which will not fit in a line by itself for + small column widths. +col7_nocomment string None +PREHOOK: query: DESCRIBE PRETTY test_table +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE PRETTY test_table +POSTHOOK: type: DESCTABLE +col_name data_type comment + +col1 int col1 one line comment +col2 string col2 + two lines comment +col3 string col3 + three lines + comment +col4 string col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines +col5 string col5 very long multi-line comment where each line is very long by itself and is likely to spill + into multiple lines. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin in dolor nisl, sodales + adipiscing tortor. Integer venenatis +col6 string This comment has a very long single word ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvzxyz123 which will not fit in a line by itself for small column widths. +col7_nocomment string None +PREHOOK: query: DESCRIBE PRETTY test_table +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE PRETTY test_table +POSTHOOK: type: DESCTABLE +col_name data_type comment + +col1 int col1 one line + comment +col2 string col2 + two lines comment +col3 string col3 + three lines + comment +col4 string col4 very long + comment that is + greater than 80 + chars and is + likely to spill + into multiple + lines +col5 string col5 very long + multi-line + comment where + each line is very + long by itself + and is likely to + spill + into multiple + lines. Lorem + ipsum dolor sit + amet, consectetur + adipiscing elit. + Proin in dolor + nisl, sodales + adipiscing + tortor. Integer + venenatis +col6 string This comment has + a very long + single word ABCDE + FGHIJKLMNOPQRSTUV + XYZabcdefghijklmn + opqrstuvzxyz123 + which will not + fit in a line by + itself for small + column widths. +col7_nocomment string None +PREHOOK: query: DESCRIBE PRETTY test_table +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE PRETTY test_table +POSTHOOK: type: DESCTABLE +col_name data_type comment + +col1 int col1 one line comment +col2 string col2 + two lines comment +col3 string col3 + three lines + comment +col4 string col4 very long comment that + is greater than 80 chars + and is likely to spill into + multiple lines +col5 string col5 very long multi-line + comment where each line is + very long by itself and is + likely to spill + into multiple lines. Lorem + ipsum dolor sit amet, + consectetur adipiscing + elit. Proin in dolor nisl, + sodales + adipiscing tortor. Integer + venenatis +col6 string This comment has a very + long single word ABCDEFGHIJ + KLMNOPQRSTUVXYZabcdefghijkl + mnopqrstuvzxyz123 which + will not fit in a line by + itself for small column + widths. +col7_nocomment string None +PREHOOK: query: CREATE TABLE test_table_very_long_column_name( + col1 INT COMMENT 'col1 one line comment', + col2_abcdefghiklmnopqrstuvxyz STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines') +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE test_table_very_long_column_name( + col1 INT COMMENT 'col1 one line comment', + col2_abcdefghiklmnopqrstuvxyz STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@test_table_very_long_column_name +PREHOOK: query: DESCRIBE PRETTY test_table_very_long_column_name +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE PRETTY test_table_very_long_column_name +POSTHOOK: type: DESCTABLE +col_name data_type comment + +col1 int col1 one line comment +col2_abcdefghiklmnopqrstuvxyz string col2 + two lines comment +col3 string col3 + three lines + comment +col4 string col4 very long comment that is + greater than 80 chars and is + likely to spill into multiple + lines +PREHOOK: query: DESCRIBE PRETTY test_table_very_long_column_name +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE PRETTY test_table_very_long_column_name +POSTHOOK: type: DESCTABLE +col_name data_type comment + +col1 int col1 one + line + comment +col2_abcdefghiklmnopqrstuvxyz string col2 + two lines + comment +col3 string col3 + three + lines + comment +col4 string col4 very + long + comment + that is + greater + than 80 + chars and + is likely + to spill + into + multiple + lines +PREHOOK: query: CREATE TABLE test_table_partitioned( + col1 INT COMMENT 'col1 one line comment', + col2 STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines', + col5 STRING COMMENT 'col5 very long multi-line comment where each line is very long by itself and is likely to spill +into multiple lines. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin in dolor nisl, sodales +adipiscing tortor. Integer venenatis', + col6 STRING COMMENT 'This comment has a very long single word ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvzxyz123 which will not fit in a line by itself for small column widths.', + col7_NoComment STRING) +COMMENT 'table comment +two lines' +PARTITIONED BY (ds STRING) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE test_table_partitioned( + col1 INT COMMENT 'col1 one line comment', + col2 STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines', + col5 STRING COMMENT 'col5 very long multi-line comment where each line is very long by itself and is likely to spill +into multiple lines. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin in dolor nisl, sodales +adipiscing tortor. Integer venenatis', + col6 STRING COMMENT 'This comment has a very long single word ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvzxyz123 which will not fit in a line by itself for small column widths.', + col7_NoComment STRING) +COMMENT 'table comment +two lines' +PARTITIONED BY (ds STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@test_table_partitioned +PREHOOK: query: DESCRIBE PRETTY test_table_partitioned +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE PRETTY test_table_partitioned +POSTHOOK: type: DESCTABLE +col_name data_type comment + +col1 int col1 one line comment +col2 string col2 + two lines comment +col3 string col3 + three lines + comment +col4 string col4 very long comment that + is greater than 80 chars + and is likely to spill into + multiple lines +col5 string col5 very long multi-line + comment where each line is + very long by itself and is + likely to spill + into multiple lines. Lorem + ipsum dolor sit amet, + consectetur adipiscing + elit. Proin in dolor nisl, + sodales + adipiscing tortor. Integer + venenatis +col6 string This comment has a very + long single word ABCDEFGHIJ + KLMNOPQRSTUVXYZabcdefghijkl + mnopqrstuvzxyz123 which + will not fit in a line by + itself for small column + widths. +col7_nocomment string None +ds string None + +# Partition Information +col_name data_type comment + +ds string None Index: ql/src/test/queries/clientpositive/describe_pretty.q =================================================================== --- ql/src/test/queries/clientpositive/describe_pretty.q (revision 0) +++ ql/src/test/queries/clientpositive/describe_pretty.q (revision 0) @@ -0,0 +1,80 @@ +-- test comment indent processing for multi-line comments + +CREATE TABLE test_table( + col1 INT COMMENT 'col1 one line comment', + col2 STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines', + col5 STRING COMMENT 'col5 very long multi-line comment where each line is very long by itself and is likely to spill +into multiple lines. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin in dolor nisl, sodales +adipiscing tortor. Integer venenatis', + col6 STRING COMMENT 'This comment has a very long single word ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvzxyz123 which will not fit in a line by itself for small column widths.', + col7_NoComment STRING) +COMMENT 'table comment +two lines'; + +SET hive.cli.pretty.output.num.cols=80; + +-- There will be an extra tab at the end of each comment line in the output. +-- This is because DESCRIBE command separates the column, type and +-- comment field using a \t. DESCRIBE PRETTY uses spaces instead +-- of \t to separate columns. Hive gets confused when it parses the string +-- table description constructed in MetaDataPrettyFormatUtils, and adds a tab +-- at the end of each line. +-- There are three ways to address this: +-- 1. Pad each row to the full terminal width with extra spaces. +-- 2. Assume a maximum tab width of 8, and subtract 2 * 8 spaces from the +-- available line width. This approach wastes upto 2 * 8 - 2 columns. +-- 3. Since the pretty output is meant only for human consumption, do nothing. +-- Just add a comment to the unit test file explaining what is happening. +-- This is the approach chosen. + +DESCRIBE PRETTY test_table; + +SET hive.cli.pretty.output.num.cols=200; +DESCRIBE PRETTY test_table; + +SET hive.cli.pretty.output.num.cols=50; +DESCRIBE PRETTY test_table; + +SET hive.cli.pretty.output.num.cols=60; +DESCRIBE PRETTY test_table; + +CREATE TABLE test_table_very_long_column_name( + col1 INT COMMENT 'col1 one line comment', + col2_abcdefghiklmnopqrstuvxyz STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines') +; + +SET hive.cli.pretty.output.num.cols=80; +DESCRIBE PRETTY test_table_very_long_column_name; + +SET hive.cli.pretty.output.num.cols=20; +DESCRIBE PRETTY test_table_very_long_column_name; + +CREATE TABLE test_table_partitioned( + col1 INT COMMENT 'col1 one line comment', + col2 STRING COMMENT 'col2 +two lines comment', + col3 STRING COMMENT 'col3 +three lines +comment', + col4 STRING COMMENT 'col4 very long comment that is greater than 80 chars and is likely to spill into multiple lines', + col5 STRING COMMENT 'col5 very long multi-line comment where each line is very long by itself and is likely to spill +into multiple lines. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin in dolor nisl, sodales +adipiscing tortor. Integer venenatis', + col6 STRING COMMENT 'This comment has a very long single word ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvzxyz123 which will not fit in a line by itself for small column widths.', + col7_NoComment STRING) +COMMENT 'table comment +two lines' +PARTITIONED BY (ds STRING); + +SET hive.cli.pretty.output.num.cols=60; +DESCRIBE PRETTY test_table_partitioned; Index: ql/src/java/org/apache/hadoop/hive/ql/plan/DescTableDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/DescTableDesc.java (revision 1435342) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DescTableDesc.java (working copy) @@ -42,6 +42,12 @@ String colPath; boolean isExt; boolean isFormatted; + + /** Show pretty output? This has more human-readable formatting than + * isFormatted mode. + */ + private boolean isPretty; + /** * table name for the result of describe table. */ @@ -63,6 +69,7 @@ Map partSpec, String colPath) { this.isExt = false; this.isFormatted = false; + this.isPretty = false; this.partSpec = partSpec; this.resFile = resFile.toString(); this.tableName = tableName; @@ -92,7 +99,7 @@ this.isExt = isExt; } - /** + /** * @return the isFormatted */ public boolean isFormatted() { @@ -107,6 +114,14 @@ this.isFormatted = isFormat; } + public boolean isPretty() { + return this.isPretty; + } + + public void setPretty(boolean isPretty) { + this.isPretty = isPretty; + } + /** * @return the tableName */ Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 1435342) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -944,7 +944,7 @@ descStatement @init { msgs.push("describe statement"); } @after { msgs.pop(); } - : (KW_DESCRIBE|KW_DESC) (descOptions=KW_FORMATTED|descOptions=KW_EXTENDED)? (parttype=descPartTypeExpr) -> ^(TOK_DESCTABLE $parttype $descOptions?) + : (KW_DESCRIBE|KW_DESC) (descOptions=KW_FORMATTED|descOptions=KW_EXTENDED|descOptions=KW_PRETTY)? (parttype=descPartTypeExpr) -> ^(TOK_DESCTABLE $parttype $descOptions?) | (KW_DESCRIBE|KW_DESC) KW_FUNCTION KW_EXTENDED? (name=descFuncNames) -> ^(TOK_DESCFUNCTION $name KW_EXTENDED?) | (KW_DESCRIBE|KW_DESC) KW_DATABASE KW_EXTENDED? (dbName=Identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) ; @@ -2519,6 +2519,7 @@ KW_EXPLAIN: 'EXPLAIN'; KW_EXTENDED: 'EXTENDED'; KW_FORMATTED: 'FORMATTED'; +KW_PRETTY: 'PRETTY'; KW_DEPENDENCY: 'DEPENDENCY'; KW_SERDE: 'SERDE'; KW_WITH: 'WITH'; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1435342) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -1737,6 +1737,7 @@ int descOptions = ast.getChild(1).getType(); descTblDesc.setFormatted(descOptions == HiveParser.KW_FORMATTED); descTblDesc.setExt(descOptions == HiveParser.KW_EXTENDED); + descTblDesc.setPretty(descOptions == HiveParser.KW_PRETTY); } rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), descTblDesc), conf)); Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataPrettyFormatUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataPrettyFormatUtils.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataPrettyFormatUtils.java (revision 0) @@ -0,0 +1,238 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata.formatting; + +import java.util.List; +import java.util.StringTokenizer; + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +import jline.Terminal; + +/** + * This class provides methods to format the output of DESCRIBE PRETTY + * in a human-readable way. + */ +public final class MetaDataPrettyFormatUtils { + + public static final int PRETTY_MAX_INTERCOL_SPACING = 4; + private static final int PRETTY_ALIGNMENT = 10; + /** + * Minimum length of the comment column. This is relevant only when the terminal width + * or hive.cli.pretty.output.num.cols is too small, or when there are very large column + * names. + * 10 was arbitrarily chosen. + */ + private static final int MIN_COMMENT_COLUMN_LEN = 10; + + private MetaDataPrettyFormatUtils() { + } + + /** + * @param prettyOutputNumCols The pretty output is formatted to fit within + * these many columns. + */ + public static String getAllColumnsInformation(List cols, + List partCols, int prettyOutputNumCols) { + StringBuilder columnInformation = new StringBuilder( + MetaDataFormatUtils.DEFAULT_STRINGBUILDER_SIZE); + int maxColNameLen = findMaxColumnNameLen(cols); + formatColumnsHeaderPretty(columnInformation, maxColNameLen, prettyOutputNumCols); + formatAllFieldsPretty(columnInformation, cols, maxColNameLen, prettyOutputNumCols); + + if ((partCols != null) && (!partCols.isEmpty())) { + columnInformation.append(MetaDataFormatUtils.LINE_DELIM) + .append("# Partition Information") + .append(MetaDataFormatUtils.LINE_DELIM); + formatColumnsHeaderPretty(columnInformation, maxColNameLen, prettyOutputNumCols); + formatAllFieldsPretty(columnInformation, partCols, maxColNameLen, prettyOutputNumCols); + } + + return columnInformation.toString(); + } + + /** + * Find the length of the largest column name. + */ + private static int findMaxColumnNameLen(List cols) { + int maxLen = -1; + for (FieldSchema col : cols) { + int colNameLen = col.getName().length(); + if (colNameLen > maxLen) { + maxLen = colNameLen; + } + } + return maxLen; + } + + /** + * @param maxColNameLen The length of the largest column name + */ + private static void formatColumnsHeaderPretty(StringBuilder columnInformation, + int maxColNameLen, int prettyOutputNumCols) { + String columnHeaders[] = MetaDataFormatUtils.getColumnsHeader(); + formatOutputPretty(columnHeaders[0], columnHeaders[1], columnHeaders[2], + columnInformation, maxColNameLen, prettyOutputNumCols); + columnInformation.append(MetaDataFormatUtils.LINE_DELIM); + } + + private static void formatAllFieldsPretty(StringBuilder tableInfo, + List cols, int maxColNameLen, int prettyOutputNumCols) { + for (FieldSchema col : cols) { + formatOutputPretty(col.getName(), col.getType(), + MetaDataFormatUtils.getComment(col), tableInfo, maxColNameLen, + prettyOutputNumCols); + } + } + + /** + * If the specified comment is too long, add line breaks at appropriate + * locations. Note that the comment may already include line-breaks + * specified by the user at table creation time. + * @param columnsAlreadyConsumed The number of columns on the current line + * that have already been consumed by the column name, column type and + * and the surrounding delimiters. + * @return The comment with line breaks added at appropriate locations. + */ + private static String breakCommentIntoMultipleLines(String comment, + int columnsAlreadyConsumed, int prettyOutputNumCols) { + + if (prettyOutputNumCols == -1) { + Terminal terminal = Terminal.getTerminal(); + prettyOutputNumCols = terminal.getTerminalWidth() - 1; + } + + int commentNumCols = prettyOutputNumCols - columnsAlreadyConsumed; + if (commentNumCols < MIN_COMMENT_COLUMN_LEN) { + commentNumCols = MIN_COMMENT_COLUMN_LEN; + } + + // Track the number of columns allocated for the comment that have + // already been consumed on the current line. + int commentNumColsConsumed = 0; + + StringTokenizer st = new StringTokenizer(comment, " \t\n\r\f", true); + // We use a StringTokenizer instead of a BreakIterator, because + // table comments often contain text that looks like code. For eg: + // 'Type0' => 0, // This is Type 0 + // 'Type1' => 1, // This is Type 1 + // BreakIterator is meant for regular text, and was found to give + // bad line breaks when we tried it out. + + StringBuilder commentBuilder = new StringBuilder(comment.length()); + while (st.hasMoreTokens()) { + String currWord = st.nextToken(); + if (currWord.equals("\n") || currWord.equals("\r") || currWord.equals("\f")) { + commentBuilder.append(currWord); + commentNumColsConsumed = 0; + continue; + } + if (commentNumColsConsumed + currWord.length() > commentNumCols) { + // currWord won't fit on the current line + if (currWord.length() > commentNumCols) { + // currWord is too long to split on a line even all by itself. + // Hence we have no option but to split it. The first chunk + // will go to the end of the current line. Subsequent chunks + // will be of length commentNumCols. The last chunk + // may be smaller. + while (currWord.length() > commentNumCols) { + int remainingLineLen = commentNumCols - commentNumColsConsumed; + String wordChunk = currWord.substring(0, remainingLineLen); + commentBuilder.append(wordChunk); + commentBuilder.append(MetaDataFormatUtils.LINE_DELIM); + commentNumColsConsumed = 0; + currWord = currWord.substring(remainingLineLen); + } + // Handle the last chunk + if (currWord.length() > 0) { + commentBuilder.append(currWord); + commentNumColsConsumed = currWord.length(); + } + } else { + // Start on a new line + commentBuilder.append(MetaDataFormatUtils.LINE_DELIM); + if (!currWord.equals(" ")) { + // When starting a new line, do not start with a space. + commentBuilder.append(currWord); + commentNumColsConsumed = currWord.length(); + } else { + commentNumColsConsumed = 0; + } + } + } else { + commentBuilder.append(currWord); + commentNumColsConsumed += currWord.length(); + } + } + return commentBuilder.toString(); + } + + /** + * Appends the specified text with alignment to sb. + * Also appends an appopriately sized delimiter. + * @return The number of columns consumed by the aligned string and the + * delimiter. + */ + private static int appendFormattedColumn(StringBuilder sb, String text, + int alignment) { + String paddedText = String.format("%-" + alignment + "s", text); + int delimCount = 0; + if (paddedText.length() < alignment + PRETTY_MAX_INTERCOL_SPACING) { + delimCount = (alignment + PRETTY_MAX_INTERCOL_SPACING) + - paddedText.length(); + } else { + delimCount = PRETTY_MAX_INTERCOL_SPACING; + } + String delim = StringUtils.repeat(" ", delimCount); + sb.append(paddedText); + sb.append(delim); + + return paddedText.length() + delim.length(); + } + + private static void formatOutputPretty(String colName, String colType, + String colComment, StringBuilder tableInfo, int maxColNameLength, + int prettyOutputNumCols) { + int colsConsumed = 0; + colsConsumed += appendFormattedColumn(tableInfo, colName, maxColNameLength + 1); + colsConsumed += appendFormattedColumn(tableInfo, colType, PRETTY_ALIGNMENT); + + colComment = breakCommentIntoMultipleLines(colComment, colsConsumed, prettyOutputNumCols); + + /* Comment indent processing for multi-line comments. + * Comments should be indented the same amount on each line + * if the first line comment starts indented by k, + * the following line comments should also be indented by k. + */ + String[] commentSegments = colComment.split("\n|\r|\r\n"); + tableInfo.append(trimTrailingWS(commentSegments[0])); + tableInfo.append(MetaDataFormatUtils.LINE_DELIM); + for (int i = 1; i < commentSegments.length; i++) { + tableInfo.append(StringUtils.repeat(" ", colsConsumed)); + tableInfo.append(trimTrailingWS(commentSegments[i])); + tableInfo.append(MetaDataFormatUtils.LINE_DELIM); + } + } + + private static String trimTrailingWS(String str) { + return str.replaceAll("\\s+$", ""); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java (revision 1435342) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java (working copy) @@ -94,7 +94,7 @@ public void describeTable(DataOutputStream out, String colPath, String tableName, Table tbl, Partition part, List cols, - boolean isFormatted, boolean isExt) + boolean isFormatted, boolean isExt, boolean isPretty) throws HiveException; /** Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java (revision 1435342) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java (working copy) @@ -54,6 +54,15 @@ private static final int separator = Utilities.tabCode; private static final int terminator = Utilities.newLineCode; + /** The number of columns to be used in pretty formatting metadata output. + * If -1, then the current terminal width is auto-detected and used. + */ + private final int prettyOutputNumCols; + + public TextMetaDataFormatter(int prettyOutputNumCols) { + this.prettyOutputNumCols = prettyOutputNumCols; + } + /** * Write an error message. */ @@ -124,13 +133,18 @@ public void describeTable(DataOutputStream outStream, String colPath, String tableName, Table tbl, Partition part, List cols, - boolean isFormatted, boolean isExt) + boolean isFormatted, boolean isExt, boolean isPretty) throws HiveException { try { if (colPath.equals(tableName)) { + List partCols = tbl.isPartitioned() ? tbl.getPartCols() : null; outStream.writeBytes( - MetaDataFormatUtils.getAllColumnsInformation( - cols, tbl.isPartitioned() ? tbl.getPartCols() : null)); + isPretty ? + MetaDataPrettyFormatUtils.getAllColumnsInformation( + cols, partCols, prettyOutputNumCols) + : + MetaDataFormatUtils.getAllColumnsInformation(cols, partCols) + ); } else { outStream.writeBytes(MetaDataFormatUtils.getAllColumnsInformation(cols)); } Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (revision 1435342) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (working copy) @@ -50,7 +50,7 @@ public static final String FIELD_DELIM = "\t"; public static final String LINE_DELIM = "\n"; - private static final int DEFAULT_STRINGBUILDER_SIZE = 2048; + static final int DEFAULT_STRINGBUILDER_SIZE = 2048; private static final int ALIGNMENT = 20; private MetaDataFormatUtils() { @@ -86,11 +86,10 @@ private static void formatAllFields(StringBuilder tableInfo, List cols) { for (FieldSchema col : cols) { - formatFieldSchemas(tableInfo, col); + formatOutput(col.getName(), col.getType(), getComment(col), tableInfo); } } - public static String getAllColumnsInformation(Index index) { StringBuilder indexInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); @@ -263,9 +262,8 @@ } } - private static void formatFieldSchemas(StringBuilder tableInfo, FieldSchema col) { - String comment = col.getComment() != null ? col.getComment() : "None"; - formatOutput(col.getName(), col.getType(), comment, tableInfo); + static String getComment(FieldSchema col) { + return col.getComment() != null ? col.getComment() : "None"; } private static String formatDate(long timeInSeconds) { Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java (revision 1435342) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java (working copy) @@ -145,7 +145,8 @@ public void describeTable(DataOutputStream out, String colPath, String tableName, Table tbl, Partition part, List cols, - boolean isFormatted, boolean isExt) + boolean isFormatted, boolean isExt, + boolean isPretty) throws HiveException { MapBuilder builder = MapBuilder.create(); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1435342) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -205,7 +205,8 @@ HiveConf.ConfVars.HIVE_DDL_OUTPUT_FORMAT.varname, "text"))) { formatter = new JsonMetaDataFormatter(); } else { - formatter = new TextMetaDataFormatter(); + formatter = new TextMetaDataFormatter( + conf.getIntVar(HiveConf.ConfVars.CLIPRETTYOUTPUTNUMCOLS)); } INTERMEDIATE_ARCHIVED_DIR_SUFFIX = @@ -2853,7 +2854,7 @@ } formatter.describeTable(outStream, colPath, tableName, tbl, part, cols, - descTbl.isFormatted(), descTbl.isExt()); + descTbl.isFormatted(), descTbl.isExt(), descTbl.isPretty()); LOG.info("DDLTask: written data for " + tbl.getTableName()); ((FSDataOutputStream) outStream).close();