diff --git beeline/src/java/org/apache/hive/beeline/BeeLine.java beeline/src/java/org/apache/hive/beeline/BeeLine.java index add0298..bc1a6c3 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -1003,6 +1003,7 @@ private String obtainPasswordFromFile(String passwordFilePath) { public void updateOptsForCli() { getOpts().updateBeeLineOptsFromConf(); getOpts().setShowHeader(false); + getOpts().setEscapeCRLF(false); getOpts().setOutputFormat("dsv"); getOpts().setDelimiterForDSV(' '); getOpts().setNullEmptyString(true); diff --git beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 6c2360c..dad91e6 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -70,6 +70,7 @@ private boolean silent = false; private boolean color = false; private boolean showHeader = true; + private boolean escapeCRLF = false; private boolean showDbInPrompt = false; private int headerInterval = 100; private boolean fastConnect = true; @@ -495,6 +496,21 @@ public boolean getShowHeader() { } } + public void setEscapeCRLF(boolean escapeCRLF) { + this.escapeCRLF = escapeCRLF; + } + + public boolean getEscapeCRLF() { + if (beeLine.isBeeLine()) { + return escapeCRLF; + } else { + boolean flag; + HiveConf conf = beeLine.getCommands().getHiveConf(true); + flag = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CLI_PRINT_ESCAPE_CRLF); + return flag; + } + } + public void setShowDbInPrompt(boolean showDbInPrompt) { this.showDbInPrompt = showDbInPrompt; } diff --git beeline/src/java/org/apache/hive/beeline/Rows.java beeline/src/java/org/apache/hive/beeline/Rows.java index 56c0069..a5e402d 100644 --- beeline/src/java/org/apache/hive/beeline/Rows.java +++ beeline/src/java/org/apache/hive/beeline/Rows.java @@ -30,6 +30,8 @@ import java.util.Arrays; import java.util.Iterator; +import org.apache.hadoop.hive.common.cli.EscapeCRLFHelper; + /** * Abstract base class representing a set of rows to be displayed. * Holds column values as strings @@ -168,6 +170,10 @@ public String toString(){ value = o.toString(); } + if (beeLine.getOpts().getEscapeCRLF()) { + value = EscapeCRLFHelper.escapeCRLF(value); + } + values[i] = value.intern(); sizes[i] = value.length(); } diff --git beeline/src/main/resources/BeeLine.properties beeline/src/main/resources/BeeLine.properties index 707188e..6fca953 100644 --- beeline/src/main/resources/BeeLine.properties +++ beeline/src/main/resources/BeeLine.properties @@ -173,6 +173,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --property-file= the file to read connection properties (url, driver, user, password) from\n \ \ --color=[true/false] control whether color is used for display\n \ \ --showHeader=[true/false] show column names in query results\n \ +\ --escapeCRLF=[true/false] show carriage return and line feeds in query results as escaped \\r and \\n \n \ \ --headerInterval=ROWS; the interval between which heades are displayed\n \ \ --fastConnect=[true/false] skip building table/column list for tab-completion\n \ \ --autoCommit=[true/false] enable/disable automatic transaction commit\n \ diff --git beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java index 068bb8d..5270cfe 100644 --- beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java +++ beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java @@ -132,6 +132,20 @@ public void testSetHeaderValue() { } @Test + public void testSetEscapeCRLF() { + verifyCMD( + "create database if not exists test;\n" + + "drop table if exists test.testTbl;\ncreate table test.testTbl(a string, b string);\n" + + "insert into table test.testTbl values(\"no cr lf\", \"a cr \r and a lf \n word word end CRLF \r\n\");\n" + + "select * from test.testTbl;\n" + + "set hive.cli.print.escape.crlf=true;\n" + + "select * from test.testTbl;\n", + "no cr lf" + "\t" + "a cr \r and a lf \n word word end CRLF \r\n\n" + + "no cr lf" + "\t" + "a cr \\r and a lf \\n word word end CRLF \\r\\n\n", + os, null, ERRNO_OK, true); + } + + @Test public void testHelp() { verifyCMD(null, "usage: hive", os, new String[] { "-H" }, ERRNO_ARGS, true); } diff --git cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 54d17df..22996f5 100644 --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hive.common.HiveInterruptUtils; import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.common.LogUtils.LogInitializationException; +import org.apache.hadoop.hive.common.cli.EscapeCRLFHelper; import org.apache.hadoop.hive.common.cli.ShellCmdExecutor; import org.apache.hadoop.hive.common.io.CachingPrintStream; import org.apache.hadoop.hive.common.io.FetchConverter; @@ -222,6 +223,7 @@ private String getFirstCmd(String cmd, int length) { int processLocalCmd(String cmd, CommandProcessor proc, CliSessionState ss) { int tryCount = 0; boolean needRetry; + boolean escapeCRLF = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CLI_PRINT_ESCAPE_CRLF); int ret = 0; do { @@ -259,6 +261,9 @@ int processLocalCmd(String cmd, CommandProcessor proc, CliSessionState ss) { } while (qp.getResults(res)) { for (String r : res) { + if (escapeCRLF) { + r = EscapeCRLFHelper.escapeCRLF(r); + } out.println(r); } counter += res.size(); diff --git common/src/java/org/apache/hadoop/hive/common/cli/EscapeCRLFHelper.java common/src/java/org/apache/hadoop/hive/common/cli/EscapeCRLFHelper.java new file mode 100644 index 0000000..6af1cf6 --- /dev/null +++ common/src/java/org/apache/hadoop/hive/common/cli/EscapeCRLFHelper.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.cli; + +public class EscapeCRLFHelper { + + private static final char CARRIAGE_RETURN = '\r'; + private static final char LINE_FEED = '\n'; + + /* + * Substitute for any carriage return or line feed characters in line with the escaped + * 2-character sequences \r or \n. + * + * @param line the string for the CRLF substitution. + * @return If there were no replacements, then just return line. Otherwise, a new String with + * escaped CRLF. + */ + public static String escapeCRLF(String line) { + + StringBuilder sb = null; + int lastNonCRLFIndex = 0; + int index = 0; + final int length = line.length(); + while (index < length) { + char ch = line.charAt(index); + if (ch == CARRIAGE_RETURN || ch == LINE_FEED) { + if (sb == null) { + + // We defer allocation until we really need it since in the common case there is + // no CRLF substitution. + sb = new StringBuilder(); + } + if (lastNonCRLFIndex < index) { + + // Copy an intervening non-CRLF characters up to but not including current 'index'. + sb.append(line.substring(lastNonCRLFIndex, index)); + } + lastNonCRLFIndex = ++index; + if (ch == CARRIAGE_RETURN) { + sb.append("\\r"); + } else { + sb.append("\\n"); + } + } else { + index++; + } + } + if (sb == null) { + + // No CRLF substitution -- return original line. + return line; + } else { + if (lastNonCRLFIndex < index) { + + // Copy an intervening non-CRLF characters up to but not including current 'index'. + sb.append(line.substring(lastNonCRLFIndex, index)); + } + return sb.toString(); + } + } +} \ No newline at end of file diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d0e2ad2..a8e7021 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2237,6 +2237,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false, "Whether to print the names of the columns in query output."), + HIVE_CLI_PRINT_ESCAPE_CRLF("hive.cli.print.escape.crlf", false, + "Whether to print carriage returns and line feeds in row output as escaped \\r and \\n"), + HIVE_CLI_TEZ_SESSION_ASYNC("hive.cli.tez.session.async", true, "Whether to start Tez\n" + "session in background when running CLI with Tez, allowing CLI to be available earlier."), diff --git data/files/test.parquet data/files/test.parquet new file mode 100644 index 0000000..3f2cd97 Binary files /dev/null and data/files/test.parquet differ diff --git itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java index d84ff39..38dd9f1 100644 --- itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java +++ itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java @@ -44,6 +44,7 @@ "!set verbose false", "!set silent true", "!set showheader false", + "!set escapeCRLF false", "USE default;", "SHOW TABLES;", }; @@ -56,6 +57,7 @@ "!set verbose true", "!set silent false", "!set showheader true", + "!set escapeCRLF false", "!set outputformat table", "USE default;" }; diff --git ql/src/test/queries/clientpositive/cli_print_escape_crlf.q ql/src/test/queries/clientpositive/cli_print_escape_crlf.q new file mode 100644 index 0000000..98775d1 --- /dev/null +++ ql/src/test/queries/clientpositive/cli_print_escape_crlf.q @@ -0,0 +1,16 @@ +create table repro (lvalue int, charstring string) stored as parquet; + +LOAD DATA LOCAL INPATH '../../data/files/test.parquet' overwrite into table repro; + +set hive.fetch.task.conversion=more; + + +select count(*) from repro; + +set hive.cli.print.escape.crlf=false; +select * from repro; + + +set hive.cli.print.escape.crlf=true; +select * from repro; + diff --git ql/src/test/results/clientpositive/cli_print_escape_crlf.q.out ql/src/test/results/clientpositive/cli_print_escape_crlf.q.out new file mode 100644 index 0000000..6455bd7 --- /dev/null +++ ql/src/test/results/clientpositive/cli_print_escape_crlf.q.out @@ -0,0 +1,50 @@ +PREHOOK: query: create table repro (lvalue int, charstring string) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@repro +POSTHOOK: query: create table repro (lvalue int, charstring string) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@repro +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/test.parquet' overwrite into table repro +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@repro +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/test.parquet' overwrite into table repro +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@repro +PREHOOK: query: select count(*) from repro +PREHOOK: type: QUERY +PREHOOK: Input: default@repro +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from repro +POSTHOOK: type: QUERY +POSTHOOK: Input: default@repro +#### A masked pattern was here #### +3 +PREHOOK: query: select * from repro +PREHOOK: type: QUERY +PREHOOK: Input: default@repro +#### A masked pattern was here #### +POSTHOOK: query: select * from repro +POSTHOOK: type: QUERY +POSTHOOK: Input: default@repro +#### A masked pattern was here #### +1 newline +here +2 carriage return +here +3 both +here +PREHOOK: query: select * from repro +PREHOOK: type: QUERY +PREHOOK: Input: default@repro +#### A masked pattern was here #### +POSTHOOK: query: select * from repro +POSTHOOK: type: QUERY +POSTHOOK: Input: default@repro +#### A masked pattern was here #### +1 newline\nhere +2 carriage return\rhere +3 both\r\nhere