commit 54b76fa12ba4488034905fe34915ab0406051f8b Author: Andrew Sherman Date: Tue Jan 2 15:55:43 2018 -0800 HIVE-18367: Fix 'describe extended' so that it works on tables that have special white space characters in the row format. Some test changes because the 'describe extended' output now always comes on one line. This seems better. Anyone who needs to parse the output (for example to get view text) can use 'describe formatted' diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 065f258abcfb4bf9ba649c8183c76c031e8fdd6f..1f5cd2ebaf0aed7535b13fdf27b845a6d962dbd9 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -1130,4 +1130,19 @@ public void testCustomDelimiterBeelineCmd() throws Throwable { argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } + + /** + * Test 'describe extended' on tables that have special white space characters in the row format. + */ + @Test + public void testDescribeExtended() throws Throwable { + String SCRIPT_TEXT = "drop table if exists describeDelim;" + + "create table describeDelim (orderid int, orderdate string, customerid int)" + + " ROW FORMAT DELIMITED FIELDS terminated by '\\t' LINES terminated by '\\n';" + + "describe extended describeDelim;"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, Arrays.asList( + new Tuple<>("Detailed Table Information.*line.delim=\\\\n", true), + new Tuple<>("Detailed Table Information.*field.delim=\\\\t", true))); + } } diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index ffeee69f801330710481c4b0b3517fcb4e17a986..7858944ff66845416ae4a45466b7edfebbb8efc5 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -1617,4 +1617,39 @@ private void checkVertices() { } return verticesLists; } + + /** + * Test 'describe extended' on tables that have special white space characters in the row format. + */ + @Test + public void testDescribe() throws Exception { + try (Statement stmt = conTestDb.createStatement()) { + String table = "testDescribe"; + stmt.execute("drop table if exists " + table); + stmt.execute("create table " + table + " (orderid int, orderdate string, customerid int)" + + " ROW FORMAT DELIMITED FIELDS terminated by '\\t' LINES terminated by '\\n'"); + String extendedDescription = getDetailedTableDescription(stmt, table); + assertNotNull("could not get Detailed Table Information", extendedDescription); + assertTrue("description appears truncated", extendedDescription.endsWith(")")); + assertTrue("bad line delimiter", extendedDescription.contains("line.delim=\\n")); + assertTrue("bad field delimiter", extendedDescription.contains("field.delim=\\t")); + } + } + + /** + * Get Detailed Table Information via jdbc + */ + private String getDetailedTableDescription(Statement stmt, String table) throws SQLException { + String extendedDescription = null; + try (ResultSet rs = stmt.executeQuery("describe extended " + table)) { + while (rs.next()) { + String out = rs.getString(1); + String tableInfo = rs.getString(2); + if ("Detailed Table Information".equals(out)) { // from TextMetaDataFormatter + extendedDescription = tableInfo; + } + } + } + return extendedDescription; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java index 0f1e893de62c2b713ecf0ed4bd37739ba22e6b7c..938de66fc75643c1baeec092be555deb89ae4223 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java @@ -210,7 +210,8 @@ public void describeTable(DataOutputStream outStream, String colPath, // show table information outStream.write(("Detailed Table Information").getBytes("UTF-8")); outStream.write(separator); - outStream.write(tbl.getTTable().toString().getBytes("UTF-8")); + String tableDesc = HiveStringUtils.escapeJava(tbl.getTTable().toString()); + outStream.write(tableDesc.getBytes("UTF-8")); outStream.write(separator); outStream.write(terminator); } diff --git ql/src/test/results/clientpositive/create_view.q.out ql/src/test/results/clientpositive/create_view.q.out index ffe69eb8c7496f3a7892044d701e46d2cd71df60..74615235415bed80b89b4850b50fa4cd31370a1a 100644 --- ql/src/test/results/clientpositive/create_view.q.out +++ ql/src/test/results/clientpositive/create_view.q.out @@ -724,8 +724,6 @@ POSTHOOK: Input: default@view8 c string #### A masked pattern was here #### -FROM table1, viewExpandedText:SELECT `_c0` AS `c` FROM (SELECT `test_translate`('abc', 'a', 'b') -FROM `default`.`table1`) `default.view8`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED view8 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view8 @@ -801,8 +799,6 @@ POSTHOOK: Input: default@view9 m int #### A masked pattern was here #### -FROM src, viewExpandedText:SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`)) -FROM `default`.`src`) `default.view9`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED view9 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view9 @@ -878,8 +874,6 @@ POSTHOOK: Input: default@view9 m int #### A masked pattern was here #### -FROM src, viewExpandedText:SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`)) -FROM `default`.`src`) `default.view9`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED view9 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view9 @@ -1021,8 +1015,6 @@ POSTHOOK: Input: default@view11 boom int #### A masked pattern was here #### -FROM table1, viewExpandedText:SELECT `test_explode`(array(1,2,3)) AS (`boom`) -FROM `default`.`table1`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED view11 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view11 @@ -1176,8 +1168,6 @@ POSTHOOK: Input: default@view13 key int #### A masked pattern was here #### -FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 ON key) s, viewExpandedText:SELECT `s`.`key` -FROM `default`.`srcbucket` TABLESAMPLE (BUCKET 1 OUT OF 5 ON `key`) `s`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED view13 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view13 @@ -1280,24 +1270,6 @@ k2 string v2 string #### A masked pattern was here #### - unionsrc2.key as k2, unionsrc2.value as v2 -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION ALL - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key), viewExpandedText:SELECT `unionsrc1`.`key` as `k1`, `unionsrc1`.`value` as `v1`, - `unionsrc2`.`key` as `k2`, `unionsrc2`.`value` as `v2` -FROM (select 'tst1' as `key`, cast(count(1) as string) as `value` from `default`.`src` `s1` - UNION ALL - select `s2`.`key` as `key`, `s2`.`value` as `value` from `default`.`src` `s2` where `s2`.`key` < 10) `unionsrc1` -JOIN - (select 'tst1' as `key`, cast(count(1) as string) as `value` from `default`.`src` `s3` - UNION ALL - select `s4`.`key` as `key`, `s4`.`value` as `value` from `default`.`src` `s4` where `s4`.`key` < 10) `unionsrc2` -ON (`unionsrc1`.`key` = `unionsrc2`.`key`), tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED view14 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view14 @@ -1412,10 +1384,6 @@ key string value_count bigint #### A masked pattern was here #### -FROM src -GROUP BY key, viewExpandedText:SELECT `src`.`key`,COUNT(`src`.`value`) AS `value_count` -FROM `default`.`src` -GROUP BY `src`.`key`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED view15 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view15 @@ -1499,8 +1467,6 @@ POSTHOOK: Input: default@view16 value string #### A masked pattern was here #### -FROM src, viewExpandedText:SELECT DISTINCT `src`.`value` -FROM `default`.`src`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED view16 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view16 diff --git ql/src/test/results/clientpositive/create_view_partitioned.q.out ql/src/test/results/clientpositive/create_view_partitioned.q.out index 3a387573f2169d4544c098c7127323514f903274..2cfb815292f74410ac8c433842f9994d15b064f8 100644 --- ql/src/test/results/clientpositive/create_view_partitioned.q.out +++ ql/src/test/results/clientpositive/create_view_partitioned.q.out @@ -45,10 +45,6 @@ value string value string #### A masked pattern was here #### -FROM src -WHERE key=86, viewExpandedText:SELECT `src`.`key`, `src`.`value` -FROM `default`.`src` -WHERE `src`.`key`=86, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: DESCRIBE FORMATTED vp1 PREHOOK: type: DESCTABLE PREHOOK: Input: default@vp1 diff --git ql/src/test/results/clientpositive/cteViews.q.out ql/src/test/results/clientpositive/cteViews.q.out index 3a9bd43701e1aca00da6102359eb29ff40837153..1f1e862ffd18261ee38778b5eee70e588c8b9d75 100644 --- ql/src/test/results/clientpositive/cteViews.q.out +++ ql/src/test/results/clientpositive/cteViews.q.out @@ -45,8 +45,6 @@ POSTHOOK: Input: default@v key string #### A masked pattern was here #### -select key from cte, viewExpandedText:with cte as (select `src`.`key`, `src`.`value` from `default`.`src` order by key limit 5) -select `cte`.`key` from cte, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: create database bug PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:bug @@ -120,8 +118,6 @@ key string value string #### A masked pattern was here #### -select * from cte, viewExpandedText:with cte as (select `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 5) -select `cte`.`key`, `cte`.`value` from cte, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: create database bug PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:bug @@ -193,8 +189,6 @@ POSTHOOK: Input: default@v key string #### A masked pattern was here #### -select * from src1, viewExpandedText:with src1 as (select `src`.`key` from `default`.`src` order by key limit 5) -select `src1`.`key` from src1, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: create database bug PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:bug @@ -258,8 +252,6 @@ POSTHOOK: Input: default@v key string #### A masked pattern was here #### -select * from src1 a where a.key is not null, viewExpandedText:with src1 as (select `src`.`key` from `default`.`src` order by key limit 5) -select `a`.`key` from src1 a where `a`.`key` is not null, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git ql/src/test/results/clientpositive/llap/cbo_rp_unionDistinct_2.q.out ql/src/test/results/clientpositive/llap/cbo_rp_unionDistinct_2.q.out index 3b3983f89633618c46db9c5d32dea894e9fc07a3..6226d14d8b1e563772e35fde99ec7acb5e5b450e 100644 --- ql/src/test/results/clientpositive/llap/cbo_rp_unionDistinct_2.q.out +++ ql/src/test/results/clientpositive/llap/cbo_rp_unionDistinct_2.q.out @@ -350,20 +350,6 @@ key string value string #### A masked pattern was here #### -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select `u2`.`key`, `u2`.`value` from `default`.`u2` -union all -select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 @@ -426,16 +412,6 @@ key string value string #### A masked pattern was here #### -( -select distinct * from u1 -union -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 @@ -493,16 +469,6 @@ key string value string #### A masked pattern was here #### -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union all -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 diff --git ql/src/test/results/clientpositive/llap/subquery_views.q.out ql/src/test/results/clientpositive/llap/subquery_views.q.out index 93913625eed847dae0271274b6c505c93839ba81..3ba7d203321cd541e17e57e86b8d580681ef267d 100644 --- ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -32,16 +32,6 @@ key string value string #### A masked pattern was here #### -from src b -where exists - (select a.key - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_9'), viewExpandedText:select `b`.`key`, `b`.`value` -from `default`.`src` `b` -where exists - (select `a`.`key` - from `default`.`src` `a` - where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_9'), tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from cv1 where cv1.key in (select key from cv1 c where c.key > '95') PREHOOK: type: QUERY @@ -95,18 +85,6 @@ key string value string #### A masked pattern was here #### -from src b -where b.key not in - (select a.key - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_11' - ), viewExpandedText:select `b`.`key`, `b`.`value` -from `default`.`src` `b` -where `b`.`key` not in - (select `a`.`key` - from `default`.`src` `a` - where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_11' - ), tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: explain select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11') @@ -451,14 +429,6 @@ value string _c2 bigint #### A masked pattern was here #### -from src b -where b.key in (select key from src where src.key > '8') -group by key, value -having count(*) in (select count(*) from src s1 where s1.key > '9' group by s1.key ), viewExpandedText:select `b`.`key`, `b`.`value`, count(*) -from `default`.`src` `b` -where `b`.`key` in (select `src`.`key` from `default`.`src` where `src`.`key` > '8') -group by `b`.`key`, `b`.`value` -having count(*) in (select count(*) from `default`.`src` `s1` where `s1`.`key` > '9' group by `s1`.`key` ), tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from cv3 PREHOOK: type: QUERY PREHOOK: Input: default@cv3 diff --git ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out index dc4e55478a24a874e42798b71b0facf763cba753..789c5601a977bc7bb8a2f427263b09c41595b7d0 100644 --- ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out +++ ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out @@ -242,20 +242,6 @@ key string value string #### A masked pattern was here #### -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select `u2`.`key`, `u2`.`value` from `default`.`u2` -union all -select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 @@ -318,16 +304,6 @@ key string value string #### A masked pattern was here #### -( -select distinct * from u1 -union -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 @@ -385,16 +361,6 @@ key string value string #### A masked pattern was here #### -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union all -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 diff --git ql/src/test/results/clientpositive/spark/subquery_views.q.out ql/src/test/results/clientpositive/spark/subquery_views.q.out index 9a1c25fffd9660bfe1509ade3eebb1ea93f66c69..815d6bb01aae9388eeb530d63639dee62fb5355d 100644 --- ql/src/test/results/clientpositive/spark/subquery_views.q.out +++ ql/src/test/results/clientpositive/spark/subquery_views.q.out @@ -32,16 +32,6 @@ key string value string #### A masked pattern was here #### -from src b -where exists - (select a.key - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_9'), viewExpandedText:select `b`.`key`, `b`.`value` -from `default`.`src` `b` -where exists - (select `a`.`key` - from `default`.`src` `a` - where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_9'), tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from cv1 where cv1.key in (select key from cv1 c where c.key > '95') PREHOOK: type: QUERY @@ -95,18 +85,6 @@ key string value string #### A masked pattern was here #### -from src b -where b.key not in - (select a.key - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_11' - ), viewExpandedText:select `b`.`key`, `b`.`value` -from `default`.`src` `b` -where `b`.`key` not in - (select `a`.`key` - from `default`.`src` `a` - where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_11' - ), tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: explain select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11') @@ -486,14 +464,6 @@ value string _c2 bigint #### A masked pattern was here #### -from src b -where b.key in (select key from src where src.key > '8') -group by key, value -having count(*) in (select count(*) from src s1 where s1.key > '9' group by s1.key ), viewExpandedText:select `b`.`key`, `b`.`value`, count(*) -from `default`.`src` `b` -where `b`.`key` in (select `src`.`key` from `default`.`src` where `src`.`key` > '8') -group by `b`.`key`, `b`.`value` -having count(*) in (select count(*) from `default`.`src` `s1` where `s1`.`key` > '9' group by `s1`.`key` ), tableType:VIRTUAL_VIEW, rewriteEnabled:false) PREHOOK: query: select * from cv3 PREHOOK: type: QUERY PREHOOK: Input: default@cv3