commit dabcb2995a56573fca4818abd045735cc6647f6e Author: Andrew Sherman Date: Tue Jan 2 15:55:43 2018 -0800 HIVE-18367: Fix 'describe extended' so that it works on tables that have special white space characters in the row format. Some test changes because the 'describe extended' output now always comes on one line. This seems better. Anyone who needs to parse the output (for example to get view text) can use 'describe formatted' diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 065f258abcfb4bf9ba649c8183c76c031e8fdd6f..1f5cd2ebaf0aed7535b13fdf27b845a6d962dbd9 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -1130,4 +1130,19 @@ public void testCustomDelimiterBeelineCmd() throws Throwable { argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } + + /** + * Test 'describe extended' on tables that have special white space characters in the row format. + */ + @Test + public void testDescribeExtended() throws Throwable { + String SCRIPT_TEXT = "drop table if exists describeDelim;" + + "create table describeDelim (orderid int, orderdate string, customerid int)" + + " ROW FORMAT DELIMITED FIELDS terminated by '\\t' LINES terminated by '\\n';" + + "describe extended describeDelim;"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, Arrays.asList( + new Tuple<>("Detailed Table Information.*line.delim=\\\\n", true), + new Tuple<>("Detailed Table Information.*field.delim=\\\\t", true))); + } } diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index ffeee69f801330710481c4b0b3517fcb4e17a986..7858944ff66845416ae4a45466b7edfebbb8efc5 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -1617,4 +1617,39 @@ private void checkVertices() { } return verticesLists; } + + /** + * Test 'describe extended' on tables that have special white space characters in the row format. + */ + @Test + public void testDescribe() throws Exception { + try (Statement stmt = conTestDb.createStatement()) { + String table = "testDescribe"; + stmt.execute("drop table if exists " + table); + stmt.execute("create table " + table + " (orderid int, orderdate string, customerid int)" + + " ROW FORMAT DELIMITED FIELDS terminated by '\\t' LINES terminated by '\\n'"); + String extendedDescription = getDetailedTableDescription(stmt, table); + assertNotNull("could not get Detailed Table Information", extendedDescription); + assertTrue("description appears truncated", extendedDescription.endsWith(")")); + assertTrue("bad line delimiter", extendedDescription.contains("line.delim=\\n")); + assertTrue("bad field delimiter", extendedDescription.contains("field.delim=\\t")); + } + } + + /** + * Get Detailed Table Information via jdbc + */ + private String getDetailedTableDescription(Statement stmt, String table) throws SQLException { + String extendedDescription = null; + try (ResultSet rs = stmt.executeQuery("describe extended " + table)) { + while (rs.next()) { + String out = rs.getString(1); + String tableInfo = rs.getString(2); + if ("Detailed Table Information".equals(out)) { // from TextMetaDataFormatter + extendedDescription = tableInfo; + } + } + } + return extendedDescription; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java index 932fc5f66e1999a04a6b1e6dfa777130ef9c409b..cddceb74a362aa009f047725ccf313802caf6b06 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java @@ -208,7 +208,8 @@ public void describeTable(DataOutputStream outStream, String colPath, // show table information outStream.write(("Detailed Table Information").getBytes("UTF-8")); outStream.write(separator); - outStream.write(tbl.getTTable().toString().getBytes("UTF-8")); + String tableDesc = HiveStringUtils.escapeJava(tbl.getTTable().toString()); + outStream.write(tableDesc.getBytes("UTF-8")); outStream.write(separator); outStream.write(terminator); } diff --git ql/src/test/results/clientpositive/create_view.q.out ql/src/test/results/clientpositive/create_view.q.out index 32532f8678a458018d14a5097ff20f7bed1389e5..74615235415bed80b89b4850b50fa4cd31370a1a 100644 --- ql/src/test/results/clientpositive/create_view.q.out +++ ql/src/test/results/clientpositive/create_view.q.out @@ -724,8 +724,6 @@ POSTHOOK: Input: default@view8 c string #### A masked pattern was here #### -FROM table1, viewExpandedText:SELECT `_c0` AS `c` FROM (SELECT `test_translate`('abc', 'a', 'b') -FROM `default`.`table1`) `default.view8`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED view8 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view8 @@ -801,8 +799,6 @@ POSTHOOK: Input: default@view9 m int #### A masked pattern was here #### -FROM src, viewExpandedText:SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`)) -FROM `default`.`src`) `default.view9`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED view9 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view9 @@ -878,8 +874,6 @@ POSTHOOK: Input: default@view9 m int #### A masked pattern was here #### -FROM src, viewExpandedText:SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`)) -FROM `default`.`src`) `default.view9`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED view9 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view9 @@ -1021,8 +1015,6 @@ POSTHOOK: Input: default@view11 boom int #### A masked pattern was here #### -FROM table1, viewExpandedText:SELECT `test_explode`(array(1,2,3)) AS (`boom`) -FROM `default`.`table1`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED view11 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view11 @@ -1176,8 +1168,6 @@ POSTHOOK: Input: default@view13 key int #### A masked pattern was here #### -FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 ON key) s, viewExpandedText:SELECT `s`.`key` -FROM `default`.`srcbucket` TABLESAMPLE (BUCKET 1 OUT OF 5 ON `key`) `s`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED view13 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view13 @@ -1280,24 +1270,6 @@ k2 string v2 string #### A masked pattern was here #### - unionsrc2.key as k2, unionsrc2.value as v2 -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION ALL - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION ALL - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key), viewExpandedText:SELECT `unionsrc1`.`key` as `k1`, `unionsrc1`.`value` as `v1`, - `unionsrc2`.`key` as `k2`, `unionsrc2`.`value` as `v2` -FROM (select 'tst1' as `key`, cast(count(1) as string) as `value` from `default`.`src` `s1` - UNION ALL - select `s2`.`key` as `key`, `s2`.`value` as `value` from `default`.`src` `s2` where `s2`.`key` < 10) `unionsrc1` -JOIN - (select 'tst1' as `key`, cast(count(1) as string) as `value` from `default`.`src` `s3` - UNION ALL - select `s4`.`key` as `key`, `s4`.`value` as `value` from `default`.`src` `s4` where `s4`.`key` < 10) `unionsrc2` -ON (`unionsrc1`.`key` = `unionsrc2`.`key`), tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED view14 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view14 @@ -1412,10 +1384,6 @@ key string value_count bigint #### A masked pattern was here #### -FROM src -GROUP BY key, viewExpandedText:SELECT `src`.`key`,COUNT(`src`.`value`) AS `value_count` -FROM `default`.`src` -GROUP BY `src`.`key`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED view15 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view15 @@ -1499,8 +1467,6 @@ POSTHOOK: Input: default@view16 value string #### A masked pattern was here #### -FROM src, viewExpandedText:SELECT DISTINCT `src`.`value` -FROM `default`.`src`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED view16 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view16 diff --git ql/src/test/results/clientpositive/create_view_partitioned.q.out ql/src/test/results/clientpositive/create_view_partitioned.q.out index 05af809907135d7e734cd7385939a3a06ad207c0..2cfb815292f74410ac8c433842f9994d15b064f8 100644 --- ql/src/test/results/clientpositive/create_view_partitioned.q.out +++ ql/src/test/results/clientpositive/create_view_partitioned.q.out @@ -45,10 +45,6 @@ value string value string #### A masked pattern was here #### -FROM src -WHERE key=86, viewExpandedText:SELECT `src`.`key`, `src`.`value` -FROM `default`.`src` -WHERE `src`.`key`=86, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: DESCRIBE FORMATTED vp1 PREHOOK: type: DESCTABLE PREHOOK: Input: default@vp1 diff --git ql/src/test/results/clientpositive/cteViews.q.out ql/src/test/results/clientpositive/cteViews.q.out index a1920ff43c7af1408690cbaf232ebbb1ef1984df..1f1e862ffd18261ee38778b5eee70e588c8b9d75 100644 --- ql/src/test/results/clientpositive/cteViews.q.out +++ ql/src/test/results/clientpositive/cteViews.q.out @@ -45,8 +45,6 @@ POSTHOOK: Input: default@v key string #### A masked pattern was here #### -select key from cte, viewExpandedText:with cte as (select `src`.`key`, `src`.`value` from `default`.`src` order by key limit 5) -select `cte`.`key` from cte, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: create database bug PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:bug @@ -120,8 +118,6 @@ key string value string #### A masked pattern was here #### -select * from cte, viewExpandedText:with cte as (select `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 5) -select `cte`.`key`, `cte`.`value` from cte, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: create database bug PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:bug @@ -193,8 +189,6 @@ POSTHOOK: Input: default@v key string #### A masked pattern was here #### -select * from src1, viewExpandedText:with src1 as (select `src`.`key` from `default`.`src` order by key limit 5) -select `src1`.`key` from src1, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: create database bug PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:bug @@ -258,8 +252,6 @@ POSTHOOK: Input: default@v key string #### A masked pattern was here #### -select * from src1 a where a.key is not null, viewExpandedText:with src1 as (select `src`.`key` from `default`.`src` order by key limit 5) -select `a`.`key` from src1 a where `a`.`key` is not null, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git ql/src/test/results/clientpositive/llap/cbo_rp_unionDistinct_2.q.out ql/src/test/results/clientpositive/llap/cbo_rp_unionDistinct_2.q.out index 6b2ca61f6fcf1d1bc19f5d5861d602ee97fe4260..6226d14d8b1e563772e35fde99ec7acb5e5b450e 100644 --- ql/src/test/results/clientpositive/llap/cbo_rp_unionDistinct_2.q.out +++ ql/src/test/results/clientpositive/llap/cbo_rp_unionDistinct_2.q.out @@ -350,20 +350,6 @@ key string value string #### A masked pattern was here #### -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select `u2`.`key`, `u2`.`value` from `default`.`u2` -union all -select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 @@ -426,16 +412,6 @@ key string value string #### A masked pattern was here #### -( -select distinct * from u1 -union -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 @@ -493,16 +469,6 @@ key string value string #### A masked pattern was here #### -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union all -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 diff --git ql/src/test/results/clientpositive/llap/subquery_views.q.out ql/src/test/results/clientpositive/llap/subquery_views.q.out index 647412f52e7bb1d80d18de5f1295f16f705b641d..3ba7d203321cd541e17e57e86b8d580681ef267d 100644 --- ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -32,16 +32,6 @@ key string value string #### A masked pattern was here #### -from src b -where exists - (select a.key - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_9'), viewExpandedText:select `b`.`key`, `b`.`value` -from `default`.`src` `b` -where exists - (select `a`.`key` - from `default`.`src` `a` - where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_9'), tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from cv1 where cv1.key in (select key from cv1 c where c.key > '95') PREHOOK: type: QUERY @@ -95,18 +85,6 @@ key string value string #### A masked pattern was here #### -from src b -where b.key not in - (select a.key - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_11' - ), viewExpandedText:select `b`.`key`, `b`.`value` -from `default`.`src` `b` -where `b`.`key` not in - (select `a`.`key` - from `default`.`src` `a` - where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_11' - ), tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: explain select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11') @@ -451,14 +429,6 @@ value string _c2 bigint #### A masked pattern was here #### -from src b -where b.key in (select key from src where src.key > '8') -group by key, value -having count(*) in (select count(*) from src s1 where s1.key > '9' group by s1.key ), viewExpandedText:select `b`.`key`, `b`.`value`, count(*) -from `default`.`src` `b` -where `b`.`key` in (select `src`.`key` from `default`.`src` where `src`.`key` > '8') -group by `b`.`key`, `b`.`value` -having count(*) in (select count(*) from `default`.`src` `s1` where `s1`.`key` > '9' group by `s1`.`key` ), tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from cv3 PREHOOK: type: QUERY PREHOOK: Input: default@cv3 diff --git ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out index a2d827ab138fbed8ec3b59ff52906a0728d83eea..789c5601a977bc7bb8a2f427263b09c41595b7d0 100644 --- ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out +++ ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out @@ -242,20 +242,6 @@ key string value string #### A masked pattern was here #### -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select `u2`.`key`, `u2`.`value` from `default`.`u2` -union all -select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 @@ -318,16 +304,6 @@ key string value string #### A masked pattern was here #### -( -select distinct * from u1 -union -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 @@ -385,16 +361,6 @@ key string value string #### A masked pattern was here #### -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union all -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from v PREHOOK: type: QUERY PREHOOK: Input: default@u1 diff --git ql/src/test/results/clientpositive/spark/subquery_views.q.out ql/src/test/results/clientpositive/spark/subquery_views.q.out index 8d7df62eb4db0a57a7e9d3bbb812dd16ee186b69..815d6bb01aae9388eeb530d63639dee62fb5355d 100644 --- ql/src/test/results/clientpositive/spark/subquery_views.q.out +++ ql/src/test/results/clientpositive/spark/subquery_views.q.out @@ -32,16 +32,6 @@ key string value string #### A masked pattern was here #### -from src b -where exists - (select a.key - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_9'), viewExpandedText:select `b`.`key`, `b`.`value` -from `default`.`src` `b` -where exists - (select `a`.`key` - from `default`.`src` `a` - where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_9'), tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from cv1 where cv1.key in (select key from cv1 c where c.key > '95') PREHOOK: type: QUERY @@ -95,18 +85,6 @@ key string value string #### A masked pattern was here #### -from src b -where b.key not in - (select a.key - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_11' - ), viewExpandedText:select `b`.`key`, `b`.`value` -from `default`.`src` `b` -where `b`.`key` not in - (select `a`.`key` - from `default`.`src` `a` - where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_11' - ), tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: explain select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11') @@ -486,14 +464,6 @@ value string _c2 bigint #### A masked pattern was here #### -from src b -where b.key in (select key from src where src.key > '8') -group by key, value -having count(*) in (select count(*) from src s1 where s1.key > '9' group by s1.key ), viewExpandedText:select `b`.`key`, `b`.`value`, count(*) -from `default`.`src` `b` -where `b`.`key` in (select `src`.`key` from `default`.`src` where `src`.`key` > '8') -group by `b`.`key`, `b`.`value` -having count(*) in (select count(*) from `default`.`src` `s1` where `s1`.`key` > '9' group by `s1`.`key` ), tableType:VIRTUAL_VIEW, rewriteEnabled:false, creationMetadata:{}) PREHOOK: query: select * from cv3 PREHOOK: type: QUERY PREHOOK: Input: default@cv3