diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index fc259fd..e0a7b24 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -713,7 +713,7 @@ HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", ""), HIVE_DDL_OUTPUT_FORMAT("hive.ddl.output.format", null), HIVE_ENTITY_SEPARATOR("hive.entity.separator", "@"), - HIVE_ENITITY_CAPTURE_INPUT_URI("hive.entity.capture.input.URI", false), + HIVE_EXTENDED_ENITITY_CAPTURE("hive.entity.capture.input.URI", false), HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS("hive.server2.thrift.min.worker.threads", 5), HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS("hive.server2.thrift.max.worker.threads", 100), diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index a0ac584..12a91fb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1958,6 +1958,19 @@ private void analyzeDescribeTable(ASTNode ast) throws SemanticException { descTblDesc.setExt(descOptions == HiveParser.KW_EXTENDED); descTblDesc.setPretty(descOptions == HiveParser.KW_PRETTY); } + + // store the read entity for the table being described + if (conf.getBoolVar(ConfVars.HIVE_EXTENDED_ENITITY_CAPTURE)) { + Table table; + // table name could be db.tab format + if (tableName.contains(".")) { + String[] tokens = tableName.split("\\."); + table = new Table (tokens[0], tokens[1]); + } else { + table = new Table (db.getCurrentDatabase(), tableName); + } + inputs.add(new ReadEntity(table)); + } rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), descTblDesc), conf)); setFetchTask(createFetchTask(DescTableDesc.getSchema())); @@ -3292,7 +3305,7 @@ private String toMessage(ErrorMsg message, Object detail) { } private void saveInputLocationEntity(String location) { - if (conf.getBoolVar(ConfVars.HIVE_ENITITY_CAPTURE_INPUT_URI) && + if (conf.getBoolVar(ConfVars.HIVE_EXTENDED_ENITITY_CAPTURE) && (location != null)) { inputs.add(new ReadEntity(location)); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 461844b..8948d22 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -267,7 +267,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { } } rootTasks.add(t); - if (conf.getBoolVar(ConfVars.HIVE_ENITITY_CAPTURE_INPUT_URI)) { + if (conf.getBoolVar(ConfVars.HIVE_EXTENDED_ENITITY_CAPTURE)) { inputs.add(new ReadEntity(fromURI.toString(), "hdfs".equals(fromURI.getScheme()))); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index a3e2ac9..593ab2d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -193,7 +193,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e .getMessage()), e); } - if (conf.getBoolVar(ConfVars.HIVE_ENITITY_CAPTURE_INPUT_URI)) { + if (conf.getBoolVar(ConfVars.HIVE_EXTENDED_ENITITY_CAPTURE)) { inputs.add(new ReadEntity(fromURI.toString(), isLocal)); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 43df410..e153e0b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -8735,6 +8735,11 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // skip the rest of this method. ctx.setResDir(null); ctx.setResFile(null); + if (conf.getBoolVar(ConfVars.HIVE_EXTENDED_ENITITY_CAPTURE)) { + for (Table tab : topToTable.values()) { + getInputs().add(new ReadEntity(tab)); + } + } return; } @@ -9373,7 +9378,7 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) } } - if (conf.getBoolVar(ConfVars.HIVE_ENITITY_CAPTURE_INPUT_URI) && + if (conf.getBoolVar(ConfVars.HIVE_EXTENDED_ENITITY_CAPTURE) && (location != null)) { inputs.add(new ReadEntity(location)); } diff --git ql/src/test/queries/clientpositive/describe_table_extended_entity.q ql/src/test/queries/clientpositive/describe_table_extended_entity.q new file mode 100644 index 0000000..b3c8475 --- /dev/null +++ ql/src/test/queries/clientpositive/describe_table_extended_entity.q @@ -0,0 +1,20 @@ +set hive.entity.capture.input.URI=true; +describe srcpart; +describe srcpart.key; +describe srcpart PARTITION(ds='2008-04-08', hr='12'); + +describe extended srcpart; +describe extended srcpart.key; +describe extended srcpart PARTITION(ds='2008-04-08', hr='12'); + +describe formatted srcpart; +describe formatted srcpart.key; +describe formatted srcpart PARTITION(ds='2008-04-08', hr='12'); + +create table srcpart_serdeprops like srcpart; +alter table srcpart_serdeprops set serdeproperties('xyz'='0'); +alter table srcpart_serdeprops set serdeproperties('pqrs'='1'); +alter table srcpart_serdeprops set serdeproperties('abcd'='2'); +alter table srcpart_serdeprops set serdeproperties('A1234'='3'); +describe formatted srcpart_serdeprops; +drop table srcpart_serdeprops; diff --git ql/src/test/queries/clientpositive/view_extended_entity.q ql/src/test/queries/clientpositive/view_extended_entity.q new file mode 100644 index 0000000..57fe2b4 --- /dev/null +++ ql/src/test/queries/clientpositive/view_extended_entity.q @@ -0,0 +1,36 @@ +DROP DATABASE IF EXISTS db1; +CREATE DATABASE db1; +USE db1; + +set hive.entity.capture.input.URI=true; + +CREATE TABLE table1 (key STRING, value STRING) +STORED AS TEXTFILE; + +CREATE TABLE table2 (key STRING, value STRING) +STORED AS TEXTFILE; + +-- relative reference, no alias +CREATE VIEW v1 AS SELECT * FROM table1; + +-- relative reference, aliased +CREATE VIEW v2 AS SELECT t1.* FROM table1 t1; + +-- relative reference, multiple tables +CREATE VIEW v3 AS SELECT t1.*, t2.key k FROM table1 t1 JOIN table2 t2 ON t1.key = t2.key; + +-- absolute reference, no alias +CREATE VIEW v4 AS SELECT * FROM db1.table1; + +-- absolute reference, aliased +CREATE VIEW v5 AS SELECT t1.* FROM db1.table1 t1; + +-- absolute reference, multiple tables +CREATE VIEW v6 AS SELECT t1.*, t2.key k FROM db1.table1 t1 JOIN db1.table2 t2 ON t1.key = t2.key; + +-- relative reference, explicit column +CREATE VIEW v7 AS SELECT key from table1; + +-- absolute reference, explicit column +CREATE VIEW v8 AS SELECT key from db1.table1; + diff --git ql/src/test/results/clientpositive/describe_table_extended_entity.q.out ql/src/test/results/clientpositive/describe_table_extended_entity.q.out new file mode 100644 index 0000000..3ba4ec0 --- /dev/null +++ ql/src/test/results/clientpositive/describe_table_extended_entity.q.out @@ -0,0 +1,260 @@ +PREHOOK: query: describe srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +key string default +value string default +ds string None +hr string None + +# Partition Information +# col_name data_type comment + +ds string None +hr string None +PREHOOK: query: describe srcpart.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe srcpart.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +key string from deserializer +PREHOOK: query: describe srcpart PARTITION(ds='2008-04-08', hr='12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe srcpart PARTITION(ds='2008-04-08', hr='12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +key string default +value string default +ds string None +hr string None + +# Partition Information +# col_name data_type comment + +ds string None +hr string None +PREHOOK: query: describe extended srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe extended srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +key string default +value string default +ds string None +hr string None + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +#### A masked pattern was here #### +PREHOOK: query: describe extended srcpart.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe extended srcpart.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +key string from deserializer +PREHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08', hr='12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe extended srcpart PARTITION(ds='2008-04-08', hr='12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +key string default +value string default +ds string None +hr string None + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +#### A masked pattern was here #### +PREHOOK: query: describe formatted srcpart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe formatted srcpart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 4 + numPartitions 4 + numRows 0 + rawDataSize 0 + totalSize 23248 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted srcpart.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe formatted srcpart.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +# col_name data_type comment + +key string from deserializer +PREHOOK: query: describe formatted srcpart PARTITION(ds='2008-04-08', hr='12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart +POSTHOOK: query: describe formatted srcpart PARTITION(ds='2008-04-08', hr='12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: srcpart +#### A masked pattern was here #### +Protect Mode: None +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table srcpart_serdeprops like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table srcpart_serdeprops like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_serdeprops +PREHOOK: query: alter table srcpart_serdeprops set serdeproperties('xyz'='0') +PREHOOK: type: ALTERTABLE_SERDEPROPERTIES +PREHOOK: Input: default@srcpart_serdeprops +PREHOOK: Output: default@srcpart_serdeprops +POSTHOOK: query: alter table srcpart_serdeprops set serdeproperties('xyz'='0') +POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES +POSTHOOK: Input: default@srcpart_serdeprops +POSTHOOK: Output: default@srcpart_serdeprops +PREHOOK: query: alter table srcpart_serdeprops set serdeproperties('pqrs'='1') +PREHOOK: type: ALTERTABLE_SERDEPROPERTIES +PREHOOK: Input: default@srcpart_serdeprops +PREHOOK: Output: default@srcpart_serdeprops +POSTHOOK: query: alter table srcpart_serdeprops set serdeproperties('pqrs'='1') +POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES +POSTHOOK: Input: default@srcpart_serdeprops +POSTHOOK: Output: default@srcpart_serdeprops +PREHOOK: query: alter table srcpart_serdeprops set serdeproperties('abcd'='2') +PREHOOK: type: ALTERTABLE_SERDEPROPERTIES +PREHOOK: Input: default@srcpart_serdeprops +PREHOOK: Output: default@srcpart_serdeprops +POSTHOOK: query: alter table srcpart_serdeprops set serdeproperties('abcd'='2') +POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES +POSTHOOK: Input: default@srcpart_serdeprops +POSTHOOK: Output: default@srcpart_serdeprops +PREHOOK: query: alter table srcpart_serdeprops set serdeproperties('A1234'='3') +PREHOOK: type: ALTERTABLE_SERDEPROPERTIES +PREHOOK: Input: default@srcpart_serdeprops +PREHOOK: Output: default@srcpart_serdeprops +POSTHOOK: query: alter table srcpart_serdeprops set serdeproperties('A1234'='3') +POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES +POSTHOOK: Input: default@srcpart_serdeprops +POSTHOOK: Output: default@srcpart_serdeprops +PREHOOK: query: describe formatted srcpart_serdeprops +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@srcpart_serdeprops +POSTHOOK: query: describe formatted srcpart_serdeprops +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@srcpart_serdeprops +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string None +hr string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + A1234 3 + abcd 2 + pqrs 1 + serialization.format 1 + xyz 0 +PREHOOK: query: drop table srcpart_serdeprops +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcpart_serdeprops +PREHOOK: Output: default@srcpart_serdeprops +POSTHOOK: query: drop table srcpart_serdeprops +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcpart_serdeprops +POSTHOOK: Output: default@srcpart_serdeprops diff --git ql/src/test/results/clientpositive/view_extended_entity.q.out ql/src/test/results/clientpositive/view_extended_entity.q.out new file mode 100644 index 0000000..01fc7a3 --- /dev/null +++ ql/src/test/results/clientpositive/view_extended_entity.q.out @@ -0,0 +1,102 @@ +PREHOOK: query: DROP DATABASE IF EXISTS db1 +PREHOOK: type: DROPDATABASE +POSTHOOK: query: DROP DATABASE IF EXISTS db1 +POSTHOOK: type: DROPDATABASE +PREHOOK: query: CREATE DATABASE db1 +PREHOOK: type: CREATEDATABASE +POSTHOOK: query: CREATE DATABASE db1 +POSTHOOK: type: CREATEDATABASE +PREHOOK: query: USE db1 +PREHOOK: type: SWITCHDATABASE +POSTHOOK: query: USE db1 +POSTHOOK: type: SWITCHDATABASE +PREHOOK: query: CREATE TABLE table1 (key STRING, value STRING) +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE table1 (key STRING, value STRING) +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: db1@table1 +PREHOOK: query: CREATE TABLE table2 (key STRING, value STRING) +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE table2 (key STRING, value STRING) +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: db1@table2 +PREHOOK: query: -- relative reference, no alias +CREATE VIEW v1 AS SELECT * FROM table1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: db1@table1 +POSTHOOK: query: -- relative reference, no alias +CREATE VIEW v1 AS SELECT * FROM table1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: db1@table1 +POSTHOOK: Output: db1@v1 +PREHOOK: query: -- relative reference, aliased +CREATE VIEW v2 AS SELECT t1.* FROM table1 t1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: db1@table1 +POSTHOOK: query: -- relative reference, aliased +CREATE VIEW v2 AS SELECT t1.* FROM table1 t1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: db1@table1 +POSTHOOK: Output: db1@v2 +PREHOOK: query: -- relative reference, multiple tables +CREATE VIEW v3 AS SELECT t1.*, t2.key k FROM table1 t1 JOIN table2 t2 ON t1.key = t2.key +PREHOOK: type: CREATEVIEW +PREHOOK: Input: db1@table1 +PREHOOK: Input: db1@table2 +POSTHOOK: query: -- relative reference, multiple tables +CREATE VIEW v3 AS SELECT t1.*, t2.key k FROM table1 t1 JOIN table2 t2 ON t1.key = t2.key +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: db1@table1 +POSTHOOK: Input: db1@table2 +POSTHOOK: Output: db1@v3 +PREHOOK: query: -- absolute reference, no alias +CREATE VIEW v4 AS SELECT * FROM db1.table1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: db1@table1 +POSTHOOK: query: -- absolute reference, no alias +CREATE VIEW v4 AS SELECT * FROM db1.table1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: db1@table1 +POSTHOOK: Output: db1@v4 +PREHOOK: query: -- absolute reference, aliased +CREATE VIEW v5 AS SELECT t1.* FROM db1.table1 t1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: db1@table1 +POSTHOOK: query: -- absolute reference, aliased +CREATE VIEW v5 AS SELECT t1.* FROM db1.table1 t1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: db1@table1 +POSTHOOK: Output: db1@v5 +PREHOOK: query: -- absolute reference, multiple tables +CREATE VIEW v6 AS SELECT t1.*, t2.key k FROM db1.table1 t1 JOIN db1.table2 t2 ON t1.key = t2.key +PREHOOK: type: CREATEVIEW +PREHOOK: Input: db1@table1 +PREHOOK: Input: db1@table2 +POSTHOOK: query: -- absolute reference, multiple tables +CREATE VIEW v6 AS SELECT t1.*, t2.key k FROM db1.table1 t1 JOIN db1.table2 t2 ON t1.key = t2.key +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: db1@table1 +POSTHOOK: Input: db1@table2 +POSTHOOK: Output: db1@v6 +PREHOOK: query: -- relative reference, explicit column +CREATE VIEW v7 AS SELECT key from table1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: db1@table1 +POSTHOOK: query: -- relative reference, explicit column +CREATE VIEW v7 AS SELECT key from table1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: db1@table1 +POSTHOOK: Output: db1@v7 +PREHOOK: query: -- absolute reference, explicit column +CREATE VIEW v8 AS SELECT key from db1.table1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: db1@table1 +POSTHOOK: query: -- absolute reference, explicit column +CREATE VIEW v8 AS SELECT key from db1.table1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: db1@table1 +POSTHOOK: Output: db1@v8