diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index ee954d9..996aa17 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -67,4 +67,5 @@ public static final String HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG = "hadoop.security.credential.provider.path"; public static final String MATERIALIZED_VIEW_REWRITING_TIME_WINDOW = "rewriting.time.window"; + public static final String TABLE_PROPERTY_MASKS = "********"; } diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0dea099..5081a7e 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4573,6 +4573,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal + ",dfs.adls.oauth2.credential" + ",fs.adl.oauth2.credential", "Comma separated list of configuration options which should not be read by normal user like passwords"), + HIVE_HIDDEN_PROPERTIES("hive.hidden.properties", Constants.JDBC_PASSWORD, + "Comma separated list of table properties which should not be read by normal user like passwords"), HIVE_CONF_INTERNAL_VARIABLE_LIST("hive.conf.internal.variable.list", "hive.added.files.path,hive.added.jars.path,hive.added.archives.path", "Comma separated list of variables which are used internally and should not be configurable."), diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index ebe6423..193290b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -83,8 +83,21 @@ public static StringBuilder dumpConfig(HiveConf conf) { * @return The list of the configuration values to hide */ public static Set getHiddenSet(Configuration configuration) { - Set hiddenSet = new HashSet(); - String hiddenListStr = HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST); + return getSetFromConf(configuration, HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST); + } + + /** + * Getting the set of the hidden table properties + * @param configuration The original configuration + * @return The list of the table properties to hide + */ + public static Set getHiddenProperties(Configuration configuration) { + return getSetFromConf(configuration, HiveConf.ConfVars.HIVE_HIDDEN_PROPERTIES); + } + + private static Set getSetFromConf(Configuration configuration, HiveConf.ConfVars var) { + Set hiddenSet = new HashSet<>(); + String hiddenListStr = HiveConf.getVar(configuration, var); if (hiddenListStr != null) { for (String entry : hiddenListStr.split(",")) { hiddenSet.add(entry.trim()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 76339f1..ecbd20c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -153,6 +153,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.NotNullConstraint; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -2346,8 +2347,12 @@ else if (sortCol.getOrder() == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_DESC) { // Table properties duplicateProps.addAll(StatsSetupConst.TABLE_PARAMS_STATS_KEYS); + Set hiddenProperties = HiveUtils.getHiddenPropertiesFromSession(); + Map origMap = HiveUtils.replaceWithMask(tbl.getTTable().getParameters(), hiddenProperties); String tbl_properties = propertiesToString(tbl.getParameters(), duplicateProps); - + if (!origMap.isEmpty()) { + tbl.getTTable().getParameters().putAll(origMap); + } createTab_stmt.add(TEMPORARY, tbl_temp); createTab_stmt.add(EXTERNAL, tbl_external); createTab_stmt.add(LIST_COLUMNS, tbl_columns); @@ -3120,6 +3125,7 @@ private int showTableProperties(Hive db, ShowTblPropertiesDesc showTblPrpt) thro // show table properties - populate the output stream Table tbl = db.getTable(tableName, false); + Set hiddenProperties = HiveUtils.getHiddenPropertiesFromSession(); try { if (tbl == null) { String errMsg = "Table " + tableName + " does not exist"; @@ -3138,6 +3144,9 @@ private int showTableProperties(Hive db, ShowTblPropertiesDesc showTblPrpt) thro builder.append(errMsg); } else { + if (hiddenProperties.contains(propertyName)) { + propertyValue = Constants.TABLE_PROPERTY_MASKS; + } appendNonNull(builder, propertyName, true); appendNonNull(builder, propertyValue); } @@ -3146,7 +3155,11 @@ private int showTableProperties(Hive db, ShowTblPropertiesDesc showTblPrpt) thro Map properties = new TreeMap(tbl.getParameters()); for (Entry entry : properties.entrySet()) { appendNonNull(builder, entry.getKey(), true); - appendNonNull(builder, entry.getValue()); + if (hiddenProperties.contains(entry.getKey())) { + appendNonNull(builder, Constants.TABLE_PROPERTY_MASKS); + } else { + appendNonNull(builder, entry.getValue()); + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java index e04a0f3..2db71c4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java @@ -19,9 +19,16 @@ package org.apache.hadoop.hive.ql.metadata; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.conf.HiveConfUtil; +import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -447,4 +454,24 @@ public static Path getDumpPath(Path root, String dbName, String tableName) { } return new Path(root, dbName); } + + public static Set getHiddenPropertiesFromSession() { + Set hiddenProperties = new HashSet<>(); + SessionState ss = SessionState.get(); + if (ss != null && ss.getConf() != null) { + hiddenProperties = HiveConfUtil.getHiddenProperties(ss.getConf()); + } + return hiddenProperties; + } + + public static Map replaceWithMask(Map properties, Set hiddenProperties) { + Map origMap = new HashMap<>(); + for (String key : properties.keySet()) { + if (hiddenProperties.contains(key)) { + origMap.put(key, properties.get(key)); + properties.put(key, Constants.TABLE_PROPERTY_MASKS); + } + } + return origMap; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java index 4180dc4..ed82f8e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java @@ -21,6 +21,7 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.TableType; @@ -46,6 +47,7 @@ import org.apache.hadoop.hive.ql.metadata.DefaultConstraint; import org.apache.hadoop.hive.ql.metadata.ForeignKeyInfo; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.NotNullConstraint; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.PrimaryKeyInfo; @@ -478,8 +480,12 @@ private static void displayAllParameters(Map params, StringBuild boolean escapeUnicode, boolean isOutputPadded) { List keys = new ArrayList(params.keySet()); Collections.sort(keys); + Set hiddenProperties = HiveUtils.getHiddenPropertiesFromSession(); for (String key : keys) { String value = params.get(key); + if (hiddenProperties.contains(key)) { + value = Constants.TABLE_PROPERTY_MASKS; + } if (key.equals(StatsSetupConst.NUM_ERASURE_CODED_FILES)) { if ("0".equals(value)) { continue; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java index fbeb9c8..26c6788 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java @@ -29,10 +29,9 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.StorageHandlerInfo; -import org.apache.hadoop.hive.ql.plan.DescTableDesc; import org.apache.hive.common.util.HiveStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -295,7 +294,12 @@ public void describeTable(DataOutputStream outStream, String colPath, // show table information outStream.write(("Detailed Table Information").getBytes("UTF-8")); outStream.write(separator); + Set hiddenProperties = HiveUtils.getHiddenPropertiesFromSession(); + Map origMap = HiveUtils.replaceWithMask(tbl.getTTable().getParameters(), hiddenProperties); String tableDesc = HiveStringUtils.escapeJava(tbl.getTTable().toString()); + if (!origMap.isEmpty()) { + tbl.getTTable().getParameters().putAll(origMap); + } outStream.write(tableDesc.getBytes("UTF-8")); outStream.write(separator); outStream.write(terminator); diff --git a/ql/src/test/queries/clientpositive/external_jdbc_table.q b/ql/src/test/queries/clientpositive/external_jdbc_table.q index 36ed93a..4988f6a 100644 --- a/ql/src/test/queries/clientpositive/external_jdbc_table.q +++ b/ql/src/test/queries/clientpositive/external_jdbc_table.q @@ -141,8 +141,14 @@ UNION SELECT bkey FROM ext_simple_derby_table2; +--Test password masking +describe extended ext_simple_derby_table1; +describe formatted ext_simple_derby_table1; +show tblproperties ext_simple_derby_table1; + +show tblproperties ext_simple_derby_table1("hive.sql.dbcp.password"); diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table.q.out index a3b0ac4..0c442aa 100644 --- a/ql/src/test/results/clientpositive/llap/external_jdbc_table.q.out +++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table.q.out @@ -552,3 +552,89 @@ POSTHOOK: Input: default@simple_hive_table1 -16 20 50 +PREHOOK: query: describe extended ext_simple_derby_table1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ext_simple_derby_table1 +POSTHOOK: query: describe extended ext_simple_derby_table1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ext_simple_derby_table1 +ikey int from deserializer +bkey bigint from deserializer +fkey float from deserializer +dkey double from deserializer + +#### A masked pattern was here #### +PREHOOK: query: describe formatted ext_simple_derby_table1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ext_simple_derby_table1 +POSTHOOK: query: describe formatted ext_simple_derby_table1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ext_simple_derby_table1 +# col_name data_type comment +ikey int from deserializer +bkey bigint from deserializer +fkey float from deserializer +dkey double from deserializer + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bkey\":\"true\",\"dkey\":\"true\",\"fkey\":\"true\",\"ikey\":\"true\"}} + EXTERNAL TRUE + bucketing_version 2 + discover.partitions true + hive.sql.database.type DERBY + hive.sql.dbcp.maxActive 1 + hive.sql.dbcp.password ******** + hive.sql.dbcp.username APP + hive.sql.jdbc.driver org.apache.derby.jdbc.EmbeddedDriver +#### A masked pattern was here #### + hive.sql.table SIMPLE_DERBY_TABLE1 + numFiles 0 + numRows 0 + rawDataSize 0 + storage_handler org.apache.hive.storage.jdbc.JdbcStorageHandler + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hive.storage.jdbc.JdbcSerDe +InputFormat: null +OutputFormat: null +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: show tblproperties ext_simple_derby_table1 +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties ext_simple_derby_table1 +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"bkey":"true","dkey":"true","fkey":"true","ikey":"true"}} +EXTERNAL TRUE +bucketing_version 2 +discover.partitions true +hive.sql.database.type DERBY +hive.sql.dbcp.maxActive 1 +hive.sql.dbcp.password ******** +hive.sql.dbcp.username APP +hive.sql.jdbc.driver org.apache.derby.jdbc.EmbeddedDriver +#### A masked pattern was here #### +hive.sql.table SIMPLE_DERBY_TABLE1 +numFiles 0 +numFilesErasureCoded 0 +numRows 0 +rawDataSize 0 +storage_handler org.apache.hive.storage.jdbc.JdbcStorageHandler +totalSize 0 +#### A masked pattern was here #### +PREHOOK: query: show tblproperties ext_simple_derby_table1("hive.sql.dbcp.password") +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties ext_simple_derby_table1("hive.sql.dbcp.password") +POSTHOOK: type: SHOW_TBLPROPERTIES +hive.sql.dbcp.password ********