Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 1182505) +++ conf/hive-default.xml (working copy) @@ -281,6 +281,15 @@ + hive.metastore.location.hosts.whitelist + + A comma separated list of host:port pairs. If a non-external, non-native table or partition + is created or altered with a location without one of these host:port pairs, the transaction + will fail. If the location is not a URI, or this list is empty, the transaction + will succeed. + + + hive.default.fileformat TextFile Default file format for CREATE TABLE statement. Options are TextFile and SequenceFile. Users can explicitly say CREATE TABLE ... STORED AS <TEXTFILE|SEQUENCEFILE> to override Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1182505) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; +import java.util.Set; import java.util.Timer; import java.util.regex.Pattern; @@ -229,6 +230,7 @@ private ClassLoader classLoader; private AlterHandler alterHandler; private List listeners; + private Set locationHostsWhitelist; { classLoader = Thread.currentThread().getContextClassLoader(); @@ -245,6 +247,7 @@ HiveAlterHandler.class.getName()); alterHandler = (AlterHandler) ReflectionUtils.newInstance(getClass( alterHandlerName, AlterHandler.class), hiveConf); + alterHandler.setConf(hiveConf); wh = new Warehouse(hiveConf); retryInterval = HiveConf.getIntVar(hiveConf, @@ -276,6 +279,8 @@ Timer cleaner = new Timer("Metastore Events Cleaner Thread", true); cleaner.schedule(new EventCleanerTask(this), cleanFreq, cleanFreq); } + + locationHostsWhitelist = MetaStoreUtils.getLocationHostsWhitelist(hiveConf); return true; } @@ -988,6 +993,11 @@ + " specified for non-external table:" + tbl.getTableName()); } tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation())); + if (!MetaStoreUtils.isValidLocation(tblPath, tbl, locationHostsWhitelist)) { + throw new MetaException("Unable to create table because the host in its " + + "location " + tbl.getSd().getLocation() + " is not in the list of approved hosts " + + HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST); + } } tbl.getSd().setLocation(tblPath.toString()); } @@ -1494,7 +1504,13 @@ throw new MetaException( "Cannot specify location for a view partition"); } + partLocation = wh.getDnsPath(new Path(partLocationStr)); + if (!MetaStoreUtils.isValidLocation(partLocation, tbl, locationHostsWhitelist)) { + throw new MetaException("Unable to add partition because the host in its location " + + partLocationStr + " is not in the list of approved hosts " + + HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST); + } } if (partLocation != null) { @@ -1824,7 +1840,25 @@ new_part.putToParameters(Constants.DDL_TIME, Long.toString(System .currentTimeMillis() / 1000)); } + Partition oldPart = ms.getPartition(db_name, tbl_name, new_part.getValues()); + // If the new partition has a new location verify it + if (new_part.getSd() != null && new_part.getSd().getLocation() != null && + !new_part.getSd().getLocation().isEmpty() && (oldPart.getSd() == null || + oldPart.getSd().getLocation() == null || oldPart.getSd().getLocation().isEmpty() || + !new_part.getSd().getLocation().equals(oldPart.getSd().getLocation()))) { + + Table tbl = ms.getTable(new_part.getDbName(), new_part.getTableName()); + String partLocationStr = new_part.getSd().getLocation(); + Path partLocationPath = wh.getDnsPath(new Path(partLocationStr)); + + if (tbl != null && !MetaStoreUtils.isValidLocation(partLocationPath, tbl, locationHostsWhitelist)) { + throw new MetaException("Unable to alter partition because the host in its new " + + "location " + partLocationStr + " is not in the list of approved hosts " + + HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST); + } + } + ms.alterPartition(db_name, tbl_name, new_part); for (MetaStoreEventListener listener : listeners) { listener.onAlterPartition(new AlterPartitionEvent(oldPart, new_part, true, this)); Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (revision 1182505) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (working copy) @@ -21,6 +21,7 @@ import java.net.URI; import java.util.Iterator; import java.util.List; +import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -28,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; @@ -44,6 +46,7 @@ private Configuration hiveConf; private static final Log LOG = LogFactory.getLog(HiveAlterHandler.class .getName()); + private Set locationHostsWhitelist; public Configuration getConf() { return hiveConf; @@ -52,6 +55,7 @@ @SuppressWarnings("nls") public void setConf(Configuration conf) { hiveConf = conf; + locationHostsWhitelist = MetaStoreUtils.getLocationHostsWhitelist(hiveConf); } public void alterTable(RawStore msdb, Warehouse wh, String dbname, @@ -112,6 +116,16 @@ } } + if (!newt.getSd().getLocation().isEmpty() && oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) != 0) { + // If the table is getting a new location, check if that location is valid + destPath = wh.getDnsPath(new Path(newt.getSd().getLocation())); + if (locationHostsWhitelist != null && !MetaStoreUtils.isValidLocation(destPath, newt, locationHostsWhitelist)) { + throw new MetaException("Unable to alter table because the host in its new " + + "location " + newTblLoc + " is not in the list of approved hosts " + + HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST); + } + } + // if this alter is a rename, the table is not a virtual view, the user // didn't change the default location (or new location is empty), and // table is not an external table, that means useris asking metastore to Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (revision 1182505) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (working copy) @@ -22,13 +22,16 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; +import java.net.URI; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -987,4 +990,45 @@ return listeners; } + + /** + * return a set of Strings parsed from the METASTOR_LOCATION_HOSTS_WHITELIST configuration variable + * @param hiveConf + * @return + */ + public static Set getLocationHostsWhitelist(Configuration hiveConf) { + Set locationHostsWhitelist = new HashSet(); + String[] locationHostsWhitelistArray = + HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST).split(","); + for (String host : locationHostsWhitelistArray) { + locationHostsWhitelist.add(host.trim()); + } + return locationHostsWhitelist; + } + + /** + * check if a location is valid for a new/altered table/partition + * + * currently this just checks if the host:port in the path is in the whitelist + * @param locationPath + * @param tbl + * @param locationHostsWhitelist + * @return + */ + public static boolean isValidLocation(Path locationPath, Table tbl, Set locationHostsWhitelist) { + if (!locationHostsWhitelist.isEmpty() && + !MetaStoreUtils.isExternalTable(tbl) && + !MetaStoreUtils.isNonNativeTable(tbl)) { + URI locationPathUri = locationPath.toUri(); + String locationPathHost = locationPathUri.getHost(); + int locationPathPort = locationPathUri.getPort(); + if (locationPathHost != null && + !locationPathHost.isEmpty() && + locationPathPort != -1 && + !locationHostsWhitelist.contains(locationPathHost + ":" + locationPathPort)) { + return false; + } + } + return true; + } } Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1182505) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -86,6 +86,7 @@ HiveConf.ConfVars.METASTORE_IDENTIFIER_FACTORY, HiveConf.ConfVars.METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK, HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS, + HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST, }; /** @@ -225,6 +226,7 @@ METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS("hive.metastore.authorization.storage.checks", false), METASTORE_EVENT_CLEAN_FREQ("hive.metastore.event.clean.freq",0L), METASTORE_EVENT_EXPIRY_DURATION("hive.metastore.event.expiry.duration",0L), + METASTORE_LOCATION_HOSTS_WHITELIST("hive.metastore.location.hosts.whitelist", ""), // Default parameters for creating tables NEWTABLEDEFAULTPARA("hive.table.parameters.default",""),