Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1182505)
+++ conf/hive-default.xml (working copy)
@@ -281,6 +281,15 @@
+ hive.metastore.location.hosts.whitelist
+
+ A comma separated list of host:port pairs. If a non-external, non-native table or partition
+ is created or altered with a location without one of these host:port pairs, the transaction
+ will fail. If the location is not a URI, or this list is empty, the transaction
+ will succeed.
+
+
+
hive.default.fileformat
TextFile
Default file format for CREATE TABLE statement. Options are TextFile and SequenceFile. Users can explicitly say CREATE TABLE ... STORED AS <TEXTFILE|SEQUENCEFILE> to override
Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1182505)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy)
@@ -33,6 +33,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Set;
import java.util.Timer;
import java.util.regex.Pattern;
@@ -229,6 +230,7 @@
private ClassLoader classLoader;
private AlterHandler alterHandler;
private List listeners;
+ private Set locationHostsWhitelist;
{
classLoader = Thread.currentThread().getContextClassLoader();
@@ -245,6 +247,7 @@
HiveAlterHandler.class.getName());
alterHandler = (AlterHandler) ReflectionUtils.newInstance(getClass(
alterHandlerName, AlterHandler.class), hiveConf);
+ alterHandler.setConf(hiveConf);
wh = new Warehouse(hiveConf);
retryInterval = HiveConf.getIntVar(hiveConf,
@@ -276,6 +279,8 @@
Timer cleaner = new Timer("Metastore Events Cleaner Thread", true);
cleaner.schedule(new EventCleanerTask(this), cleanFreq, cleanFreq);
}
+
+ locationHostsWhitelist = MetaStoreUtils.getLocationHostsWhitelist(hiveConf);
return true;
}
@@ -988,6 +993,11 @@
+ " specified for non-external table:" + tbl.getTableName());
}
tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation()));
+ if (!MetaStoreUtils.isValidLocation(tblPath, tbl, locationHostsWhitelist)) {
+ throw new MetaException("Unable to create table because the host in its " +
+ "location " + tbl.getSd().getLocation() + " is not in the list of approved hosts " +
+ HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST);
+ }
}
tbl.getSd().setLocation(tblPath.toString());
}
@@ -1494,7 +1504,13 @@
throw new MetaException(
"Cannot specify location for a view partition");
}
+
partLocation = wh.getDnsPath(new Path(partLocationStr));
+ if (!MetaStoreUtils.isValidLocation(partLocation, tbl, locationHostsWhitelist)) {
+ throw new MetaException("Unable to add partition because the host in its location " +
+ partLocationStr + " is not in the list of approved hosts " +
+ HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST);
+ }
}
if (partLocation != null) {
@@ -1824,7 +1840,25 @@
new_part.putToParameters(Constants.DDL_TIME, Long.toString(System
.currentTimeMillis() / 1000));
}
+
Partition oldPart = ms.getPartition(db_name, tbl_name, new_part.getValues());
+ // If the new partition has a new location verify it
+ if (new_part.getSd() != null && new_part.getSd().getLocation() != null &&
+ !new_part.getSd().getLocation().isEmpty() && (oldPart.getSd() == null ||
+ oldPart.getSd().getLocation() == null || oldPart.getSd().getLocation().isEmpty() ||
+ !new_part.getSd().getLocation().equals(oldPart.getSd().getLocation()))) {
+
+ Table tbl = ms.getTable(new_part.getDbName(), new_part.getTableName());
+ String partLocationStr = new_part.getSd().getLocation();
+ Path partLocationPath = wh.getDnsPath(new Path(partLocationStr));
+
+ if (tbl != null && !MetaStoreUtils.isValidLocation(partLocationPath, tbl, locationHostsWhitelist)) {
+ throw new MetaException("Unable to alter partition because the host in its new " +
+ "location " + partLocationStr + " is not in the list of approved hosts " +
+ HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST);
+ }
+ }
+
ms.alterPartition(db_name, tbl_name, new_part);
for (MetaStoreEventListener listener : listeners) {
listener.onAlterPartition(new AlterPartitionEvent(oldPart, new_part, true, this));
Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (revision 1182505)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (working copy)
@@ -21,6 +21,7 @@
import java.net.URI;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -28,6 +29,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
@@ -44,6 +46,7 @@
private Configuration hiveConf;
private static final Log LOG = LogFactory.getLog(HiveAlterHandler.class
.getName());
+ private Set locationHostsWhitelist;
public Configuration getConf() {
return hiveConf;
@@ -52,6 +55,7 @@
@SuppressWarnings("nls")
public void setConf(Configuration conf) {
hiveConf = conf;
+ locationHostsWhitelist = MetaStoreUtils.getLocationHostsWhitelist(hiveConf);
}
public void alterTable(RawStore msdb, Warehouse wh, String dbname,
@@ -112,6 +116,16 @@
}
}
+ if (!newt.getSd().getLocation().isEmpty() && oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) != 0) {
+ // If the table is getting a new location, check if that location is valid
+ destPath = wh.getDnsPath(new Path(newt.getSd().getLocation()));
+ if (locationHostsWhitelist != null && !MetaStoreUtils.isValidLocation(destPath, newt, locationHostsWhitelist)) {
+ throw new MetaException("Unable to alter table because the host in its new " +
+ "location " + newTblLoc + " is not in the list of approved hosts " +
+ HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST);
+ }
+ }
+
// if this alter is a rename, the table is not a virtual view, the user
// didn't change the default location (or new location is empty), and
// table is not an external table, that means useris asking metastore to
Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (revision 1182505)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (working copy)
@@ -22,13 +22,16 @@
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -987,4 +990,45 @@
return listeners;
}
+
+ /**
+ * return a set of Strings parsed from the METASTOR_LOCATION_HOSTS_WHITELIST configuration variable
+ * @param hiveConf
+ * @return
+ */
+ public static Set getLocationHostsWhitelist(Configuration hiveConf) {
+ Set locationHostsWhitelist = new HashSet();
+ String[] locationHostsWhitelistArray =
+ HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST).split(",");
+ for (String host : locationHostsWhitelistArray) {
+ locationHostsWhitelist.add(host.trim());
+ }
+ return locationHostsWhitelist;
+ }
+
+ /**
+ * check if a location is valid for a new/altered table/partition
+ *
+ * currently this just checks if the host:port in the path is in the whitelist
+ * @param locationPath
+ * @param tbl
+ * @param locationHostsWhitelist
+ * @return
+ */
+ public static boolean isValidLocation(Path locationPath, Table tbl, Set locationHostsWhitelist) {
+ if (!locationHostsWhitelist.isEmpty() &&
+ !MetaStoreUtils.isExternalTable(tbl) &&
+ !MetaStoreUtils.isNonNativeTable(tbl)) {
+ URI locationPathUri = locationPath.toUri();
+ String locationPathHost = locationPathUri.getHost();
+ int locationPathPort = locationPathUri.getPort();
+ if (locationPathHost != null &&
+ !locationPathHost.isEmpty() &&
+ locationPathPort != -1 &&
+ !locationHostsWhitelist.contains(locationPathHost + ":" + locationPathPort)) {
+ return false;
+ }
+ }
+ return true;
+ }
}
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1182505)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -86,6 +86,7 @@
HiveConf.ConfVars.METASTORE_IDENTIFIER_FACTORY,
HiveConf.ConfVars.METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK,
HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS,
+ HiveConf.ConfVars.METASTORE_LOCATION_HOSTS_WHITELIST,
};
/**
@@ -225,6 +226,7 @@
METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS("hive.metastore.authorization.storage.checks", false),
METASTORE_EVENT_CLEAN_FREQ("hive.metastore.event.clean.freq",0L),
METASTORE_EVENT_EXPIRY_DURATION("hive.metastore.event.expiry.duration",0L),
+ METASTORE_LOCATION_HOSTS_WHITELIST("hive.metastore.location.hosts.whitelist", ""),
// Default parameters for creating tables
NEWTABLEDEFAULTPARA("hive.table.parameters.default",""),