diff --git hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
index 05f4ae0..0739e72 100644
--- hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
+++ hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
@@ -286,6 +286,46 @@ Using localhost in metastore uri does not work with kerberos security.
principal.
+
+ | webhcat.proxyuser.#USER#.hosts |
+ None |
+ List of client hosts from which the '#USER#' user is allowed to perform
+ 'doAs' operations.
+
+ The '#USER#' must be replaced with the username of the user who is
+ allowed to perform 'doAs' operations.
+
+ The value can be the '*' wildcard, which means every host is allowed,
+ or a comma-separated list of hostnames.
+
+ If value is a blank string or webhcat.proxyuser.#USER#.hosts is missing,
+ no hosts will be allowed.
+
+ For multiple users copy this property and replace the user name
+ in the property name. |
+
+
+ | webhcat.proxyuser.#USER#.groups |
+ None |
+ List of groups the '#USER#' user is allowed to impersonate users
+ from to perform 'doAs' operations.
+
+ The '#USER#' must be replaced with the username of the user who is
+ allowed to perform 'doAs' operations.
+
+ The value can be the '*' wildcard, which means any doAs value is
+ allowed, or a comma-separated list of groups.
+
+ If value is an empty list or webhcat.proxyuser.#USER#.groups is missing,
+ every doAs call value will fail.
+
+ For multiple users copy this property and replace the user name
+ in the property name.
+
+ The username->usergroup mapping is performed using Hadoop API which is
+ controlled by hadoop.security.group.mapping property. |
+
+
diff --git hcatalog/src/test/e2e/templeton/README.txt hcatalog/src/test/e2e/templeton/README.txt
index 647cfc1..e92bb7b 100644
--- hcatalog/src/test/e2e/templeton/README.txt
+++ hcatalog/src/test/e2e/templeton/README.txt
@@ -123,6 +123,29 @@ ant test-hcat-authorization -Dkeytab.dir=
The is expected to have keytab filenames of the form - user_name.*keytab .
+Running WebHCat doas tests
+--------------------------
+ant clean test-doas -Dinpdir.hdfs=/user/ekoifman/webhcate2e -Dsecure.mode=no
+ -Dharness.webhdfs.url=http://localhost:8085 -Dharness.templeton.url=http://localhost:50111
+ -Dtests.to.run='-t doAsTests' -Dtest.user.name=hue -Ddoas.user=joe
+
+The canonical example, is WebHCat server is running as user 'hcat', end user 'joe' is using Hue,
+which generates a request to WebHCat. If Hue specifies doAs=joe, then the commands that WebHCat
+submits to Hadoop will be run as user 'joe'.
+
+In order for this test suite to work, webhcat-site.xml should have webhcat.proxyuser.hue.groups
+and webhcat.proxyuser.hue.hosts defined, i.e. 'hue' should be allowed to impersonate 'joe'.
+[Of course, 'hcat' proxyuser should be configured in core-site.xml for the command to succeed.]
+
+Furthermore, metastore side file based security should be enabled. To do this 3 properties in
+hive-site.xml should be configured:
+1) hive.security.metastore.authorization.manager set to
+ org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider
+2) hive.security.metastore.authenticator.manager set to
+ org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator
+3) hive.metastore.pre.event.listeners set to
+ org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener
+4) hive.metastore.execute.setugi set to true
Notes
-----
diff --git hcatalog/src/test/e2e/templeton/build.xml hcatalog/src/test/e2e/templeton/build.xml
index f1361ea..7f59522 100644
--- hcatalog/src/test/e2e/templeton/build.xml
+++ hcatalog/src/test/e2e/templeton/build.xml
@@ -83,11 +83,15 @@
-
-
+ Defaults are 1, which means *no* parellelization:
+ if group=3, then 3 .conf files will be processed in parallel
+ if conf.file=2 there will be 2 thread per .conf file, each thread
+ executing a single group (identified by 'name' element) -->
+
+
+
@@ -141,6 +145,42 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
index 54c365d..7f1c967 100644
--- hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
+++ hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
@@ -174,6 +174,7 @@ sub globalSetup
$globalHash->{'webhdfs_url'} = $ENV{'WEBHDFS_URL'};
$globalHash->{'templeton_url'} = $ENV{'TEMPLETON_URL'};
$globalHash->{'current_user'} = $ENV{'USER_NAME'};
+ $globalHash->{'DOAS_USER'} = $ENV{'DOAS_USER'};
$globalHash->{'current_group_user'} = $ENV{'GROUP_USER_NAME'};
$globalHash->{'current_other_user'} = $ENV{'OTHER_USER_NAME'};
$globalHash->{'current_group'} = $ENV{'GROUP_NAME'};
@@ -317,6 +318,7 @@ sub replaceParametersInArg
}
my $outdir = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'};
$arg =~ s/:UNAME:/$testCmd->{'current_user'}/g;
+ $arg =~ s/:DOAS:/$testCmd->{'DOAS_USER'}/g;
$arg =~ s/:UNAME_GROUP:/$testCmd->{'current_group_user'}/g;
$arg =~ s/:UNAME_OTHER:/$testCmd->{'current_other_user'}/g;
$arg =~ s/:UGROUP:/$testCmd->{'current_group'}/g;
diff --git hcatalog/src/test/e2e/templeton/tests/doas.conf hcatalog/src/test/e2e/templeton/tests/doas.conf
new file mode 100644
index 0000000..ccd1269
--- /dev/null
+++ hcatalog/src/test/e2e/templeton/tests/doas.conf
@@ -0,0 +1,155 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+###############################################################################
+# curl command tests for templeton
+#
+#
+
+#use Yahoo::Miners::Test::PigSetup;
+
+#PigSetup::setup();
+
+#my $me = `whoami`;
+#chomp $me;
+
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+##=============================================================================================================
+#This suite tests support for doAs user in WebHCat.
+#This suite of tests requires some set up. They test that security context is properly propagated.
+#These tests are meant to run in File based security mode. Also, 2 users need to be created.
+#See README.txt for details on set up.
+#
+#
+
+ {
+ 'name' => 'doAsTests',
+ 'tests' =>
+ [
+
+ {
+ #drop table if exists to clean up from previous run
+ 'num' => 1,
+ 'method' => 'DELETE',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:&ifExists=true',
+ 'status_code' => 200,
+ 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'},
+ },
+ {
+ # create a table and set permission so that it's only accessible by owner
+ #(i.e. user issuing request)
+ 'num' => 2,
+ 'method' => 'PUT',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:',
+ 'format_header' => 'Content-Type: application/json',
+ 'post_options' => [' {
+ "columns": [
+ { "name": "id", "type": "bigint" },
+ { "name": "price", "type": "float"} ],
+ "partitionedBy": [
+ { "name": "country", "type": "string" } ],
+ "format" : { "storedAs" : "textfile"},
+ "permissions" : "rwx------"
+ }'],
+ 'status_code' => 200,
+ 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'},
+ },
+ {
+ #describe table with doAs user that UNAME is not allowed to impersonate
+ 'num' => 3,
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab1?user.name=:UNAME:&doAs=no_such_user',
+ 'status_code' => 401,
+ 'json_field_substr_match' => {'error' => 'Unauthorized proxyuser \[:UNAME:\] for doAsUser \[no_such_user\], not in proxyuser groups'},
+ },
+ {
+ #try to describe tale as a user that does not own (doesn't have read permissions on ) the table
+ #this is not going to work in secure mode
+ 'ignore' => 'will not work in secure mode',
+ 'num' => 4,
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab1?user.name=no_such_user',
+ 'status_code' => 401,
+ 'json_field_substr_match' => {'error' => 'Unauthorized proxyuser \[:UNAME:\] for doAsUser \[no_such_user\], not in proxyuser groups'},
+ },
+
+ {
+ #descbe the table (as the table owner)
+ #this should succeed
+ 'num' => 5,
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:',
+ 'status_code' => 200,
+ 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'},
+ },
+
+ {
+ #descbe the table (as the table owner but using doAs)
+ #this should succeed (it seems reading metadata is allowed even if reading data is not)
+ 'num' => 6,
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2/partition?user.name=:UNAME:?doAs=:DOAS:',
+ 'status_code' => 200,
+ 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'},
+ },
+
+ {
+ #this should fail
+ 'num' => 7,
+ 'method' => 'DELETE',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:&doAs=:DOAS:',
+ 'status_code' => 500,
+ 'json_field_substr_match' => {'error' => 'FAILED: Execution Error, return code 1 from org\.apache\.hadoop\.hive\.ql\.exec\.DDLTask\. MetaException\(message:java\.security\.AccessControlException: action WRITE not permitted on path.* for user :DOAS:\)'},
+ },
+ {
+ #descbe the table....
+ #this should succeed
+ 'num' => 8,
+ 'ignore' => 'foo',
+ 'method' => 'DELETE',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:',
+ 'status_code' => 200,
+ 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'},
+ },
+ ],
+ },
+ {
+ 'name' => 'bugs',
+ 'tests'=>
+ [
+ {
+ #update permissions to rwx------
+ 'num' => 1,
+ 'ignore' => 'permission setting seems broken - has no effect (not related to doAs) HIVE-5094',
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab1?user.name=:UNAME:',
+ 'format_header' => 'Content-Type: application/json',
+ 'post_options' => ['rename=:UNAME:_doastab2', 'permissions=rwx------'],
+ 'status_code' => 200,
+ 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'},
+ },
+ ]
+ },
+ ]
+},
+ ;
+
diff --git hcatalog/webhcat/svr/src/main/config/webhcat-default.xml hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
index d00f728..d60d806 100644
--- hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
+++ hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
@@ -234,5 +234,49 @@
in the cluster are taken over by Templeton launcher tasks.
+
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
index 8c143a8..9e15698 100644
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
@@ -138,6 +138,7 @@ private void init() {
String hadoopConfDir = getHadoopConfDir();
for (String fname : HADOOP_CONF_FILENAMES)
loadOneFileConfig(hadoopConfDir, fname);
+ ProxyUserSupport.processProxyuserConfig(this);
}
public void startCleanup() {
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
index cb89409..20c7a1f 100644
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
@@ -41,8 +41,7 @@
* launch child jobs.
*/
public class LauncherDelegator extends TempletonDelegator {
- private static final Log LOG = LogFactory.getLog(Server.class);
- public static final String JAR_CLASS = TempletonControllerJob.class.getName();
+ private static final Log LOG = LogFactory.getLog(LauncherDelegator.class);
protected String runAs = null;
public LauncherDelegator(AppConfig appConf) {
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ProxyUserSupport.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ProxyUserSupport.java
new file mode 100644
index 0000000..5aa032b
--- /dev/null
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ProxyUserSupport.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.Groups;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.MessageFormat;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * When WebHCat is run with doAs query parameter this class ensures that user making the
+ * call is allowed to impersonate doAs user and is making a call from authorized host.
+ */
+final class ProxyUserSupport {
+ private static final Log LOG = LogFactory.getLog(ProxyUserSupport.class);
+ private static final String CONF_PROXYUSER_PREFIX = "webhcat.proxyuser.";
+ private static final String CONF_GROUPS_SUFFIX = ".groups";
+ private static final String CONF_HOSTS_SUFFIX = ".hosts";
+ private static final Set WILD_CARD = Collections.unmodifiableSet(new HashSet(0));
+ private static final Map> proxyUserGroups = new HashMap>();
+ private static final Map> proxyUserHosts = new HashMap>();
+
+ static void processProxyuserConfig(AppConfig conf) {
+ for(Map.Entry confEnt : conf) {
+ if(confEnt.getKey().startsWith(CONF_PROXYUSER_PREFIX)
+ && confEnt.getKey().endsWith(CONF_GROUPS_SUFFIX)) {
+ //process user groups for which doAs is authorized
+ String proxyUser =
+ confEnt.getKey().substring(CONF_PROXYUSER_PREFIX.length(),
+ confEnt.getKey().lastIndexOf(CONF_GROUPS_SUFFIX));
+ Set groups;
+ if("*".equals(confEnt.getValue())) {
+ groups = WILD_CARD;
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("User [" + proxyUser + "] is authorized to do doAs any user.");
+ }
+ }
+ else if(confEnt.getValue() != null && confEnt.getValue().trim().length() > 0) {
+ groups = new HashSet(Arrays.asList(confEnt.getValue().trim().split(",")));
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("User [" + proxyUser +
+ "] is authorized to do doAs for users in the following groups: ["
+ + confEnt.getValue().trim() + "]");
+ }
+ }
+ else {
+ groups = Collections.emptySet();
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("User [" + proxyUser +
+ "] is authorized to do doAs for users in the following groups: []");
+ }
+ }
+ proxyUserGroups.put(proxyUser, groups);
+ }
+ else if(confEnt.getKey().startsWith(CONF_PROXYUSER_PREFIX)
+ && confEnt.getKey().endsWith(CONF_HOSTS_SUFFIX)) {
+ //process hosts from which doAs requests are authorized
+ String proxyUser = confEnt.getKey().substring(CONF_PROXYUSER_PREFIX.length(),
+ confEnt.getKey().lastIndexOf(CONF_HOSTS_SUFFIX));
+ Set hosts;
+ if("*".equals(confEnt.getValue())) {
+ hosts = WILD_CARD;
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("User [" + proxyUser + "] is authorized to do doAs from any host.");
+ }
+ }
+ else if(confEnt.getValue() != null && confEnt.getValue().trim().length() > 0) {
+ String[] hostValues = confEnt.getValue().trim().split(",");
+ hosts = new HashSet();
+ for(String hostname : hostValues) {
+ String nhn = normalizeHostname(hostname);
+ if(nhn != null) {
+ hosts.add(nhn);
+ }
+ }
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("User [" + proxyUser +
+ "] is authorized to do doAs from the following hosts: ["
+ + confEnt.getValue().trim() + "]");
+ }
+ }
+ else {
+ hosts = Collections.emptySet();
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("User [" + proxyUser
+ + "] is authorized to do doAs from the following hosts: []");
+ }
+ }
+ proxyUserHosts.put(proxyUser, hosts);
+ }
+ }
+ }
+ /**
+ * Verifies a that proxyUser is making the request from authorized host and that doAs user
+ * belongs to one of the groups for which proxyUser is allowed to impersonate users.
+ *
+ * @param proxyUser user name of the proxy (logged in) user.
+ * @param proxyHost host the proxy user is making the request from.
+ * @param doAsUser user the proxy user is impersonating.
+ * @throws NotAuthorizedException thrown if the user is not allowed to perform the proxyuser request.
+ */
+ static void validate(String proxyUser, String proxyHost, String doAsUser) throws
+ NotAuthorizedException {
+ assertNotEmpty(proxyUser, "proxyUser",
+ "If you're attempting to use user-impersonation via a proxy user, please make sure that "
+ + CONF_PROXYUSER_PREFIX + "#USER#" + CONF_HOSTS_SUFFIX + " and "
+ + CONF_PROXYUSER_PREFIX + "#USER#" + CONF_GROUPS_SUFFIX
+ + " are configured correctly");
+ assertNotEmpty(proxyHost, "proxyHost",
+ "If you're attempting to use user-impersonation via a proxy user, please make sure that "
+ + CONF_PROXYUSER_PREFIX + proxyUser + CONF_HOSTS_SUFFIX + " and "
+ + CONF_PROXYUSER_PREFIX + proxyUser + CONF_GROUPS_SUFFIX
+ + " are configured correctly");
+ assertNotEmpty(doAsUser, Server.DO_AS_PARAM);
+ LOG.debug(MessageFormat.format("Authorization check proxyuser [{0}] host [{1}] doAs [{2}]",
+ proxyUser, proxyHost, doAsUser));
+ if (proxyUserHosts.containsKey(proxyUser)) {
+ proxyHost = normalizeHostname(proxyHost);
+ validateRequestorHost(proxyUser, proxyHost);
+ validateGroup(proxyUser, doAsUser);
+ }
+ else {
+ throw new NotAuthorizedException(MessageFormat.format(
+ "User [{0}] not defined as proxyuser", proxyUser));
+ }
+ }
+
+ private static void validateRequestorHost(String proxyUser, String hostname) throws
+ NotAuthorizedException {
+ Set validHosts = proxyUserHosts.get(proxyUser);
+ if (validHosts == WILD_CARD) {
+ return;
+ }
+ if (validHosts == null || !validHosts.contains(hostname)) {
+ throw new NotAuthorizedException(MessageFormat.format(
+ "Unauthorized host [{0}] for proxyuser [{1}]", hostname, proxyUser));
+ }
+ }
+
+ private static void validateGroup(String proxyUser, String doAsUser) throws
+ NotAuthorizedException {
+ Set validGroups = proxyUserGroups.get(proxyUser);
+ if(validGroups == WILD_CARD) {
+ return;
+ }
+ else if(validGroups == null || validGroups.isEmpty()) {
+ throw new NotAuthorizedException(
+ MessageFormat.format(
+ "Unauthorized proxyuser [{0}] for doAsUser [{1}], not in proxyuser groups",
+ proxyUser, doAsUser));
+ }
+ Groups groupsInfo = new Groups(Main.getAppConfigInstance());
+ try {
+ List userGroups = groupsInfo.getGroups(doAsUser);
+ for (String g : validGroups) {
+ if (userGroups.contains(g)) {
+ return;
+ }
+ }
+ }
+ catch (IOException ex) {//thrown, for example, if there is no such user on the system
+ LOG.warn(MessageFormat.format("Unable to get list of groups for doAsUser [{0}].",
+ doAsUser), ex);
+ }
+ throw new NotAuthorizedException(
+ MessageFormat.format(
+ "Unauthorized proxyuser [{0}] for doAsUser [{1}], not in proxyuser groups",
+ proxyUser, doAsUser));
+ }
+
+ private static String normalizeHostname(String name) {
+ try {
+ InetAddress address = InetAddress.getByName(
+ "localhost".equalsIgnoreCase(name) ? null : name);
+ return address.getCanonicalHostName();
+ }
+ catch (UnknownHostException ex) {
+ LOG.warn(MessageFormat.format("Unable to normalize hostname [{0}]", name));
+ return null;
+ }
+ }
+ /**
+ * Check that a string is not null and not empty. If null or empty
+ * throws an IllegalArgumentException.
+ *
+ * @param str value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ private static String assertNotEmpty(String str, String name) {
+ return assertNotEmpty(str, name, null);
+ }
+
+ /**
+ * Check that a string is not null and not empty. If null or empty
+ * throws an IllegalArgumentException.
+ *
+ * @param str value.
+ * @param name parameter name for the exception message.
+ * @param info additional information to be printed with the exception message
+ * @return the given value.
+ */
+ private static String assertNotEmpty(String str, String name, String info) {
+ if (str == null) {
+ throw new IllegalArgumentException(
+ name + " cannot be null" + (info == null ? "" : ", " + info));
+ }
+ if (str.length() == 0) {
+ throw new IllegalArgumentException(
+ name + " cannot be empty" + (info == null ? "" : ", " + info));
+ }
+ return str;
+ }
+}
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java
index 29ac4b3..a6e2774 100644
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java
@@ -19,6 +19,9 @@
package org.apache.hcatalog.templeton;
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -26,6 +29,7 @@
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.DELETE;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
@@ -54,6 +58,7 @@
@Path("/v1")
public class Server {
public static final String VERSION = "v1";
+ public static final String DO_AS_PARAM = "doAs";
/**
* The status message. Always "ok"
@@ -113,6 +118,8 @@
private
@Context
UriInfo theUriInfo;
+ private @QueryParam(DO_AS_PARAM) String doAs;
+ private @Context HttpServletRequest request;
private static final Log LOG = LogFactory.getLog(Server.class);
@@ -161,7 +168,7 @@ public ExecBean ddl(@FormParam("exec") String exec,
verifyParam(exec, "exec");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.run(getUser(), exec, false, group, permissions);
+ return d.run(getDoAsUser(), exec, false, group, permissions);
}
/**
@@ -180,7 +187,7 @@ public Response listTables(@PathParam("db") String db,
HcatDelegator d = new HcatDelegator(appConf, execService);
if (!TempletonUtils.isset(tablePattern))
tablePattern = "*";
- return d.listTables(getUser(), db, tablePattern);
+ return d.listTables(getDoAsUser(), db, tablePattern);
}
/**
@@ -200,7 +207,7 @@ public Response createTable(@PathParam("db") String db,
desc.table = table;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.createTable(getUser(), db, desc);
+ return d.createTable(getDoAsUser(), db, desc);
}
/**
@@ -223,7 +230,7 @@ public Response createTableLike(@PathParam("db") String db,
desc.newTable = newTable;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.createTableLike(getUser(), db, desc);
+ return d.createTableLike(getDoAsUser(), db, desc);
}
/**
@@ -245,9 +252,9 @@ public Response descTable(@PathParam("db") String db,
HcatDelegator d = new HcatDelegator(appConf, execService);
if ("extended".equals(format))
- return d.descExtendedTable(getUser(), db, table);
+ return d.descExtendedTable(getDoAsUser(), db, table);
else
- return d.descTable(getUser(), db, table, false);
+ return d.descTable(getDoAsUser(), db, table, false);
}
/**
@@ -268,7 +275,7 @@ public Response dropTable(@PathParam("db") String db,
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.dropTable(getUser(), db, table, ifExists, group, permissions);
+ return d.dropTable(getDoAsUser(), db, table, ifExists, group, permissions);
}
/**
@@ -290,7 +297,7 @@ public Response renameTable(@PathParam("db") String db,
verifyDdlParam(newTable, "rename");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.renameTable(getUser(), db, oldTable, newTable, group, permissions);
+ return d.renameTable(getDoAsUser(), db, oldTable, newTable, group, permissions);
}
/**
@@ -310,7 +317,7 @@ public Response descOneTableProperty(@PathParam("db") String db,
verifyDdlParam(property, ":property");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.descTableProperty(getUser(), db, table, property);
+ return d.descTableProperty(getDoAsUser(), db, table, property);
}
/**
@@ -328,7 +335,7 @@ public Response listTableProperties(@PathParam("db") String db,
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.listTableProperties(getUser(), db, table);
+ return d.listTableProperties(getDoAsUser(), db, table);
}
/**
@@ -350,7 +357,7 @@ public Response addOneTableProperty(@PathParam("db") String db,
desc.name = property;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.addOneTableProperty(getUser(), db, table, desc);
+ return d.addOneTableProperty(getDoAsUser(), db, table, desc);
}
/**
@@ -368,7 +375,7 @@ public Response listPartitions(@PathParam("db") String db,
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.listPartitions(getUser(), db, table);
+ return d.listPartitions(getDoAsUser(), db, table);
}
/**
@@ -388,7 +395,7 @@ public Response descPartition(@PathParam("db") String db,
verifyParam(partition, ":partition");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.descOnePartition(getUser(), db, table, partition);
+ return d.descOnePartition(getDoAsUser(), db, table, partition);
}
/**
@@ -409,7 +416,7 @@ public Response addOnePartition(@PathParam("db") String db,
verifyParam(partition, ":partition");
desc.partition = partition;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.addOnePartition(getUser(), db, table, desc);
+ return d.addOnePartition(getDoAsUser(), db, table, desc);
}
/**
@@ -431,8 +438,8 @@ public Response dropPartition(@PathParam("db") String db,
verifyDdlParam(table, ":table");
verifyParam(partition, ":partition");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.dropPartition(getUser(), db, table, partition, ifExists,
- group, permissions);
+ return d.dropPartition(getDoAsUser(), db, table, partition, ifExists,
+ group, permissions);
}
/**
@@ -449,7 +456,7 @@ public Response listDatabases(@QueryParam("like") String dbPattern)
HcatDelegator d = new HcatDelegator(appConf, execService);
if (!TempletonUtils.isset(dbPattern))
dbPattern = "*";
- return d.listDatabases(getUser(), dbPattern);
+ return d.listDatabases(getDoAsUser(), dbPattern);
}
/**
@@ -465,7 +472,7 @@ public Response descDatabase(@PathParam("db") String db,
verifyUser();
verifyDdlParam(db, ":db");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.descDatabase(getUser(), db, "extended".equals(format));
+ return d.descDatabase(getDoAsUser(), db, "extended".equals(format));
}
/**
@@ -482,7 +489,7 @@ public Response createDatabase(@PathParam("db") String db,
verifyDdlParam(db, ":db");
desc.database = db;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.createDatabase(getUser(), desc);
+ return d.createDatabase(getDoAsUser(), desc);
}
/**
@@ -503,8 +510,8 @@ public Response dropDatabase(@PathParam("db") String db,
if (TempletonUtils.isset(option))
verifyDdlParam(option, "option");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.dropDatabase(getUser(), db, ifExists, option,
- group, permissions);
+ return d.dropDatabase(getDoAsUser(), db, ifExists, option,
+ group, permissions);
}
/**
@@ -523,7 +530,7 @@ public Response listColumns(@PathParam("db") String db,
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.listColumns(getUser(), db, table);
+ return d.listColumns(getDoAsUser(), db, table);
}
/**
@@ -543,7 +550,7 @@ public Response descColumn(@PathParam("db") String db,
verifyParam(column, ":column");
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.descOneColumn(getUser(), db, table, column);
+ return d.descOneColumn(getDoAsUser(), db, table, column);
}
/**
@@ -566,7 +573,7 @@ public Response addOneColumn(@PathParam("db") String db,
desc.name = column;
HcatDelegator d = new HcatDelegator(appConf, execService);
- return d.addOneColumn(getUser(), db, table, desc);
+ return d.addOneColumn(getDoAsUser(), db, table, desc);
}
/**
@@ -593,7 +600,7 @@ public EnqueueBean mapReduceStreaming(@FormParam("input") List inputs,
verifyParam(reducer, "reducer");
StreamingDelegator d = new StreamingDelegator(appConf);
- return d.run(getUser(), inputs, output, mapper, reducer,
+ return d.run(getDoAsUser(), inputs, output, mapper, reducer,
files, defines, cmdenvs, args,
statusdir, callback, getCompletedUrl());
}
@@ -619,7 +626,7 @@ public EnqueueBean mapReduceJar(@FormParam("jar") String jar,
verifyParam(mainClass, "class");
JarDelegator d = new JarDelegator(appConf);
- return d.run(getUser(),
+ return d.run(getDoAsUser(),
jar, mainClass,
libjars, files, args, defines,
statusdir, callback, getCompletedUrl());
@@ -644,7 +651,7 @@ public EnqueueBean pig(@FormParam("execute") String execute,
throw new BadParam("Either execute or file parameter required");
PigDelegator d = new PigDelegator(appConf);
- return d.run(getUser(),
+ return d.run(getDoAsUser(),
execute, srcFile,
pigArgs, otherFiles,
statusdir, callback, getCompletedUrl());
@@ -668,7 +675,7 @@ public EnqueueBean hive(@FormParam("execute") String execute,
throw new BadParam("Either execute or file parameter required");
HiveDelegator d = new HiveDelegator(appConf);
- return d.run(getUser(), execute, srcFile, defines,
+ return d.run(getDoAsUser(), execute, srcFile, defines,
statusdir, callback, getCompletedUrl());
}
@@ -685,7 +692,7 @@ public QueueStatusBean showQueueId(@PathParam("jobid") String jobid)
verifyParam(jobid, ":jobid");
StatusDelegator d = new StatusDelegator(appConf);
- return d.run(getUser(), jobid);
+ return d.run(getDoAsUser(), jobid);
}
/**
@@ -701,7 +708,7 @@ public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid)
verifyParam(jobid, ":jobid");
DeleteDelegator d = new DeleteDelegator(appConf);
- return d.run(getUser(), jobid);
+ return d.run(getDoAsUser(), jobid);
}
/**
@@ -716,7 +723,7 @@ public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid)
verifyUser();
ListDelegator d = new ListDelegator(appConf);
- return d.run(getUser());
+ return d.run(getDoAsUser());
}
/**
@@ -734,16 +741,30 @@ public CompleteBean completeJob(@PathParam("jobid") String jobid)
/**
* Verify that we have a valid user. Throw an exception if invalid.
*/
- public void verifyUser()
- throws NotAuthorizedException {
- if (getUser() == null) {
+ public void verifyUser() throws NotAuthorizedException {
+ String requestingUser = getRequestingUser();
+ if (requestingUser == null) {
String msg = "No user found.";
if (!UserGroupInformation.isSecurityEnabled())
msg += " Missing " + PseudoAuthenticator.USER_NAME + " parameter.";
throw new NotAuthorizedException(msg);
}
+ if(doAs != null && !doAs.equals(requestingUser)) {
+ /*if doAs user is different than logged in user, need to check that
+ that logged in user is authorized to run as 'doAs'*/
+ ProxyUserSupport.validate(requestingUser, getRequestingHost(requestingUser, request), doAs);
+ }
+ }
+ /**
+ * All 'tasks' spawned by WebHCat should be run as this user. W/o doAs query parameter
+ * this is just the user making the request (or
+ * {@link org.apache.hadoop.security.authentication.client.PseudoAuthenticator#USER_NAME}
+ * query param).
+ * @return value of doAs query parameter or {@link #getRequestingUser()}
+ */
+ private String getDoAsUser() {
+ return doAs != null && !doAs.equals(getRequestingUser()) ? doAs : getRequestingUser();
}
-
/**
* Verify that the parameter exists. Throw an exception if invalid.
*/
@@ -777,16 +798,20 @@ public void verifyDdlParam(String param, String name)
if (!m.matches())
throw new BadParam("Invalid DDL identifier " + name);
}
-
/**
- * Get the user name from the security context.
+ * Get the user name from the security context, i.e. the user making the HTTP request.
+ * With simple/pseudo security mode this should return the
+ * value of user.name query param, in kerberos mode it's the kinit'ed user.
*/
- public String getUser() {
+ private String getRequestingUser() {
if (theSecurityContext == null)
return null;
if (theSecurityContext.getUserPrincipal() == null)
return null;
- return theSecurityContext.getUserPrincipal().getName();
+ //map hue/foo.bar@something.com->hue since user group checks
+ // and config files are in terms of short name
+ return UserGroupInformation.createRemoteUser(
+ theSecurityContext.getUserPrincipal().getName()).getShortUserName();
}
/**
@@ -800,4 +825,32 @@ public String getCompletedUrl() {
return theUriInfo.getBaseUri() + VERSION
+ "/internal/complete/$jobId";
}
+ /**
+ * Returns canonical host name from which the request is made; used for doAs validation
+ */
+ private static String getRequestingHost(String requestingUser, HttpServletRequest request) {
+ final String unkHost = "???";
+ if(request == null) {
+ LOG.warn("request is null; cannot determine hostname");
+ return unkHost;
+ }
+ try {
+ String address = request.getRemoteAddr();//returns IP addr
+ if(address == null) {
+ LOG.warn(MessageFormat.format("Request remote address is NULL for user [{0}]", requestingUser));
+ return unkHost;
+ }
+
+ //Inet4Address/Inet6Address
+ String hostName = InetAddress.getByName(address).getCanonicalHostName();
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(MessageFormat.format("Resolved remote hostname: [{0}]", hostName));
+ }
+ return hostName;
+
+ } catch (UnknownHostException ex) {
+ LOG.warn(MessageFormat.format("Request remote address could not be resolved, {0}", ex.toString(), ex));
+ return unkHost;
+ }
+ }
}
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java
index f617341..d717771 100644
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java
@@ -27,7 +27,7 @@
private static ConcurrentHashMap userUgiMap =
new ConcurrentHashMap();
- static UserGroupInformation getUgi(String user) throws IOException {
+ public static UserGroupInformation getUgi(String user) throws IOException {
UserGroupInformation ugi = userUgiMap.get(user);
if (ugi == null) {
//create new ugi and add to map
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java
index 04887ba..801546d 100644
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java
@@ -19,6 +19,7 @@
package org.apache.hcatalog.templeton.tool;
import java.io.BufferedReader;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@@ -68,31 +69,29 @@ public void saveField(Type type, String id, String key, String val)
return;
}
PrintWriter out = null;
+ //todo: FileSystem#setPermission() - should this make sure to set 777 on jobs/ ?
+ Path keyfile= new Path(getPath(type) + "/" + id + "/" + key);
try {
- Path keyfile = new Path(getPath(type) + "/" + id + "/" + key);
// This will replace the old value if there is one
// Overwrite the existing file
out = new PrintWriter(new OutputStreamWriter(fs.create(keyfile)));
out.write(val);
- } catch (IOException e) {
- LOG.info("Couldn't write to " + getPath(type) + "/" + id + ": "
- + e.getMessage());
+ out.flush();
+ } catch (Exception e) {
+ String errMsg = "Couldn't write to " + keyfile + ": " + e.getMessage();
+ LOG.error(errMsg, e);
+ throw new NotFoundException(errMsg, e);
} finally {
- try {
- out.flush();
- out.close();
- } catch (Exception e) {
- // fail
- }
+ close(out);
}
}
@Override
public String getField(Type type, String id, String key) {
BufferedReader in = null;
+ Path p = new Path(getPath(type) + "/" + id + "/" + key);
try {
- in = new BufferedReader(new InputStreamReader(fs.open(new Path(getPath(type) + "/" +
- id + "/" + key))));
+ in = new BufferedReader(new InputStreamReader(fs.open(p)));
String line = null;
String val = "";
while ((line = in.readLine()) != null) {
@@ -102,15 +101,10 @@ public String getField(Type type, String id, String key) {
val += line;
}
return val;
- } catch (IOException e) {
- LOG.trace("Couldn't find " + getPath(type) + "/" + id + "/" + key
- + ": " + e.getMessage());
+ } catch (Exception e) {
+ LOG.info("Couldn't find " + p + ": " + e.getMessage(), e);
} finally {
- try {
- in.close();
- } catch (Exception e) {
- // fail
- }
+ close(in);
}
return null;
}
@@ -119,8 +113,9 @@ public String getField(Type type, String id, String key) {
public Map getFields(Type type, String id) {
HashMap map = new HashMap();
BufferedReader in = null;
+ Path p = new Path(getPath(type) + "/" + id);
try {
- for (FileStatus status : fs.listStatus(new Path(getPath(type) + "/" + id))) {
+ for (FileStatus status : fs.listStatus(p)) {
in = new BufferedReader(new InputStreamReader(fs.open(status.getPath())));
String line = null;
String val = "";
@@ -133,23 +128,20 @@ public String getField(Type type, String id, String key) {
map.put(status.getPath().getName(), val);
}
} catch (IOException e) {
- LOG.trace("Couldn't find " + getPath(type) + "/" + id);
+ LOG.trace("Couldn't find " + p);
} finally {
- try {
- in.close();
- } catch (Exception e) {
- // fail
- }
+ close(in);
}
return map;
}
@Override
public boolean delete(Type type, String id) throws NotFoundException {
+ Path p = new Path(getPath(type) + "/" + id);
try {
- fs.delete(new Path(getPath(type) + "/" + id), true);
+ fs.delete(p, true);
} catch (IOException e) {
- throw new NotFoundException("Node " + id + " was not found: " +
+ throw new NotFoundException("Node " + p + " was not found: " +
e.getMessage());
}
return false;
@@ -251,4 +243,15 @@ public static String getPath(Type type, String root) {
}
return typepath;
}
+ private void close(Closeable is) {
+ if(is == null) {
+ return;
+ }
+ try {
+ is.close();
+ }
+ catch (IOException ex) {
+ LOG.trace("Failed to close InputStream: " + ex.getMessage());
+ }
+ }
}
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java
index 1710869..d49f05a 100644
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java
@@ -27,4 +27,7 @@
public NotFoundException(String msg) {
super(msg);
}
+ public NotFoundException(String msg, Throwable rootCause) {
+ super(msg, rootCause);
+ }
}
diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java
index af4e1cf..5936a48 100644
--- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java
+++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java
@@ -38,6 +38,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hcatalog.templeton.UgiFactory;
/**
* General utility methods.
@@ -210,23 +211,19 @@ public static boolean hadoopFsIsMissing(FileSystem fs, Path p) {
}
}
- public static Path hadoopFsPath(String fname, Configuration conf, String user)
- throws URISyntaxException, FileNotFoundException, IOException,
+ public static Path hadoopFsPath(final String fname, final Configuration conf, String user)
+ throws URISyntaxException, IOException,
InterruptedException {
if (fname == null || conf == null) {
return null;
}
- final Configuration fConf = new Configuration(conf);
- final String finalFName = new String(fname);
-
- UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+ UserGroupInformation ugi = UgiFactory.getUgi(user);
final FileSystem defaultFs =
ugi.doAs(new PrivilegedExceptionAction() {
public FileSystem run()
- throws URISyntaxException, FileNotFoundException, IOException,
- InterruptedException {
- return FileSystem.get(new URI(finalFName), fConf);
+ throws URISyntaxException, IOException, InterruptedException {
+ return FileSystem.get(new URI(fname), conf);
}
});