diff --git hcatalog/src/test/e2e/templeton/README.txt hcatalog/src/test/e2e/templeton/README.txt index 647cfc1..151fdec 100644 --- hcatalog/src/test/e2e/templeton/README.txt +++ hcatalog/src/test/e2e/templeton/README.txt @@ -123,6 +123,28 @@ ant test-hcat-authorization -Dkeytab.dir= The is expected to have keytab filenames of the form - user_name.*keytab . +Running WebHCat doas tests +-------------------------- +ant clean test-doas -Dinpdir.hdfs=/user/ekoifman/webhcate2e -Dsecure.mode=no + -Dharness.webhdfs.url=http://localhost:8085 -Dharness.templeton.url=http://localhost:50111 + -Dtests.to.run='-t doAsTests' -Dtest.user.name=hue -Ddoas.user=joe + +The canonical example, is WebHCat server is running as user 'hcat', end user 'joe' is using Hue, +which generates a request to WebHCat. If Hue specifies doAs=joe, then the commands that WebHCat +submits to Hadoop will be run as user 'joe'. + +In order for this test suite to work, webhcat-site.xml should have webhcat.proxyuser.hue.groups +and webhcat.proxyuser.hue.hosts defined, i.e. 'hue' should be allowed to impersonate 'joe'. +[Of course, 'hcat' proxyuser should be configured in core-site.xml for the command to succeed.] + +Furthermore, metastore side file based security should be enabled. To do this 3 properties in +hive-site.xml should be configured: +1) hive.security.metastore.authorization.manager set to + org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider +2) hive.security.metastore.authenticator.manager set to + org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator +3) hive.metastore.pre.event.listeners set to + org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener Notes ----- diff --git hcatalog/src/test/e2e/templeton/build.xml hcatalog/src/test/e2e/templeton/build.xml index f1361ea..b8be6bd 100644 --- hcatalog/src/test/e2e/templeton/build.xml +++ hcatalog/src/test/e2e/templeton/build.xml @@ -83,11 +83,15 @@ - - + Defaults are 1, which means *no* parellelization: + if group=3, then 3 .conf files will be processed in parallel + if conf.file=2 there will be 2 thread per .conf file, each thread + executing a single group (identified by 'name' element) --> + + + @@ -101,6 +105,7 @@ + @@ -109,6 +114,7 @@ + @@ -141,6 +147,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm index 54c365d..220d0f8 100644 --- hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm +++ hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm @@ -174,6 +174,7 @@ sub globalSetup $globalHash->{'webhdfs_url'} = $ENV{'WEBHDFS_URL'}; $globalHash->{'templeton_url'} = $ENV{'TEMPLETON_URL'}; $globalHash->{'current_user'} = $ENV{'USER_NAME'}; + $globalHash->{'doas.user'} = $ENV{'doas.user'}; $globalHash->{'current_group_user'} = $ENV{'GROUP_USER_NAME'}; $globalHash->{'current_other_user'} = $ENV{'OTHER_USER_NAME'}; $globalHash->{'current_group'} = $ENV{'GROUP_NAME'}; @@ -317,6 +318,7 @@ sub replaceParametersInArg } my $outdir = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'}; $arg =~ s/:UNAME:/$testCmd->{'current_user'}/g; + $arg =~ s/:DOAS:/$testCmd->{'doas.user'}/g; $arg =~ s/:UNAME_GROUP:/$testCmd->{'current_group_user'}/g; $arg =~ s/:UNAME_OTHER:/$testCmd->{'current_other_user'}/g; $arg =~ s/:UGROUP:/$testCmd->{'current_group'}/g; diff --git hcatalog/src/test/e2e/templeton/tests/doas.conf hcatalog/src/test/e2e/templeton/tests/doas.conf new file mode 100644 index 0000000..219950e --- /dev/null +++ hcatalog/src/test/e2e/templeton/tests/doas.conf @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +############################################################################### +# curl command tests for templeton +# +# + +#use Yahoo::Miners::Test::PigSetup; + +#PigSetup::setup(); + +#my $me = `whoami`; +#chomp $me; + +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ +##============================================================================================================= +#This suite tests support for doAs user in WebHCat. +#This suite of tests requires some set up. They test that security context is properly propagated. +#These tests are meant to run in File based security mode. Also, 2 users need to be created. +#See README.txt for details on set up. +# +# + + { + 'name' => 'doAsTests', + 'tests' => + [ + + { + #drop table if exists to clean up from previous run + 'num' => 1, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:&ifExists=true', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'}, + }, + { + # create a table and set permission so that it's only accessible by owner + #(i.e. user issuing request) + 'num' => 2, + 'method' => 'PUT', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => [' { + "columns": [ + { "name": "id", "type": "bigint" }, + { "name": "price", "type": "float"} ], + "partitionedBy": [ + { "name": "country", "type": "string" } ], + "format" : { "storedAs" : "textfile"}, + "permissions" : "rwx------" + }'], + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'}, + }, + { + #describe table with doAs user that UNAME is not allowed to impersonate + 'num' => 3, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab1?user.name=:UNAME:&doAs=no_such_user', + 'status_code' => 401, + 'json_field_substr_match' => {'error' => 'Unauthorized proxyuser \[:UNAME:\] for doAsUser \[no_such_user\], not in proxyuser groups'}, + }, + { + #try to describe tale as a user that does not own (doesn't have read permissions on ) the table + #this is not going to work in secure mode + 'ignore' => 'will not work in secure mode', + 'num' => 4, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab1?user.name=no_such_user', + 'status_code' => 401, + 'json_field_substr_match' => {'error' => 'Unauthorized proxyuser \[:UNAME:\] for doAsUser \[no_such_user\], not in proxyuser groups'}, + }, + + { + #descbe the table (as the table owner) + #this should succeed + 'num' => 5, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'}, + }, + + { + #descbe the table (as the table owner but using doAs) + #this should succeed (it seems reading metadata is allowed even if reading data is not) + 'num' => 6, + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2/partition?user.name=:UNAME:?doAs=:DOAS:', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'}, + }, + + { + #this should fail + 'num' => 7, + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:&doAs=:DOAS:', + 'status_code' => 500, + 'json_field_substr_match' => {'error' => 'FAILED: Execution Error, return code 1 from org\.apache\.hadoop\.hive\.ql\.exec\.DDLTask\. MetaException\(message:java\.security\.AccessControlException: action WRITE not permitted on path.* for user :DOAS:\)'}, + }, + { + #descbe the table.... + #this should succeed + 'num' => 8, + 'ignore' => 'foo', + 'method' => 'DELETE', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:', + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'}, + }, + ], + }, + { + 'name' => 'bugs', + 'tests'=> + [ + { + #update permissions to rwx------ //todo: try again and file a bug + 'num' => 1, + 'ignore' => 'permission setting seems broken - has no effect (not related to doAs)', + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab1?user.name=:UNAME:', + 'format_header' => 'Content-Type: application/json', + 'post_options' => ['rename=:UNAME:_doastab2', 'permissions=rwx------'], + 'status_code' => 200, + 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'}, + }, + ] + }, + ] +}, + ; + diff --git hcatalog/webhcat/svr/src/main/config/webhcat-default.xml hcatalog/webhcat/svr/src/main/config/webhcat-default.xml index d00f728..8977ee3 100644 --- hcatalog/webhcat/svr/src/main/config/webhcat-default.xml +++ hcatalog/webhcat/svr/src/main/config/webhcat-default.xml @@ -234,5 +234,46 @@ in the cluster are taken over by Templeton launcher tasks. + diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java index 8c143a8..9e15698 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java @@ -138,6 +138,7 @@ private void init() { String hadoopConfDir = getHadoopConfDir(); for (String fname : HADOOP_CONF_FILENAMES) loadOneFileConfig(hadoopConfDir, fname); + ProxyUserSupport.processProxyuserConfig(this); } public void startCleanup() { diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java index cb89409..20c7a1f 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java @@ -41,8 +41,7 @@ * launch child jobs. */ public class LauncherDelegator extends TempletonDelegator { - private static final Log LOG = LogFactory.getLog(Server.class); - public static final String JAR_CLASS = TempletonControllerJob.class.getName(); + private static final Log LOG = LogFactory.getLog(LauncherDelegator.class); protected String runAs = null; public LauncherDelegator(AppConfig appConf) { diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ProxyUserSupport.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ProxyUserSupport.java new file mode 100644 index 0000000..e431ecc --- /dev/null +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ProxyUserSupport.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hcatalog.templeton; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.Groups; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * When WebHCat is run with doAs query parameter this class ensures that user making the + * call is allowed to impersonate doAs user and is making a call from authorized host. + */ +final class ProxyUserSupport { + private static final Log LOG = LogFactory.getLog(ProxyUserSupport.class); + private static final String CONF_PROXYUSER_PREFIX = "webhcat.proxyuser."; + private static final String CONF_GROUPS_SUFFIX = ".groups"; + private static final String CONF_HOSTS_SUFFIX = ".hosts"; + private static final Set WILD_CARD = Collections.unmodifiableSet(new HashSet(0)); + private static final Map> proxyUserGroups = new HashMap>(); + private static final Map> proxyUserHosts = new HashMap>(); + + static void processProxyuserConfig(AppConfig conf) { + for(Map.Entry confEnt : conf) { + if(confEnt.getKey().startsWith(CONF_PROXYUSER_PREFIX) + && confEnt.getKey().endsWith(CONF_GROUPS_SUFFIX)) { + //process user groups for which doAs is authorized + String proxyUser = normalizeUsername( + confEnt.getKey().substring(CONF_PROXYUSER_PREFIX.length(), + confEnt.getKey().lastIndexOf(CONF_GROUPS_SUFFIX))); + Set groups; + if("*".equals(confEnt.getValue())) { + groups = WILD_CARD; + if(LOG.isDebugEnabled()) { + LOG.debug("User [" + proxyUser + "] is authorized to do doAs any user."); + } + } + else if(confEnt.getValue() != null && confEnt.getValue().trim().length() > 0) { + groups = new HashSet(Arrays.asList(confEnt.getValue().trim().split(","))); + if(LOG.isDebugEnabled()) { + LOG.debug("User [" + proxyUser + + "] is authorized to do doAs for users in the following groups: [" + + confEnt.getValue().trim() + "]"); + } + } + else { + groups = Collections.emptySet(); + if(LOG.isDebugEnabled()) { + LOG.debug("User [" + proxyUser + + "] is authorized to do doAs for users in the following groups: []"); + } + } + proxyUserGroups.put(proxyUser, groups); + } + else if(confEnt.getKey().startsWith(CONF_PROXYUSER_PREFIX) + && confEnt.getKey().endsWith(CONF_HOSTS_SUFFIX)) { + //process hosts from which doAs requests are authorized + String proxyUser = normalizeUsername(confEnt.getKey().substring(CONF_PROXYUSER_PREFIX.length(), + confEnt.getKey().lastIndexOf(CONF_HOSTS_SUFFIX))); + Set hosts; + if("*".equals(confEnt.getValue())) { + hosts = WILD_CARD; + if(LOG.isDebugEnabled()) { + LOG.debug("User [" + proxyUser + "] is authorized to do doAs from any host."); + } + } + else if(confEnt.getValue() != null && confEnt.getValue().trim().length() > 0) { + String[] hostValues = confEnt.getValue().trim().split(","); + hosts = new HashSet(); + for(String hostname : hostValues) { + String nhn = normalizeHostname(hostname); + if(nhn != null) { + hosts.add(nhn); + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("User [" + proxyUser + + "] is authorized to do doAs from the following hosts: [" + + confEnt.getValue().trim() + "]"); + } + } + else { + hosts = Collections.emptySet(); + if(LOG.isDebugEnabled()) { + LOG.debug("User [" + proxyUser + + "] is authorized to do doAs from the following hosts: []"); + } + } + proxyUserHosts.put(proxyUser, hosts); + } + } + } + + /** + * In secure mode Server#getUser() returns name as username@EXAMPLE.COM + * Requiring short name in webhcat-site.xml is consistent with hadoop and + * oozie proxyuser support + */ + static String normalizeUsername(String username) { + int atPos; + if(username == null || (atPos = username.indexOf("@")) < 0) { + return username; + } + return username.substring(0, atPos); + } + /** + * Verifies a that proxyUser is making the request from authorized host and that doAs user + * belongs to one of the groups for which proxyUser is allowed to impersonate users. + * + * @param proxyUser user name of the proxy (logged in) user. + * @param proxyHost host the proxy user is making the request from. + * @param doAsUser user the proxy user is impersonating. + * @throws NotAuthorizedException thrown if the user is not allowed to perform the proxyuser request. + */ + static void validate(String proxyUser, String proxyHost, String doAsUser) throws + NotAuthorizedException { + proxyUser = normalizeUsername(proxyUser); + assertNotEmpty(proxyUser, "proxyUser", + "If you're attempting to use user-impersonation via a proxy user, please make sure that " + + CONF_PROXYUSER_PREFIX + "#USER#" + CONF_HOSTS_SUFFIX + " and " + + CONF_PROXYUSER_PREFIX + "#USER#" + CONF_GROUPS_SUFFIX + + " are configured correctly"); + assertNotEmpty(proxyHost, "proxyHost", + "If you're attempting to use user-impersonation via a proxy user, please make sure that " + + CONF_PROXYUSER_PREFIX + proxyUser + CONF_HOSTS_SUFFIX + " and " + + CONF_PROXYUSER_PREFIX + proxyUser + CONF_GROUPS_SUFFIX + + " are configured correctly"); + assertNotEmpty(doAsUser, Server.DO_AS_PARAM); + LOG.debug(MessageFormat.format("Authorization check proxyuser [{0}] host [{1}] doAs [{2}]", + proxyUser, proxyHost, doAsUser)); + if (proxyUserHosts.containsKey(proxyUser)) { + proxyHost = normalizeHostname(proxyHost); + validateRequestorHost(proxyUser, proxyHost); + validateGroup(proxyUser, doAsUser); + } + else { + throw new NotAuthorizedException(MessageFormat.format( + "User [{0}] not defined as proxyuser", proxyUser)); + } + } + + private static void validateRequestorHost(String proxyUser, String hostname) throws + NotAuthorizedException { + Set validHosts = proxyUserHosts.get(proxyUser); + if (validHosts == WILD_CARD) { + return; + } + if (validHosts == null || !validHosts.contains(hostname)) { + throw new NotAuthorizedException(MessageFormat.format( + "Unauthorized host [{0}] for proxyuser [{1}]", hostname, proxyUser)); + } + } + + private static void validateGroup(String proxyUser, String doAsUser) throws + NotAuthorizedException { + Set validGroups = proxyUserGroups.get(proxyUser); + if(validGroups == WILD_CARD) { + return; + } + else if(validGroups == null || validGroups.isEmpty()) { + throw new NotAuthorizedException( + MessageFormat.format( + "Unauthorized proxyuser [{0}] for doAsUser [{1}], not in proxyuser groups", + proxyUser, doAsUser)); + } + Groups groupsInfo = new Groups(Main.getAppConfigInstance()); + try { + List userGroups = groupsInfo.getGroups(doAsUser); + for (String g : validGroups) { + if (userGroups.contains(g)) { + return; + } + } + } + catch (IOException ex) {//thrown, for example, if there is no such user on the system + LOG.warn(MessageFormat.format("Unable to get list of groups for doAsUser [{0}].", + doAsUser), ex); + } + throw new NotAuthorizedException( + MessageFormat.format( + "Unauthorized proxyuser [{0}] for doAsUser [{1}], not in proxyuser groups", + proxyUser, doAsUser)); + } + + private static String normalizeHostname(String name) { + try { + InetAddress address = InetAddress.getByName( + "localhost".equalsIgnoreCase(name) ? null : name); + return address.getCanonicalHostName(); + } + catch (UnknownHostException ex) { + LOG.warn(MessageFormat.format("Unable to normalize hostname [{0}]", name)); + return null; + } + } + /** + * Check that a string is not null and not empty. If null or empty + * throws an IllegalArgumentException. + * + * @param str value. + * @param name parameter name for the exception message. + * @return the given value. + */ + private static String assertNotEmpty(String str, String name) { + return assertNotEmpty(str, name, null); + } + + /** + * Check that a string is not null and not empty. If null or empty + * throws an IllegalArgumentException. + * + * @param str value. + * @param name parameter name for the exception message. + * @param info additional information to be printed with the exception message + * @return the given value. + */ + private static String assertNotEmpty(String str, String name, String info) { + if (str == null) { + throw new IllegalArgumentException( + name + " cannot be null" + (info == null ? "" : ", " + info)); + } + if (str.length() == 0) { + throw new IllegalArgumentException( + name + " cannot be empty" + (info == null ? "" : ", " + info)); + } + return str; + } +} diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java index 29ac4b3..415ddf3 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java @@ -19,6 +19,9 @@ package org.apache.hcatalog.templeton; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -26,6 +29,7 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.DELETE; import javax.ws.rs.FormParam; import javax.ws.rs.GET; @@ -54,6 +58,7 @@ @Path("/v1") public class Server { public static final String VERSION = "v1"; + public static final String DO_AS_PARAM = "doAs"; /** * The status message. Always "ok" @@ -113,6 +118,8 @@ private @Context UriInfo theUriInfo; + private @QueryParam(DO_AS_PARAM) String doAs; + private @Context HttpServletRequest request; private static final Log LOG = LogFactory.getLog(Server.class); @@ -161,7 +168,7 @@ public ExecBean ddl(@FormParam("exec") String exec, verifyParam(exec, "exec"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.run(getUser(), exec, false, group, permissions); + return d.run(getDoAsUser(), exec, false, group, permissions); } /** @@ -180,7 +187,7 @@ public Response listTables(@PathParam("db") String db, HcatDelegator d = new HcatDelegator(appConf, execService); if (!TempletonUtils.isset(tablePattern)) tablePattern = "*"; - return d.listTables(getUser(), db, tablePattern); + return d.listTables(getDoAsUser(), db, tablePattern); } /** @@ -200,7 +207,7 @@ public Response createTable(@PathParam("db") String db, desc.table = table; HcatDelegator d = new HcatDelegator(appConf, execService); - return d.createTable(getUser(), db, desc); + return d.createTable(getDoAsUser(), db, desc); } /** @@ -223,7 +230,7 @@ public Response createTableLike(@PathParam("db") String db, desc.newTable = newTable; HcatDelegator d = new HcatDelegator(appConf, execService); - return d.createTableLike(getUser(), db, desc); + return d.createTableLike(getDoAsUser(), db, desc); } /** @@ -245,9 +252,9 @@ public Response descTable(@PathParam("db") String db, HcatDelegator d = new HcatDelegator(appConf, execService); if ("extended".equals(format)) - return d.descExtendedTable(getUser(), db, table); + return d.descExtendedTable(getDoAsUser(), db, table); else - return d.descTable(getUser(), db, table, false); + return d.descTable(getDoAsUser(), db, table, false); } /** @@ -268,7 +275,7 @@ public Response dropTable(@PathParam("db") String db, verifyDdlParam(table, ":table"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.dropTable(getUser(), db, table, ifExists, group, permissions); + return d.dropTable(getDoAsUser(), db, table, ifExists, group, permissions); } /** @@ -290,7 +297,7 @@ public Response renameTable(@PathParam("db") String db, verifyDdlParam(newTable, "rename"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.renameTable(getUser(), db, oldTable, newTable, group, permissions); + return d.renameTable(getDoAsUser(), db, oldTable, newTable, group, permissions); } /** @@ -310,7 +317,7 @@ public Response descOneTableProperty(@PathParam("db") String db, verifyDdlParam(property, ":property"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.descTableProperty(getUser(), db, table, property); + return d.descTableProperty(getDoAsUser(), db, table, property); } /** @@ -328,7 +335,7 @@ public Response listTableProperties(@PathParam("db") String db, verifyDdlParam(table, ":table"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.listTableProperties(getUser(), db, table); + return d.listTableProperties(getDoAsUser(), db, table); } /** @@ -350,7 +357,7 @@ public Response addOneTableProperty(@PathParam("db") String db, desc.name = property; HcatDelegator d = new HcatDelegator(appConf, execService); - return d.addOneTableProperty(getUser(), db, table, desc); + return d.addOneTableProperty(getDoAsUser(), db, table, desc); } /** @@ -368,7 +375,7 @@ public Response listPartitions(@PathParam("db") String db, verifyDdlParam(table, ":table"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.listPartitions(getUser(), db, table); + return d.listPartitions(getDoAsUser(), db, table); } /** @@ -388,7 +395,7 @@ public Response descPartition(@PathParam("db") String db, verifyParam(partition, ":partition"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.descOnePartition(getUser(), db, table, partition); + return d.descOnePartition(getDoAsUser(), db, table, partition); } /** @@ -409,7 +416,7 @@ public Response addOnePartition(@PathParam("db") String db, verifyParam(partition, ":partition"); desc.partition = partition; HcatDelegator d = new HcatDelegator(appConf, execService); - return d.addOnePartition(getUser(), db, table, desc); + return d.addOnePartition(getDoAsUser(), db, table, desc); } /** @@ -431,7 +438,7 @@ public Response dropPartition(@PathParam("db") String db, verifyDdlParam(table, ":table"); verifyParam(partition, ":partition"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.dropPartition(getUser(), db, table, partition, ifExists, + return d.dropPartition(getDoAsUser(), db, table, partition, ifExists, group, permissions); } @@ -449,7 +456,7 @@ public Response listDatabases(@QueryParam("like") String dbPattern) HcatDelegator d = new HcatDelegator(appConf, execService); if (!TempletonUtils.isset(dbPattern)) dbPattern = "*"; - return d.listDatabases(getUser(), dbPattern); + return d.listDatabases(getDoAsUser(), dbPattern); } /** @@ -465,7 +472,7 @@ public Response descDatabase(@PathParam("db") String db, verifyUser(); verifyDdlParam(db, ":db"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.descDatabase(getUser(), db, "extended".equals(format)); + return d.descDatabase(getDoAsUser(), db, "extended".equals(format)); } /** @@ -482,7 +489,7 @@ public Response createDatabase(@PathParam("db") String db, verifyDdlParam(db, ":db"); desc.database = db; HcatDelegator d = new HcatDelegator(appConf, execService); - return d.createDatabase(getUser(), desc); + return d.createDatabase(getDoAsUser(), desc); } /** @@ -503,7 +510,7 @@ public Response dropDatabase(@PathParam("db") String db, if (TempletonUtils.isset(option)) verifyDdlParam(option, "option"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.dropDatabase(getUser(), db, ifExists, option, + return d.dropDatabase(getDoAsUser(), db, ifExists, option, group, permissions); } @@ -523,7 +530,7 @@ public Response listColumns(@PathParam("db") String db, verifyDdlParam(table, ":table"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.listColumns(getUser(), db, table); + return d.listColumns(getDoAsUser(), db, table); } /** @@ -543,7 +550,7 @@ public Response descColumn(@PathParam("db") String db, verifyParam(column, ":column"); HcatDelegator d = new HcatDelegator(appConf, execService); - return d.descOneColumn(getUser(), db, table, column); + return d.descOneColumn(getDoAsUser(), db, table, column); } /** @@ -566,7 +573,7 @@ public Response addOneColumn(@PathParam("db") String db, desc.name = column; HcatDelegator d = new HcatDelegator(appConf, execService); - return d.addOneColumn(getUser(), db, table, desc); + return d.addOneColumn(getDoAsUser(), db, table, desc); } /** @@ -593,7 +600,7 @@ public EnqueueBean mapReduceStreaming(@FormParam("input") List inputs, verifyParam(reducer, "reducer"); StreamingDelegator d = new StreamingDelegator(appConf); - return d.run(getUser(), inputs, output, mapper, reducer, + return d.run(getDoAsUser(), inputs, output, mapper, reducer, files, defines, cmdenvs, args, statusdir, callback, getCompletedUrl()); } @@ -619,7 +626,7 @@ public EnqueueBean mapReduceJar(@FormParam("jar") String jar, verifyParam(mainClass, "class"); JarDelegator d = new JarDelegator(appConf); - return d.run(getUser(), + return d.run(getDoAsUser(), jar, mainClass, libjars, files, args, defines, statusdir, callback, getCompletedUrl()); @@ -644,7 +651,7 @@ public EnqueueBean pig(@FormParam("execute") String execute, throw new BadParam("Either execute or file parameter required"); PigDelegator d = new PigDelegator(appConf); - return d.run(getUser(), + return d.run(getDoAsUser(), execute, srcFile, pigArgs, otherFiles, statusdir, callback, getCompletedUrl()); @@ -668,7 +675,7 @@ public EnqueueBean hive(@FormParam("execute") String execute, throw new BadParam("Either execute or file parameter required"); HiveDelegator d = new HiveDelegator(appConf); - return d.run(getUser(), execute, srcFile, defines, + return d.run(getDoAsUser(), execute, srcFile, defines, statusdir, callback, getCompletedUrl()); } @@ -685,7 +692,7 @@ public QueueStatusBean showQueueId(@PathParam("jobid") String jobid) verifyParam(jobid, ":jobid"); StatusDelegator d = new StatusDelegator(appConf); - return d.run(getUser(), jobid); + return d.run(getDoAsUser(), jobid); } /** @@ -701,7 +708,7 @@ public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid) verifyParam(jobid, ":jobid"); DeleteDelegator d = new DeleteDelegator(appConf); - return d.run(getUser(), jobid); + return d.run(getDoAsUser(), jobid); } /** @@ -716,7 +723,7 @@ public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid) verifyUser(); ListDelegator d = new ListDelegator(appConf); - return d.run(getUser()); + return d.run(getDoAsUser()); } /** @@ -734,16 +741,30 @@ public CompleteBean completeJob(@PathParam("jobid") String jobid) /** * Verify that we have a valid user. Throw an exception if invalid. */ - public void verifyUser() - throws NotAuthorizedException { - if (getUser() == null) { + public void verifyUser() throws NotAuthorizedException { + String requestingUser = getRequestingUser(); + if (requestingUser == null) { String msg = "No user found."; if (!UserGroupInformation.isSecurityEnabled()) msg += " Missing " + PseudoAuthenticator.USER_NAME + " parameter."; throw new NotAuthorizedException(msg); } + if(doAs != null && !doAs.equals(ProxyUserSupport.normalizeUsername(requestingUser))) { + /*if doAs user is different than logged in user, need to check that + that logged in user is authorized to run as 'doAs'*/ + ProxyUserSupport.validate(requestingUser, getRequestingHost(requestingUser, request), doAs); + } + } + /** + * All 'tasks' spawned by WebHCat should be run as this user. W/o doAs query parameter + * this is just the user making the request (or + * {@link org.apache.hadoop.security.authentication.client.PseudoAuthenticator#USER_NAME} + * query param). + * @return value of doAs query parameter or {@link #getRequestingUser()} + */ + private String getDoAsUser() { + return doAs != null && !doAs.equals(getRequestingUser()) ? doAs : getRequestingUser(); } - /** * Verify that the parameter exists. Throw an exception if invalid. */ @@ -777,11 +798,12 @@ public void verifyDdlParam(String param, String name) if (!m.matches()) throw new BadParam("Invalid DDL identifier " + name); } - /** - * Get the user name from the security context. + * Get the user name from the security context, i.e. the user making the HTTP request. + * With simple/pseudo security mode this should return the + * value of user.name query param, in kerberos mode it's the kinit'ed user. */ - public String getUser() { + private String getRequestingUser() { if (theSecurityContext == null) return null; if (theSecurityContext.getUserPrincipal() == null) @@ -800,4 +822,32 @@ public String getCompletedUrl() { return theUriInfo.getBaseUri() + VERSION + "/internal/complete/$jobId"; } + /** + * Returns canonical host name from which the request is made; used for doAs validation + */ + private static String getRequestingHost(String requestingUser, HttpServletRequest request) { + final String unkHost = "???"; + if(request == null) { + LOG.warn("request is null; cannot determine hostname"); + return unkHost; + } + try { + String address = request.getRemoteAddr();//returns IP addr + if(address == null) { + LOG.warn(MessageFormat.format("Request remote address is NULL for user [{0}]", requestingUser)); + return unkHost; + } + + //Inet4Address/Inet6Address + String hostName = InetAddress.getByName(address).getCanonicalHostName(); + if(LOG.isDebugEnabled()) { + LOG.debug(MessageFormat.format("Resolved remote hostname: [{0}]", hostName)); + } + return hostName; + + } catch (UnknownHostException ex) { + LOG.warn(MessageFormat.format("Request remote address could not be resolved, {0}", ex.toString(), ex)); + return unkHost; + } + } } diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java index f617341..d717771 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java @@ -27,7 +27,7 @@ private static ConcurrentHashMap userUgiMap = new ConcurrentHashMap(); - static UserGroupInformation getUgi(String user) throws IOException { + public static UserGroupInformation getUgi(String user) throws IOException { UserGroupInformation ugi = userUgiMap.get(user); if (ugi == null) { //create new ugi and add to map diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java index 04887ba..801546d 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java @@ -19,6 +19,7 @@ package org.apache.hcatalog.templeton.tool; import java.io.BufferedReader; +import java.io.Closeable; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; @@ -68,31 +69,29 @@ public void saveField(Type type, String id, String key, String val) return; } PrintWriter out = null; + //todo: FileSystem#setPermission() - should this make sure to set 777 on jobs/ ? + Path keyfile= new Path(getPath(type) + "/" + id + "/" + key); try { - Path keyfile = new Path(getPath(type) + "/" + id + "/" + key); // This will replace the old value if there is one // Overwrite the existing file out = new PrintWriter(new OutputStreamWriter(fs.create(keyfile))); out.write(val); - } catch (IOException e) { - LOG.info("Couldn't write to " + getPath(type) + "/" + id + ": " - + e.getMessage()); + out.flush(); + } catch (Exception e) { + String errMsg = "Couldn't write to " + keyfile + ": " + e.getMessage(); + LOG.error(errMsg, e); + throw new NotFoundException(errMsg, e); } finally { - try { - out.flush(); - out.close(); - } catch (Exception e) { - // fail - } + close(out); } } @Override public String getField(Type type, String id, String key) { BufferedReader in = null; + Path p = new Path(getPath(type) + "/" + id + "/" + key); try { - in = new BufferedReader(new InputStreamReader(fs.open(new Path(getPath(type) + "/" + - id + "/" + key)))); + in = new BufferedReader(new InputStreamReader(fs.open(p))); String line = null; String val = ""; while ((line = in.readLine()) != null) { @@ -102,15 +101,10 @@ public String getField(Type type, String id, String key) { val += line; } return val; - } catch (IOException e) { - LOG.trace("Couldn't find " + getPath(type) + "/" + id + "/" + key - + ": " + e.getMessage()); + } catch (Exception e) { + LOG.info("Couldn't find " + p + ": " + e.getMessage(), e); } finally { - try { - in.close(); - } catch (Exception e) { - // fail - } + close(in); } return null; } @@ -119,8 +113,9 @@ public String getField(Type type, String id, String key) { public Map getFields(Type type, String id) { HashMap map = new HashMap(); BufferedReader in = null; + Path p = new Path(getPath(type) + "/" + id); try { - for (FileStatus status : fs.listStatus(new Path(getPath(type) + "/" + id))) { + for (FileStatus status : fs.listStatus(p)) { in = new BufferedReader(new InputStreamReader(fs.open(status.getPath()))); String line = null; String val = ""; @@ -133,23 +128,20 @@ public String getField(Type type, String id, String key) { map.put(status.getPath().getName(), val); } } catch (IOException e) { - LOG.trace("Couldn't find " + getPath(type) + "/" + id); + LOG.trace("Couldn't find " + p); } finally { - try { - in.close(); - } catch (Exception e) { - // fail - } + close(in); } return map; } @Override public boolean delete(Type type, String id) throws NotFoundException { + Path p = new Path(getPath(type) + "/" + id); try { - fs.delete(new Path(getPath(type) + "/" + id), true); + fs.delete(p, true); } catch (IOException e) { - throw new NotFoundException("Node " + id + " was not found: " + + throw new NotFoundException("Node " + p + " was not found: " + e.getMessage()); } return false; @@ -251,4 +243,15 @@ public static String getPath(Type type, String root) { } return typepath; } + private void close(Closeable is) { + if(is == null) { + return; + } + try { + is.close(); + } + catch (IOException ex) { + LOG.trace("Failed to close InputStream: " + ex.getMessage()); + } + } } diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java index 1710869..d49f05a 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/NotFoundException.java @@ -27,4 +27,7 @@ public NotFoundException(String msg) { super(msg); } + public NotFoundException(String msg, Throwable rootCause) { + super(msg, rootCause); + } } diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java index af4e1cf..5936a48 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.hcatalog.templeton.UgiFactory; /** * General utility methods. @@ -210,23 +211,19 @@ public static boolean hadoopFsIsMissing(FileSystem fs, Path p) { } } - public static Path hadoopFsPath(String fname, Configuration conf, String user) - throws URISyntaxException, FileNotFoundException, IOException, + public static Path hadoopFsPath(final String fname, final Configuration conf, String user) + throws URISyntaxException, IOException, InterruptedException { if (fname == null || conf == null) { return null; } - final Configuration fConf = new Configuration(conf); - final String finalFName = new String(fname); - - UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + UserGroupInformation ugi = UgiFactory.getUgi(user); final FileSystem defaultFs = ugi.doAs(new PrivilegedExceptionAction() { public FileSystem run() - throws URISyntaxException, FileNotFoundException, IOException, - InterruptedException { - return FileSystem.get(new URI(finalFName), fConf); + throws URISyntaxException, IOException, InterruptedException { + return FileSystem.get(new URI(fname), conf); } });