diff --git a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java index d8ba3aa..a24927f 100644 --- a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java +++ b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java @@ -53,7 +53,7 @@ KERBEROS("KERBEROS"), CUSTOM("CUSTOM"); - private String authType; // Auth type for SASL + private String authType; AuthTypes(String authType) { this.authType = authType; @@ -67,37 +67,47 @@ public String getAuthName() { private HadoopThriftAuthBridge.Server saslServer = null; private String authTypeStr; + private String transportMode; HiveConf conf; public HiveAuthFactory() throws TTransportException { conf = new HiveConf(); - + transportMode = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE); authTypeStr = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION); - if (authTypeStr == null) { - authTypeStr = AuthTypes.NONE.getAuthName(); + + // In http mode we use NOSASL as the default auth type + if (transportMode.equalsIgnoreCase("http")) { + if (authTypeStr == null) { + authTypeStr = AuthTypes.NOSASL.getAuthName(); + } } - if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName()) - && ShimLoader.getHadoopShims().isSecureShimImpl()) { - saslServer = ShimLoader.getHadoopThriftAuthBridge().createServer( - conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB), - conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL) - ); + else { + if (authTypeStr == null) { + authTypeStr = AuthTypes.NONE.getAuthName(); + } + if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName()) + && ShimLoader.getHadoopShims().isSecureShimImpl()) { + saslServer = ShimLoader.getHadoopThriftAuthBridge().createServer( + conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB), + conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL) + ); + } } } public Map getSaslProperties() { Map saslProps = new HashMap(); SaslQOP saslQOP = - SaslQOP.fromString(conf.getVar(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP)); + SaslQOP.fromString(conf.getVar(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP)); // hadoop.rpc.protection being set to a higher level than hive.server2.thrift.rpc.protection // does not make sense in most situations. Log warning message in such cases. Map hadoopSaslProps = ShimLoader.getHadoopThriftAuthBridge(). - getHadoopSaslProperties(conf); + getHadoopSaslProperties(conf); SaslQOP hadoopSaslQOP = SaslQOP.fromString(hadoopSaslProps.get(Sasl.QOP)); if(hadoopSaslQOP.ordinal() > saslQOP.ordinal()) { LOG.warn(MessageFormat.format("\"hadoop.rpc.protection\" is set to higher security level " + - "{0} then {1} which is set to {2}", hadoopSaslQOP.toString(), - ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP.varname, saslQOP.toString())); + "{0} then {1} which is set to {2}", hadoopSaslQOP.toString(), + ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP.varname, saslQOP.toString())); } saslProps.put(Sasl.QOP, saslQOP.toString()); saslProps.put(Sasl.SERVER_AUTH, "true"); @@ -130,10 +140,15 @@ public TTransportFactory getAuthTransFactory() throws LoginException { public TProcessorFactory getAuthProcFactory(ThriftCLIService service) throws LoginException { - if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) { - return KerberosSaslHelper.getKerberosProcessorFactory(saslServer, service); - } else { - return PlainSaslHelper.getPlainProcessorFactory(service); + if (transportMode.equalsIgnoreCase("http")) { + return HttpAuthHelper.getAuthProcFactory(service); + } + else { + if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) { + return KerberosSaslHelper.getKerberosProcessorFactory(saslServer, service); + } else { + return PlainSaslHelper.getPlainProcessorFactory(service); + } } } diff --git a/service/src/java/org/apache/hive/service/auth/HttpAuthHelper.java b/service/src/java/org/apache/hive/service/auth/HttpAuthHelper.java new file mode 100644 index 0000000..bee1987 --- /dev/null +++ b/service/src/java/org/apache/hive/service/auth/HttpAuthHelper.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hive.service.auth; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.cli.thrift.TCLIService; +import org.apache.hive.service.cli.thrift.TCLIService.Iface; +import org.apache.hive.service.cli.thrift.ThriftCLIService; +import org.apache.thrift.TProcessor; +import org.apache.thrift.TProcessorFactory; +import org.apache.thrift.transport.TTransport; + +public class HttpAuthHelper { + + public static final String WWW_AUTHENTICATE = "WWW-Authenticate"; + public static final String AUTHORIZATION = "Authorization"; + public static final String BASIC = "Basic"; + public static final String NEGOTIATE = "Negotiate"; + public static final String COMPLETE = "Complete"; + + private static class HttpCLIServiceProcessorFactory extends TProcessorFactory { + private final ThriftCLIService service; + private final HiveConf hiveConf; + private final boolean isDoAsEnabled; + + public HttpCLIServiceProcessorFactory(ThriftCLIService service) { + super(null); + this.service = service; + this.hiveConf = service.getHiveConf(); + this.isDoAsEnabled = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS); + } + + @Override + public TProcessor getProcessor(TTransport trans) { + TProcessor baseProcessor = new TCLIService.Processor(service); + return isDoAsEnabled ? new HttpCLIServiceProcessor(baseProcessor) : + new HttpCLIServiceProcessor(baseProcessor); + } + } + + public static TProcessorFactory getAuthProcFactory(ThriftCLIService service) { + return new HttpCLIServiceProcessorFactory(service); + } + +} diff --git a/service/src/java/org/apache/hive/service/auth/HttpCLIServiceProcessor.java b/service/src/java/org/apache/hive/service/auth/HttpCLIServiceProcessor.java new file mode 100644 index 0000000..2aaf575 --- /dev/null +++ b/service/src/java/org/apache/hive/service/auth/HttpCLIServiceProcessor.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.auth; + +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; + +public class HttpCLIServiceProcessor implements TProcessor { + + private final TProcessor underlyingProcessor; + + public HttpCLIServiceProcessor(TProcessor underlyingProcessor) { + this.underlyingProcessor = underlyingProcessor; + } + + @Override + public boolean process(final TProtocol in, final TProtocol out) throws TException { + return underlyingProcessor.process(in, out); + } +} diff --git a/service/src/java/org/apache/hive/service/auth/HttpCLIServiceUGIProcessor.java b/service/src/java/org/apache/hive/service/auth/HttpCLIServiceUGIProcessor.java new file mode 100644 index 0000000..75770de --- /dev/null +++ b/service/src/java/org/apache/hive/service/auth/HttpCLIServiceUGIProcessor.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.auth; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; + +import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hive.service.cli.session.SessionManager; +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; + +public class HttpCLIServiceUGIProcessor implements TProcessor { + + private final TProcessor underlyingProcessor; + private final HadoopShims shim; + + public HttpCLIServiceUGIProcessor(TProcessor underlyingProcessor) { + this.underlyingProcessor = underlyingProcessor; + this.shim = ShimLoader.getHadoopShims(); + } + + @Override + public boolean process(final TProtocol in, final TProtocol out) throws TException { + /** + * Build the client UGI from threadlocal username [SessionManager.getUserName()]. + * The threadlocal username is set in the ThriftHttpServlet. + */ + UserGroupInformation clientUgi = null; + try { + clientUgi = shim.createRemoteUser(SessionManager.getUserName(), new ArrayList()); + // Clear the thread local username since we set it in each http request + SessionManager.clearUserName(); + return shim.doAs(clientUgi, new PrivilegedExceptionAction() { + @Override + public Boolean run() { + try { + return underlyingProcessor.process(in, out); + } catch (TException te) { + throw new RuntimeException(te); + } + } + }); + } + catch (RuntimeException rte) { + if (rte.getCause() instanceof TException) { + throw (TException)rte.getCause(); + } + throw rte; + } catch (InterruptedException ie) { + throw new RuntimeException(ie); // unexpected! + } catch (IOException ioe) { + throw new RuntimeException(ioe); // unexpected! + } + } +} diff --git a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java index bfe0e7b..9e0890b 100644 --- a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java +++ b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java @@ -100,14 +100,14 @@ public SessionHandle openSession(TProtocolVersion protocol, String username, Str public SessionHandle openSession(TProtocolVersion protocol, String username, String password, Map sessionConf, boolean withImpersonation, String delegationToken) - throws HiveSQLException { + throws HiveSQLException { if (username == null) { username = threadLocalUserName.get(); } HiveSession session; if (withImpersonation) { HiveSessionImplwithUGI hiveSessionUgi = new HiveSessionImplwithUGI(protocol, username, password, - sessionConf, delegationToken); + sessionConf, delegationToken); session = HiveSessionProxy.getProxy(hiveSessionUgi, hiveSessionUgi.getSessionUgi()); hiveSessionUgi.setProxySession(session); } else { @@ -157,7 +157,11 @@ public static void setIpAddress(String ipAddress) { threadLocalIpAddress.set(ipAddress); } - private void clearIpAddress() { + public static String getIpAddress() { + return threadLocalIpAddress.get(); + } + + public static void clearIpAddress() { threadLocalIpAddress.remove(); } @@ -172,7 +176,11 @@ public static void setUserName(String userName) { threadLocalUserName.set(userName); } - private void clearUserName() { + public static String getUserName() { + return threadLocalUserName.get(); + } + + public static void clearUserName() { threadLocalUserName.remove(); } diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index a6ff6ce..c4c7cd7 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -21,8 +21,11 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.util.Shell; +import org.apache.hive.service.auth.HiveAuthFactory; import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; import org.apache.hive.service.cli.CLIService; +import org.apache.thrift.TProcessor; +import org.apache.thrift.TProcessorFactory; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocolFactory; import org.apache.thrift.server.TServlet; @@ -59,23 +62,7 @@ public void run() { minWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MIN_WORKER_THREADS); maxWorkerThreads = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_WORKER_THREADS); - String httpPath = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH); - // The config parameter can be like "path", "/path", "/path/", "path/*", "/path1/path2/*" and so on. - // httpPath should end up as "/*", "/path/*" or "/path1/../pathN/*" - if(httpPath == null || httpPath.equals("")) { - httpPath = "/*"; - } - else { - if(!httpPath.startsWith("/")) { - httpPath = "/" + httpPath; - } - if(httpPath.endsWith("/")) { - httpPath = httpPath + "*"; - } - if(!httpPath.endsWith("/*")) { - httpPath = httpPath + "/*"; - } - } + String httpPath = getHttpPath(hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); httpServer = new org.eclipse.jetty.server.Server(); QueuedThreadPool threadPool = new QueuedThreadPool(); @@ -85,6 +72,8 @@ public void run() { SelectChannelConnector connector; Boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL); String schemeName = useSsl ? "https" : "http"; + String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION); + boolean isDoAsEnabled = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS); if (useSsl) { String keyStorePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH).trim(); @@ -106,13 +95,16 @@ public void run() { connector.setReuseAddress(!Shell.WINDOWS); httpServer.addConnector(connector); - TCLIService.Processor processor = - new TCLIService.Processor(new EmbeddedThriftBinaryCLIService()); + hiveAuthFactory = new HiveAuthFactory(); + TProcessorFactory processorFactory = hiveAuthFactory.getAuthProcFactory(this); + TProcessor processor = processorFactory.getProcessor(null); TProtocolFactory protocolFactory = new TBinaryProtocol.Factory(); - TServlet thriftHttpServlet = new ThriftHttpServlet(processor, protocolFactory); + TServlet thriftHttpServlet = new ThriftHttpServlet(processor, + protocolFactory, authType, isDoAsEnabled); - final ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + final ServletContextHandler context = new ServletContextHandler( + ServletContextHandler.SESSIONS); context.setContextPath("/"); httpServer.setHandler(context); context.addServlet(new ServletHolder(thriftHttpServlet), httpPath); @@ -130,6 +122,30 @@ public void run() { } /** + * The config parameter can be like "path", "/path", "/path/", "path/*", "/path1/path2/*" and so on. + * httpPath should end up as "/*", "/path/*" or "/path1/../pathN/*" + * @param httpPath + * @return + */ + private String getHttpPath(String httpPath) { + if(httpPath == null || httpPath.equals("")) { + httpPath = "/*"; + } + else { + if(!httpPath.startsWith("/")) { + httpPath = "/" + httpPath; + } + if(httpPath.endsWith("/")) { + httpPath = httpPath + "*"; + } + if(!httpPath.endsWith("/*")) { + httpPath = httpPath + "/*"; + } + } + return httpPath; + } + + /** * Verify that this configuration is supported by transportMode of HTTP * @param hiveConf */ diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index e77f043..87c7908 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -28,6 +28,8 @@ import org.apache.commons.codec.binary.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hive.service.auth.HttpAuthHelper; +import org.apache.hive.service.cli.session.SessionManager; import org.apache.thrift.TProcessor; import org.apache.thrift.protocol.TProtocolFactory; import org.apache.thrift.server.TServlet; @@ -36,51 +38,64 @@ private static final long serialVersionUID = 1L; public static final Log LOG = LogFactory.getLog(ThriftHttpServlet.class.getName()); + private final String authType; + private final boolean isDoAsEnabled; - public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory) { + public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory, + String authType, boolean isDoAsEnabled) { super(processor, protocolFactory); + this.authType = authType; + this.isDoAsEnabled = isDoAsEnabled; } @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logRequestHeader(request); + /** + * Set the username and ip address in the session manager's threadlocals + * We will use these for doAs + */ + if(isDoAsEnabled) { + SessionManager.setUserName(getUsername(request)); + } + logBasicRequestHeader(request); super.doPost(request, response); } - protected void logRequestHeader(HttpServletRequest request) { - String authHeaderBase64 = request.getHeader("Authorization"); - if(authHeaderBase64 == null) { - LOG.warn("ThriftHttpServlet: no HTTP Authorization header"); - } - else { - if(!authHeaderBase64.startsWith("Basic")) { - LOG.warn("ThriftHttpServlet: HTTP Authorization header exists but is not Basic."); - } - else if(LOG.isDebugEnabled()) { - String authHeaderBase64_Payload = authHeaderBase64.substring("Basic ".length()); - String authHeaderString = StringUtils.newStringUtf8( - Base64.decodeBase64(authHeaderBase64_Payload.getBytes())); - String[] creds = authHeaderString.split(":"); - String username = null; - String password = null; + private String getUsername(HttpServletRequest request) { + String authHeaderBase64 = request.getHeader(HttpAuthHelper.AUTHORIZATION); + String authHeaderBase64Payload = authHeaderBase64.substring((HttpAuthHelper.BASIC + " ").length()); + String authHeaderString = StringUtils.newStringUtf8( + Base64.decodeBase64(authHeaderBase64Payload.getBytes())); + String[] creds = authHeaderString.split(":"); + return creds[0]; + } - if(creds.length >= 1) { - username = creds[0]; - } - if(creds.length >= 2) { - password = creds[1]; - } - if(password == null || password.equals("null") || password.equals("")) { - password = ""; - } - else { - // don't log the actual password. - password = "******"; - } - LOG.debug("HttpServlet: HTTP Authorization header:: username=" + username + - " password=" + password); + protected void logBasicRequestHeader(HttpServletRequest request) { + String authHeaderBase64 = request.getHeader(HttpAuthHelper.AUTHORIZATION); + if(LOG.isDebugEnabled()) { + String authHeaderBase64Payload = authHeaderBase64.substring((HttpAuthHelper.BASIC + " ").length()); + String authHeaderString = StringUtils.newStringUtf8( + Base64.decodeBase64(authHeaderBase64Payload.getBytes())); + String[] creds = authHeaderString.split(":"); + String username = null; + String password = null; + + if(creds.length >= 1) { + username = creds[0]; + } + if(creds.length >= 2) { + password = creds[1]; + } + if(password == null || password.equals("null") || password.equals("")) { + password = ""; + } + else { + // don't log the actual password. + password = "******"; } + LOG.debug("HttpServlet: HTTP Authorization header:: username=" + username + + " password=" + password); } } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 9e9a60d..62b45ca 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -77,15 +77,15 @@ * @return TaskAttempt Log Url */ String getTaskAttemptLogUrl(JobConf conf, - String taskTrackerHttpAddress, - String taskAttemptId) - throws MalformedURLException; + String taskTrackerHttpAddress, + String taskAttemptId) + throws MalformedURLException; /** * Returns a shim to wrap MiniMrCluster */ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir) throws IOException; + String nameNode, int numDir) throws IOException; /** * Shim for MiniMrCluster @@ -122,7 +122,7 @@ int createHadoopArchive(Configuration conf, Path parentDir, Path destDir, String archiveName) throws Exception; public URI getHarUri(URI original, URI base, URI originalBase) - throws URISyntaxException; + throws URISyntaxException; /** * Hive uses side effect files exclusively for it's output. It also manages * the setup/cleanup/commit of output from the hive client. As a result it does @@ -162,7 +162,7 @@ public URI getHarUri(URI original, URI base, URI originalBase) * @throws InterruptedException */ public T doAs(UserGroupInformation ugi, PrivilegedExceptionAction pvea) throws - IOException, InterruptedException; + IOException, InterruptedException; /** * Once a delegation token is stored in a file, the location is specified @@ -185,7 +185,7 @@ public URI getHarUri(URI original, URI base, URI originalBase) /** - * Used by metastore server to creates UGI object for a remote user. + * Used to creates UGI object for a remote user. * @param userName remote User Name * @param groupNames group names associated with remote user name * @return UGI created for the remote user. @@ -237,7 +237,7 @@ public URI getHarUri(URI original, URI base, URI originalBase) * @throws IOException */ void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenService) - throws IOException; + throws IOException; enum JobTrackerState { INITIALIZING, RUNNING }; @@ -319,7 +319,7 @@ void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenService) * @throws IOException */ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf) - throws IOException; + throws IOException; /** * Get the default block size for the path. FileSystem alone is not sufficient to @@ -363,6 +363,7 @@ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration co public interface InputSplitShim extends InputSplit { JobConf getJob(); + @Override long getLength(); /** Returns an array containing the startoffsets of the files in the split. */ @@ -387,14 +388,18 @@ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration co Path[] getPaths(); /** Returns all the Paths where this input-split resides. */ + @Override String[] getLocations() throws IOException; void shrinkSplit(long length); + @Override String toString(); + @Override void readFields(DataInput in) throws IOException; + @Override void write(DataOutput out) throws IOException; } @@ -426,7 +431,7 @@ RecordReader getRecordReader(JobConf job, InputSplitShim split, Reporter reporte * @throws IOException */ Iterator listLocatedStatus(FileSystem fs, Path path, - PathFilter filter) throws IOException; + PathFilter filter) throws IOException; /** * For file status returned by listLocatedStatus, convert them into a list @@ -437,7 +442,7 @@ RecordReader getRecordReader(JobConf job, InputSplitShim split, Reporter reporte * @throws IOException */ BlockLocation[] getLocations(FileSystem fs, - FileStatus status) throws IOException; + FileStatus status) throws IOException; public HCatHadoopShims getHCatShim(); public interface HCatHadoopShims { @@ -449,10 +454,10 @@ RecordReader getRecordReader(JobConf job, InputSplitShim split, Reporter reporte public TaskAttemptID createTaskAttemptID(); public org.apache.hadoop.mapreduce.TaskAttemptContext createTaskAttemptContext(Configuration conf, - TaskAttemptID taskId); + TaskAttemptID taskId); public org.apache.hadoop.mapred.TaskAttemptContext createTaskAttemptContext(JobConf conf, - org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable); + org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable); public JobContext createJobContext(Configuration conf, JobID jobId); @@ -521,7 +526,7 @@ RecordReader getRecordReader(JobConf job, InputSplitShim split, Reporter reporte public Map getHadoopConfNames(); - + /** * Get configuration from JobContext */