diff --git hadoop-common-project/hadoop-common/CHANGES.txt hadoop-common-project/hadoop-common/CHANGES.txt index aa93ffc..93d6431 100644 --- hadoop-common-project/hadoop-common/CHANGES.txt +++ hadoop-common-project/hadoop-common/CHANGES.txt @@ -413,9 +413,6 @@ Release 2.5.0 - UNRELEASED HADOOP-10614. CBZip2InputStream is not threadsafe (Xiangrui Meng via Sandy Ryza) - HADOOP-10618. Remove SingleNodeSetup.apt.vm. (Akira Ajisaka via - Arpit Agarwal) - OPTIMIZATIONS BUG FIXES diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index b79196e..d2f38cb 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2140,21 +2140,9 @@ protected Path resolveLink(Path f) throws IOException { * in the corresponding FileSystem. */ public FileChecksum getFileChecksum(Path f) throws IOException { - return getFileChecksum(f, Long.MAX_VALUE); - } - - /** - * Get the checksum of a file, from the beginning of the file till the - * specific length. - * @param f The file path - * @param length The length of the file range for checksum calculation - * @return The file checksum. - */ - public FileChecksum getFileChecksum(Path f, final long length) - throws IOException { return null; } - + /** * Set the verify checksum flag. This is only applicable if the * corresponding FileSystem supports checksum. By default doesn't do anything. diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index b98ff40..ac09427 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Progressable; @@ -427,12 +428,7 @@ protected Path resolveLink(Path f) throws IOException { public FileChecksum getFileChecksum(Path f) throws IOException { return fs.getFileChecksum(f); } - - @Override - public FileChecksum getFileChecksum(Path f, long length) throws IOException { - return fs.getFileChecksum(f, length); - } - + @Override public void setVerifyChecksum(boolean verifyChecksum) { fs.setVerifyChecksum(verifyChecksum); diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 3ba6de1..04780d0 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -687,7 +687,7 @@ private HarStatus getFileHarStatus(Path f) throws IOException { * @return null since no checksum algorithm is implemented. */ @Override - public FileChecksum getFileChecksum(Path f, long length) { + public FileChecksum getFileChecksum(Path f) { return null; } diff --git hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm index eb0c801..ebfb146 100644 --- hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm +++ hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm @@ -18,7 +18,210 @@ Single Node Setup - This page will be removed in the next major release. +%{toc|section=1|fromDepth=0} - See {{{./SingleCluster.html}Single Cluster Setup}} to set up and configure a - single-node Hadoop installation. +* Purpose + + This document describes how to set up and configure a single-node + Hadoop installation so that you can quickly perform simple operations + using Hadoop MapReduce and the Hadoop Distributed File System (HDFS). + +* Prerequisites + +** Supported Platforms + + * GNU/Linux is supported as a development and production platform. + Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes. + + * Windows is also a supported platform. + +** Required Software + + Required software for Linux and Windows include: + + [[1]] Java^TM 1.6.x, preferably from Sun, must be installed. + + [[2]] ssh must be installed and sshd must be running to use the Hadoop + scripts that manage remote Hadoop daemons. + +** Installing Software + + If your cluster doesn't have the requisite software you will need to + install it. + + For example on Ubuntu Linux: + +---- + $ sudo apt-get install ssh + $ sudo apt-get install rsync +---- + +* Download + + To get a Hadoop distribution, download a recent stable release from one + of the Apache Download Mirrors. + +* Prepare to Start the Hadoop Cluster + + Unpack the downloaded Hadoop distribution. In the distribution, edit + the file <<>> to define at least <<>> to be the root + of your Java installation. + + Try the following command: + +---- + $ bin/hadoop +---- + + This will display the usage documentation for the hadoop script. + + Now you are ready to start your Hadoop cluster in one of the three + supported modes: + + * Local (Standalone) Mode + + * Pseudo-Distributed Mode + + * Fully-Distributed Mode + +* Standalone Operation + + By default, Hadoop is configured to run in a non-distributed mode, as a + single Java process. This is useful for debugging. + + The following example copies the unpacked conf directory to use as + input and then finds and displays every match of the given regular + expression. Output is written to the given output directory. + +---- + $ mkdir input + $ cp conf/*.xml input + $ bin/hadoop jar hadoop-*-examples.jar grep input output 'dfs[a-z.]+' + $ cat output/* +--- + +* Pseudo-Distributed Operation + + Hadoop can also be run on a single-node in a pseudo-distributed mode + where each Hadoop daemon runs in a separate Java process. + +** Configuration + + Use the following: + + conf/core-site.xml: + +---- + + + fs.defaultFS + hdfs://localhost:9000 + + +---- + + conf/hdfs-site.xml: + +---- + + + dfs.replication + 1 + + +---- + + conf/mapred-site.xml: + +---- + + + mapred.job.tracker + localhost:9001 + + +---- + +** Setup passphraseless ssh + + Now check that you can ssh to the localhost without a passphrase: + +---- + $ ssh localhost +---- + + If you cannot ssh to localhost without a passphrase, execute the + following commands: + +---- + $ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa + $ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys +---- + +** Execution + + Format a new distributed-filesystem: + +---- + $ bin/hadoop namenode -format +---- + + Start the hadoop daemons: + +---- + $ bin/start-all.sh +---- + + The hadoop daemon log output is written to the <<<${HADOOP_LOG_DIR}>>> + directory (defaults to <<<${HADOOP_PREFIX}/logs>>>). + + Browse the web interface for the NameNode and the JobTracker; by + default they are available at: + + * NameNode - <<>> + + * JobTracker - <<>> + + Copy the input files into the distributed filesystem: + +---- + $ bin/hadoop fs -put conf input +---- + + Run some of the examples provided: + +---- + $ bin/hadoop jar hadoop-*-examples.jar grep input output 'dfs[a-z.]+' +---- + + Examine the output files: + + Copy the output files from the distributed filesystem to the local + filesytem and examine them: + +---- + $ bin/hadoop fs -get output output + $ cat output/* +---- + + or + + View the output files on the distributed filesystem: + +---- + $ bin/hadoop fs -cat output/* +---- + + When you're done, stop the daemons with: + +---- + $ bin/stop-all.sh +---- + +* Fully-Distributed Operation + + For information on setting up fully-distributed, non-trivial clusters + see {{{./ClusterSetup.html}Cluster Setup}}. + + Java and JNI are trademarks or registered trademarks of Sun + Microsystems, Inc. in the United States and other countries. diff --git hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java index 16db5b1..1614d6b 100644 --- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java +++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java @@ -139,7 +139,6 @@ public void primitiveMkdir(Path f, FsPermission absolutePermission, public int getDefaultPort(); public String getCanonicalServiceName(); public Token getDelegationToken(String renewer) throws IOException; - public FileChecksum getFileChecksum(Path f) throws IOException; public boolean deleteOnExit(Path f) throws IOException; public boolean cancelDeleteOnExit(Path f) throws IOException; public Token[] addDelegationTokens(String renewer, Credentials creds) @@ -224,16 +223,10 @@ static void checkInvalidPath(String s, Configuration conf) { } @Test - public void testFileChecksum() throws Exception { + public void testFileChecksum() { final Path p = new Path("har://file-localhost/foo.har/file1"); final HarFileSystem harfs = new HarFileSystem(); - try { - Assert.assertEquals(null, harfs.getFileChecksum(p)); - } finally { - if (harfs != null) { - harfs.close(); - } - } + Assert.assertEquals(null, harfs.getFileChecksum(p)); } /** diff --git hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java index 10930b9..1ed3d3f 100644 --- hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java +++ hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java @@ -18,14 +18,8 @@ package org.apache.hadoop.nfs.nfs3; import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -50,17 +44,6 @@ static final String MAC_GET_ALL_USERS_CMD = "dscl . -list /Users UniqueID"; static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID"; - // Used for finding the configured static mapping file. - static final String NFS_STATIC_MAPPING_FILE_KEY = "dfs.nfs.static.mapping.file"; - private static final String NFS_STATIC_MAPPING_FILE_DEFAULT = "/etc/nfs.map"; - private final File staticMappingFile; - - // Used for parsing the static mapping file. - private static final Pattern EMPTY_LINE = Pattern.compile("^\\s*$"); - private static final Pattern COMMENT_LINE = Pattern.compile("^\\s*#.*$"); - private static final Pattern MAPPING_LINE = - Pattern.compile("^(uid|gid)\\s+(\\d+)\\s+(\\d+)\\s*(#.*)?$"); - // Do update every 15 minutes by default final static long TIMEOUT_DEFAULT = 15 * 60 * 1000; // ms final static long TIMEOUT_MIN = 1 * 60 * 1000; // ms @@ -75,7 +58,6 @@ public IdUserGroup() throws IOException { timeout = TIMEOUT_DEFAULT; - staticMappingFile = new File(NFS_STATIC_MAPPING_FILE_DEFAULT); updateMaps(); } @@ -89,11 +71,6 @@ public IdUserGroup(Configuration conf) throws IOException { } else { timeout = updateTime; } - - String staticFilePath = conf.get(NFS_STATIC_MAPPING_FILE_KEY, - NFS_STATIC_MAPPING_FILE_DEFAULT); - staticMappingFile = new File(staticFilePath); - updateMaps(); } @@ -160,8 +137,7 @@ private static Integer parseId(final String idStr) { */ @VisibleForTesting public static void updateMapInternal(BiMap map, String mapName, - String command, String regex, Map staticMapping) - throws IOException { + String command, String regex) throws IOException { BufferedReader br = null; try { Process process = Runtime.getRuntime().exec( @@ -175,7 +151,7 @@ public static void updateMapInternal(BiMap map, String mapName, } LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]); // HDFS can't differentiate duplicate names with simple authentication - final Integer key = staticMapping.get(parseId(nameId[1])); + final Integer key = parseId(nameId[1]); final String value = nameId[0]; if (map.containsKey(key)) { final String prevValue = map.get(key); @@ -197,7 +173,7 @@ public static void updateMapInternal(BiMap map, String mapName, } map.put(key, value); } - LOG.info("Updated " + mapName + " map size: " + map.size()); + LOG.info("Updated " + mapName + " map size:" + map.size()); } catch (IOException e) { LOG.error("Can't update " + mapName + " map"); @@ -223,115 +199,20 @@ synchronized public void updateMaps() throws IOException { + " 'nobody' will be used for any user and group."); return; } - - StaticMapping staticMapping = new StaticMapping( - new HashMap(), new HashMap()); - if (staticMappingFile.exists()) { - LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping..."); - staticMapping = parseStaticMap(staticMappingFile); - } else { - LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile - + "' does not exist."); - } if (OS.startsWith("Linux")) { - updateMapInternal(uMap, "user", LINUX_GET_ALL_USERS_CMD, ":", - staticMapping.uidMapping); - updateMapInternal(gMap, "group", LINUX_GET_ALL_GROUPS_CMD, ":", - staticMapping.gidMapping); + updateMapInternal(uMap, "user", LINUX_GET_ALL_USERS_CMD, ":"); + updateMapInternal(gMap, "group", LINUX_GET_ALL_GROUPS_CMD, ":"); } else { // Mac - updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+", - staticMapping.uidMapping); - updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+", - staticMapping.gidMapping); + updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+"); + updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+"); } uidNameMap = uMap; gidNameMap = gMap; lastUpdateTime = Time.monotonicNow(); } - - @SuppressWarnings("serial") - static final class PassThroughMap extends HashMap { - - public PassThroughMap() { - this(new HashMap()); - } - - public PassThroughMap(Map mapping) { - super(); - for (Map.Entry entry : mapping.entrySet()) { - super.put(entry.getKey(), entry.getValue()); - } - } - - @SuppressWarnings("unchecked") - @Override - public K get(Object key) { - if (super.containsKey(key)) { - return super.get(key); - } else { - return (K) key; - } - } - } - - @VisibleForTesting - static final class StaticMapping { - final Map uidMapping; - final Map gidMapping; - - public StaticMapping(Map uidMapping, - Map gidMapping) { - this.uidMapping = new PassThroughMap(uidMapping); - this.gidMapping = new PassThroughMap(gidMapping); - } - } - - static StaticMapping parseStaticMap(File staticMapFile) - throws IOException { - - Map uidMapping = new HashMap(); - Map gidMapping = new HashMap(); - - BufferedReader in = new BufferedReader(new InputStreamReader( - new FileInputStream(staticMapFile))); - - try { - String line = null; - while ((line = in.readLine()) != null) { - // Skip entirely empty and comment lines. - if (EMPTY_LINE.matcher(line).matches() || - COMMENT_LINE.matcher(line).matches()) { - continue; - } - - Matcher lineMatcher = MAPPING_LINE.matcher(line); - if (!lineMatcher.matches()) { - LOG.warn("Could not parse line '" + line + "'. Lines should be of " + - "the form '[uid|gid] [remote id] [local id]'. Blank lines and " + - "everything following a '#' on a line will be ignored."); - continue; - } - - // We know the line is fine to parse without error checking like this - // since it matched the regex above. - String firstComponent = lineMatcher.group(1); - int remoteId = Integer.parseInt(lineMatcher.group(2)); - int localId = Integer.parseInt(lineMatcher.group(3)); - if (firstComponent.equals("uid")) { - uidMapping.put(localId, remoteId); - } else { - gidMapping.put(localId, remoteId); - } - } - } finally { - in.close(); - } - - return new StaticMapping(uidMapping, gidMapping); - } synchronized public int getUid(String user) throws IOException { checkAndUpdateMaps(); diff --git hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/ACCESS3Response.java hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/ACCESS3Response.java index 5720fc6..8ba07aa 100644 --- hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/ACCESS3Response.java +++ hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/ACCESS3Response.java @@ -46,12 +46,10 @@ public ACCESS3Response(int status, Nfs3FileAttributes postOpAttr, int access) { @Override public XDR writeHeaderAndResponse(XDR out, int xid, Verifier verifier) { super.writeHeaderAndResponse(out, xid, verifier); + out.writeBoolean(true); + postOpAttr.serialize(out); if (this.getStatus() == Nfs3Status.NFS3_OK) { - out.writeBoolean(true); - postOpAttr.serialize(out); out.writeInt(access); - } else { - out.writeBoolean(false); } return out; } diff --git hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java index bca53d0..77477ff 100644 --- hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java +++ hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java @@ -19,97 +19,15 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.nfs.nfs3.IdUserGroup.PassThroughMap; -import org.apache.hadoop.nfs.nfs3.IdUserGroup.StaticMapping; import org.junit.Test; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; public class TestIdUserGroup { - - private static final Map EMPTY_PASS_THROUGH_MAP = - new PassThroughMap(); - - @Test - public void testStaticMapParsing() throws IOException { - File tempStaticMapFile = File.createTempFile("nfs-", ".map"); - final String staticMapFileContents = - "uid 10 100\n" + - "gid 10 200\n" + - "uid 11 201 # comment at the end of a line\n" + - "uid 12 301\n" + - "# Comment at the beginning of a line\n" + - " # Comment that starts late in the line\n" + - "uid 10000 10001# line without whitespace before comment\n" + - "uid 13 302\n" + - "gid\t11\t201\n" + // Tabs instead of spaces. - "\n" + // Entirely empty line. - "gid 12 202"; - OutputStream out = new FileOutputStream(tempStaticMapFile); - out.write(staticMapFileContents.getBytes()); - out.close(); - StaticMapping parsedMap = IdUserGroup.parseStaticMap(tempStaticMapFile); - - assertEquals(10, (int)parsedMap.uidMapping.get(100)); - assertEquals(11, (int)parsedMap.uidMapping.get(201)); - assertEquals(12, (int)parsedMap.uidMapping.get(301)); - assertEquals(13, (int)parsedMap.uidMapping.get(302)); - assertEquals(10, (int)parsedMap.gidMapping.get(200)); - assertEquals(11, (int)parsedMap.gidMapping.get(201)); - assertEquals(12, (int)parsedMap.gidMapping.get(202)); - assertEquals(10000, (int)parsedMap.uidMapping.get(10001)); - // Ensure pass-through of unmapped IDs works. - assertEquals(1000, (int)parsedMap.uidMapping.get(1000)); - } - - @Test - public void testStaticMapping() throws IOException { - Map uidStaticMap = new PassThroughMap(); - Map gidStaticMap = new PassThroughMap(); - - uidStaticMap.put(11501, 10); - gidStaticMap.put(497, 200); - - // Maps for id to name map - BiMap uMap = HashBiMap.create(); - BiMap gMap = HashBiMap.create(); - - String GET_ALL_USERS_CMD = - "echo \"atm:x:1000:1000:Aaron T. Myers,,,:/home/atm:/bin/bash\n" - + "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\"" - + " | cut -d: -f1,3"; - - String GET_ALL_GROUPS_CMD = "echo \"hdfs:*:11501:hrt_hdfs\n" - + "mapred:x:497\n" - + "mapred2:x:498\"" - + " | cut -d: -f1,3"; - - IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":", - uidStaticMap); - IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":", - gidStaticMap); - - assertEquals("hdfs", uMap.get(10)); - assertEquals(10, (int)uMap.inverse().get("hdfs")); - assertEquals("atm", uMap.get(1000)); - assertEquals(1000, (int)uMap.inverse().get("atm")); - - assertEquals("hdfs", gMap.get(11501)); - assertEquals(11501, (int)gMap.inverse().get("hdfs")); - assertEquals("mapred", gMap.get(200)); - assertEquals(200, (int)gMap.inverse().get("mapred")); - assertEquals("mapred2", gMap.get(498)); - assertEquals(498, (int)gMap.inverse().get("mapred2")); - } @Test public void testDuplicates() throws IOException { @@ -133,17 +51,15 @@ public void testDuplicates() throws IOException { BiMap uMap = HashBiMap.create(); BiMap gMap = HashBiMap.create(); - IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":", - EMPTY_PASS_THROUGH_MAP); - assertEquals(5, uMap.size()); + IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); + assertTrue(uMap.size() == 5); assertEquals("root", uMap.get(0)); assertEquals("hdfs", uMap.get(11501)); assertEquals("hdfs2",uMap.get(11502)); assertEquals("bin", uMap.get(2)); assertEquals("daemon", uMap.get(1)); - IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":", - EMPTY_PASS_THROUGH_MAP); + IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); assertTrue(gMap.size() == 3); assertEquals("hdfs",gMap.get(11501)); assertEquals("mapred", gMap.get(497)); @@ -174,8 +90,7 @@ public void testIdOutOfIntegerRange() throws IOException { BiMap uMap = HashBiMap.create(); BiMap gMap = HashBiMap.create(); - IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":", - EMPTY_PASS_THROUGH_MAP); + IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); assertTrue(uMap.size() == 7); assertEquals("nfsnobody", uMap.get(-2)); assertEquals("nfsnobody1", uMap.get(-1)); @@ -185,8 +100,7 @@ public void testIdOutOfIntegerRange() throws IOException { assertEquals("hdfs",uMap.get(11501)); assertEquals("daemon", uMap.get(2)); - IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":", - EMPTY_PASS_THROUGH_MAP); + IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); assertTrue(gMap.size() == 7); assertEquals("hdfs",gMap.get(11501)); assertEquals("rpcuser", gMap.get(29)); diff --git hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java index ae97ffe..95dacd8 100644 --- hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java +++ hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java @@ -54,7 +54,6 @@ import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.VerifierNone; import org.apache.hadoop.util.Daemon; -import org.apache.hadoop.util.Time; import org.jboss.netty.channel.Channel; import com.google.common.annotations.VisibleForTesting; @@ -137,7 +136,7 @@ long getStartTime() { this.channel = channel; this.xid = xid; this.preOpAttr = preOpAttr; - this.startTime = Time.monotonicNow(); + this.startTime = System.currentTimeMillis(); } @Override @@ -159,11 +158,11 @@ public String toString() { private Daemon dumpThread; private void updateLastAccessTime() { - lastAccessTime = Time.monotonicNow(); + lastAccessTime = System.currentTimeMillis(); } private boolean checkStreamTimeout(long streamTimeout) { - return Time.monotonicNow() - lastAccessTime > streamTimeout; + return System.currentTimeMillis() - lastAccessTime > streamTimeout; } long getLastAccessTime() { @@ -697,7 +696,7 @@ private WRITE3Response processPerfectOverWrite(DFSClient dfsClient, + " updating the mtime, then return success"); Nfs3FileAttributes postOpAttr = null; try { - dfsClient.setTimes(path, Time.monotonicNow(), -1); + dfsClient.setTimes(path, System.currentTimeMillis(), -1); postOpAttr = Nfs3Utils.getFileAttr(dfsClient, path, iug); } catch (IOException e) { LOG.info("Got error when processing perfect overwrite, path=" + path @@ -998,7 +997,7 @@ private void processCommits(long offset) { if (LOG.isDebugEnabled()) { LOG.debug("FileId: " + latestAttr.getFileId() + " Service time:" - + (Time.monotonicNow() - commit.getStartTime()) + + (System.currentTimeMillis() - commit.getStartTime()) + "ms. Sent response for commit:" + commit); } entry = pendingCommits.firstEntry(); diff --git hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java index 1c06cf4..d255e3d 100644 --- hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java +++ hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java @@ -28,7 +28,6 @@ import org.apache.hadoop.nfs.nfs3.FileHandle; import org.apache.hadoop.nfs.nfs3.Nfs3Constant; import org.apache.hadoop.util.Daemon; -import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -100,7 +99,7 @@ LOG.warn("No eviction candidate. All streams have pending work."); return null; } else { - long idleTime = Time.monotonicNow() + long idleTime = System.currentTimeMillis() - idlest.getValue().getLastAccessTime(); if (idleTime < Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT) { if (LOG.isDebugEnabled()) { @@ -251,7 +250,7 @@ public void run() { // Check if it can sleep try { - long workedTime = Time.monotonicNow() - lastWakeupTime; + long workedTime = System.currentTimeMillis() - lastWakeupTime; if (workedTime < rotation) { if (LOG.isTraceEnabled()) { LOG.trace("StreamMonitor can still have a sleep:" @@ -259,7 +258,7 @@ public void run() { } Thread.sleep(rotation - workedTime); } - lastWakeupTime = Time.monotonicNow(); + lastWakeupTime = System.currentTimeMillis(); } catch (InterruptedException e) { LOG.info("StreamMonitor got interrupted"); @@ -268,4 +267,4 @@ public void run() { } } } -} +} \ No newline at end of file diff --git hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index 62aa8cb..f78598f 100644 --- hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -124,7 +124,6 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authorize.AuthorizationException; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.channel.Channel; @@ -271,17 +270,6 @@ public GETATTR3Response getattr(XDR xdr, SecurityHandler securityHandler, Nfs3FileAttributes attrs = null; try { attrs = writeManager.getFileAttr(dfsClient, handle, iug); - } catch (RemoteException r) { - LOG.warn("Exception ", r); - IOException io = r.unwrapRemoteException(); - /** - * AuthorizationException can be thrown if the user can't be proxy'ed. - */ - if (io instanceof AuthorizationException) { - return new GETATTR3Response(Nfs3Status.NFS3ERR_ACCES); - } else { - return new GETATTR3Response(Nfs3Status.NFS3ERR_IO); - } } catch (IOException e) { LOG.info("Can't get file attribute, fileId=" + handle.getFileId(), e); response.setStatus(Nfs3Status.NFS3ERR_IO); @@ -511,17 +499,6 @@ public ACCESS3Response access(XDR xdr, SecurityHandler securityHandler, securityHandler.getUid(), securityHandler.getGid(), attrs); return new ACCESS3Response(Nfs3Status.NFS3_OK, attrs, access); - } catch (RemoteException r) { - LOG.warn("Exception ", r); - IOException io = r.unwrapRemoteException(); - /** - * AuthorizationException can be thrown if the user can't be proxy'ed. - */ - if (io instanceof AuthorizationException) { - return new ACCESS3Response(Nfs3Status.NFS3ERR_ACCES); - } else { - return new ACCESS3Response(Nfs3Status.NFS3ERR_IO); - } } catch (IOException e) { LOG.warn("Exception ", e); return new ACCESS3Response(Nfs3Status.NFS3ERR_IO); @@ -1703,17 +1680,6 @@ public FSSTAT3Response fsstat(XDR xdr, SecurityHandler securityHandler, return new FSSTAT3Response(Nfs3Status.NFS3_OK, attrs, totalBytes, freeBytes, freeBytes, maxFsObjects, maxFsObjects, maxFsObjects, 0); - } catch (RemoteException r) { - LOG.warn("Exception ", r); - IOException io = r.unwrapRemoteException(); - /** - * AuthorizationException can be thrown if the user can't be proxy'ed. - */ - if (io instanceof AuthorizationException) { - return new FSSTAT3Response(Nfs3Status.NFS3ERR_ACCES); - } else { - return new FSSTAT3Response(Nfs3Status.NFS3ERR_IO); - } } catch (IOException e) { LOG.warn("Exception ", e); return new FSSTAT3Response(Nfs3Status.NFS3ERR_IO); diff --git hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5814821..1c04100 100644 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -317,9 +317,6 @@ Trunk (Unreleased) HDFS-6374. setXAttr should require the user to be the owner of the file or directory (Charles Lamb via wang) - HDFS-6110 adding more slow action log in critical write path - (Liang Xie via stack) - Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -440,15 +437,6 @@ Release 2.5.0 - UNRELEASED HDFS-6432. Add snapshot related APIs to webhdfs. (jing9) - HDFS-6396. Remove support for ACL feature from INodeSymlink. - (Charles Lamb via wang) - - HDFS-6435. Add support for specifying a static uid/gid mapping for the NFS - gateway. (atm via wang) - - HDFS-6416. Use Time#monotonicNow in OpenFileCtx and OpenFileCtxCatch to - avoid system clock bugs (Abhiraj Butala via brandonli) - OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) @@ -581,23 +569,6 @@ Release 2.5.0 - UNRELEASED HDFS-6419. TestBookKeeperHACheckpoints#TestSBNCheckpoints fails on trunk. (Akira AJISAKA via kihwal) - HDFS-6409. Fix typo in log message about NameNode layout version upgrade. - (Chen He via cnauroth) - - HDFS-6433. Replace BytesMoved class with AtomicLong. - (Benoy Antony via cnauroth) - - HDFS-6438. DeleteSnapshot should be a DELETE request in WebHdfs. (jing9) - - HDFS-6423. Diskspace quota usage should be updated when appending data to - partial block. (jing9) - - HDFS-6443. Fix MiniQJMHACluster related test failures. (Zesheng Wu via - Arpit Agarwal) - - HDFS-6227. ShortCircuitCache#unref should purge ShortCircuitReplicas whose - streams have been closed by java interrupts. (Colin Patrick McCabe via jing9) - Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES @@ -681,9 +652,6 @@ Release 2.4.1 - UNRELEASED HDFS-6397. NN shows inconsistent value in deadnode count. (Mohammad Kamrul Islam via kihwal) - HDFS-6411. nfs-hdfs-gateway mount raises I/O error and hangs when a - unauthorized user attempts to access it (brandonli) - Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index e308a96..4f51fc1 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -116,7 +116,7 @@ public boolean delete(Path f, boolean recursive) @Override public FileChecksum getFileChecksum(Path f) throws IOException, UnresolvedLinkException { - return dfs.getFileChecksum(getUriPath(f), Long.MAX_VALUE); + return dfs.getFileChecksum(getUriPath(f)); } @Override diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 4d2da79..38eca80 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -276,7 +276,6 @@ final int retryTimesForGetLastBlockLength; final int retryIntervalForGetLastBlockLength; final long datanodeRestartTimeout; - final long dfsclientSlowIoWarningThresholdMs; final boolean useLegacyBlockReader; final boolean useLegacyBlockReaderLocal; @@ -431,9 +430,6 @@ public Conf(Configuration conf) { datanodeRestartTimeout = conf.getLong( DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY, DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT) * 1000; - dfsclientSlowIoWarningThresholdMs = conf.getLong( - DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY, - DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT); } public boolean isUseLegacyBlockReaderLocal() { @@ -1805,19 +1801,15 @@ public HdfsFileStatus getFileLinkInfo(String src) throws IOException { } /** - * Get the checksum of the whole file of a range of the file. Note that the - * range always starts from the beginning of the file. + * Get the checksum of a file. * @param src The file path - * @param length The length of the range * @return The checksum * @see DistributedFileSystem#getFileChecksum(Path) */ - public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length) - throws IOException { + public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException { checkOpen(); - Preconditions.checkArgument(length >= 0); - return getFileChecksum(src, length, clientName, namenode, - socketFactory, dfsClientConf.socketTimeout, getDataEncryptionKey(), + return getFileChecksum(src, clientName, namenode, socketFactory, + dfsClientConf.socketTimeout, getDataEncryptionKey(), dfsClientConf.connectToDnViaHostname); } @@ -1858,9 +1850,8 @@ public DataEncryptionKey getDataEncryptionKey() } /** - * Get the checksum of the whole file or a range of the file. + * Get the checksum of a file. * @param src The file path - * @param length the length of the range, i.e., the range is [0, length] * @param clientName the name of the client requesting the checksum. * @param namenode the RPC proxy for the namenode * @param socketFactory to create sockets to connect to DNs @@ -1870,13 +1861,12 @@ public DataEncryptionKey getDataEncryptionKey() * @return The checksum */ private static MD5MD5CRC32FileChecksum getFileChecksum(String src, - long length, String clientName, ClientProtocol namenode, - SocketFactory socketFactory, int socketTimeout, + String clientName, + ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout, DataEncryptionKey encryptionKey, boolean connectToDnViaHostname) throws IOException { - //get block locations for the file range - LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, - length); + //get all block locations + LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE); if (null == blockLocations) { throw new FileNotFoundException("File does not exist: " + src); } @@ -1888,11 +1878,10 @@ private static MD5MD5CRC32FileChecksum getFileChecksum(String src, boolean refetchBlocks = false; int lastRetriedIndex = -1; - // get block checksum for each block - long remaining = length; - for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) { + //get block checksum for each block + for(int i = 0; i < locatedblocks.size(); i++) { if (refetchBlocks) { // refetch to get fresh tokens - blockLocations = callGetBlockLocations(namenode, src, 0, length); + blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE); if (null == blockLocations) { throw new FileNotFoundException("File does not exist: " + src); } @@ -1901,10 +1890,6 @@ private static MD5MD5CRC32FileChecksum getFileChecksum(String src, } LocatedBlock lb = locatedblocks.get(i); final ExtendedBlock block = lb.getBlock(); - if (remaining < block.getNumBytes()) { - block.setNumBytes(remaining); - } - remaining -= block.getNumBytes(); final DatanodeInfo[] datanodes = lb.getLocations(); //try each datanode location of the block diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 5835f21..e9a4358 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -645,12 +645,5 @@ public static final int DFS_NFS_REGISTRATION_PORT_DEFAULT = 40; // Currently unassigned. public static final String DFS_NFS_ALLOW_INSECURE_PORTS_KEY = "dfs.nfs.allow.insecure.ports"; public static final boolean DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT = true; - - // Slow io warning log threshold settings for dfsclient and datanode. - public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY = - "dfs.client.slow.io.warning.threshold.ms"; - public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000; - public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY = - "dfs.datanode.slow.io.warning.threshold.ms"; - public static final long DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 300; + } diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 8ae750d..e2bc316 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -122,7 +122,6 @@ implements Syncable, CanSetDropBehind { private static final int MAX_PACKETS = 80; // each packet 64K, total 5MB private final DFSClient dfsClient; - private final long dfsclientSlowLogThresholdMs; private Socket s; // closed is accessed by different threads under different locks. private volatile boolean closed = false; @@ -789,19 +788,11 @@ public void run() { // process responses from datanodes. try { // read an ack from the pipeline - long begin = Time.monotonicNow(); ack.readFields(blockReplyStream); - long duration = Time.monotonicNow() - begin; - if (duration > dfsclientSlowLogThresholdMs - && ack.getSeqno() != Packet.HEART_BEAT_SEQNO) { - DFSClient.LOG - .warn("Slow ReadProcessor read fields took " + duration - + "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms); ack: " - + ack + ", targets: " + Arrays.asList(targets)); - } else if (DFSClient.LOG.isDebugEnabled()) { + if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("DFSClient " + ack); } - + long seqno = ack.getSeqno(); // processes response status from datanodes. for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) { @@ -1579,8 +1570,6 @@ private DFSOutputStream(DFSClient dfsClient, String src, Progressable progress, } this.checksum = checksum; - this.dfsclientSlowLogThresholdMs = - dfsClient.getConf().dfsclientSlowIoWarningThresholdMs; } /** Construct a new output stream for creating a file. */ @@ -2012,7 +2001,6 @@ private void waitForAckedSeqno(long seqno) throws IOException { if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Waiting for ack for: " + seqno); } - long begin = Time.monotonicNow(); try { synchronized (dataQueue) { while (!closed) { @@ -2032,11 +2020,6 @@ private void waitForAckedSeqno(long seqno) throws IOException { checkClosed(); } catch (ClosedChannelException e) { } - long duration = Time.monotonicNow() - begin; - if (duration > dfsclientSlowLogThresholdMs) { - DFSClient.LOG.warn("Slow waitForAckedSeqno took " + duration - + "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms)"); - } } private synchronized void start() { diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 1f57727..38bb95a 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -68,12 +68,14 @@ import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; @@ -83,6 +85,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; @@ -1139,7 +1142,7 @@ public FileChecksum getFileChecksum(Path f) throws IOException { @Override public FileChecksum doCall(final Path p) throws IOException, UnresolvedLinkException { - return dfs.getFileChecksum(getPathName(p), Long.MAX_VALUE); + return dfs.getFileChecksum(getPathName(p)); } @Override @@ -1151,32 +1154,6 @@ public FileChecksum next(final FileSystem fs, final Path p) } @Override - public FileChecksum getFileChecksum(Path f, final long length) - throws IOException { - statistics.incrementReadOps(1); - Path absF = fixRelativePart(f); - return new FileSystemLinkResolver() { - @Override - public FileChecksum doCall(final Path p) - throws IOException, UnresolvedLinkException { - return dfs.getFileChecksum(getPathName(p), length); - } - - @Override - public FileChecksum next(final FileSystem fs, final Path p) - throws IOException { - if (fs instanceof DistributedFileSystem) { - return ((DistributedFileSystem) fs).getFileChecksum(p, length); - } else { - throw new UnsupportedFileSystemException( - "getFileChecksum(Path, long) is not supported by " - + fs.getClass().getSimpleName()); - } - } - }.resolve(this, absF); - } - - @Override public void setPermission(Path p, final FsPermission permission ) throws IOException { statistics.incrementWriteOps(1); diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index f6e05a0..f1c9fb3 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -47,7 +47,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -366,7 +365,7 @@ private void dispatch() { sendRequest(out); receiveResponse(in); - bytesMoved.addAndGet(block.getNumBytes()); + bytesMoved.inc(block.getNumBytes()); LOG.info("Successfully moved " + this); } catch (IOException e) { LOG.warn("Failed to move " + this + ": " + e.getMessage()); @@ -1112,7 +1111,17 @@ C chooseCandidate(D dn, Iterator candidates, Matcher matcher) { return null; } - private final AtomicLong bytesMoved = new AtomicLong(); + private static class BytesMoved { + private long bytesMoved = 0L;; + private synchronized void inc( long bytes ) { + bytesMoved += bytes; + } + + private synchronized long get() { + return bytesMoved; + } + }; + private final BytesMoved bytesMoved = new BytesMoved(); /* Start a thread to dispatch block moves for each source. * The thread selects blocks to move & sends request to proxy source to diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index e36d40c..a7643b5 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -69,7 +69,7 @@ @VisibleForTesting static long CACHE_DROP_LAG_BYTES = 8 * 1024 * 1024; - private final long datanodeSlowLogThresholdMs; + private DataInputStream in = null; // from where data are read private DataChecksum clientChecksum; // checksum used by client private DataChecksum diskChecksum; // checksum we write to disk @@ -140,7 +140,7 @@ this.isDatanode = clientname.length() == 0; this.isClient = !this.isDatanode; this.restartBudget = datanode.getDnConf().restartReplicaExpiry; - this.datanodeSlowLogThresholdMs = datanode.getDnConf().datanodeSlowIoWarningThresholdMs; + //for datanode, we have //1: clientName.length() == 0, and //2: stage == null or PIPELINE_SETUP_CREATE @@ -335,7 +335,6 @@ public void close() throws IOException { */ void flushOrSync(boolean isSync) throws IOException { long flushTotalNanos = 0; - long begin = Time.monotonicNow(); if (checksumOut != null) { long flushStartNanos = System.nanoTime(); checksumOut.flush(); @@ -364,12 +363,6 @@ void flushOrSync(boolean isSync) throws IOException { datanode.metrics.incrFsyncCount(); } } - long duration = Time.monotonicNow() - begin; - if (duration > datanodeSlowLogThresholdMs) { - LOG.warn("Slow flushOrSync took " + duration + "ms (threshold=" - + datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos=" - + flushTotalNanos + "ns"); - } } /** @@ -495,14 +488,8 @@ private int receivePacket() throws IOException { //First write the packet to the mirror: if (mirrorOut != null && !mirrorError) { try { - long begin = Time.monotonicNow(); packetReceiver.mirrorPacketTo(mirrorOut); mirrorOut.flush(); - long duration = Time.monotonicNow() - begin; - if (duration > datanodeSlowLogThresholdMs) { - LOG.warn("Slow BlockReceiver write packet to mirror took " + duration - + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); - } } catch (IOException e) { handleMirrorOutError(e); } @@ -585,13 +572,7 @@ private int receivePacket() throws IOException { int numBytesToDisk = (int)(offsetInBlock-onDiskLen); // Write data to disk. - long begin = Time.monotonicNow(); out.write(dataBuf.array(), startByteToDisk, numBytesToDisk); - long duration = Time.monotonicNow() - begin; - if (duration > datanodeSlowLogThresholdMs) { - LOG.warn("Slow BlockReceiver write data to disk cost:" + duration - + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); - } // If this is a partial chunk, then verify that this is the only // chunk in the packet. Calculate new crc for this chunk. @@ -657,7 +638,6 @@ private void manageWriterOsCache(long offsetInBlock) { try { if (outFd != null && offsetInBlock > lastCacheManagementOffset + CACHE_DROP_LAG_BYTES) { - long begin = Time.monotonicNow(); // // For SYNC_FILE_RANGE_WRITE, we want to sync from // lastCacheManagementOffset to a position "two windows ago" @@ -690,11 +670,6 @@ private void manageWriterOsCache(long offsetInBlock) { NativeIO.POSIX.POSIX_FADV_DONTNEED); } lastCacheManagementOffset = offsetInBlock; - long duration = Time.monotonicNow() - begin; - if (duration > datanodeSlowLogThresholdMs) { - LOG.warn("Slow manageWriterOsCache took " + duration - + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); - } } } catch (Throwable t) { LOG.warn("Error managing cache for writer of block " + block, t); @@ -1324,15 +1299,9 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno, replicaInfo.setBytesAcked(offsetInBlock); } // send my ack back to upstream datanode - long begin = Time.monotonicNow(); replyAck.write(upstreamOut); upstreamOut.flush(); - long duration = Time.monotonicNow() - begin; - if (duration > datanodeSlowLogThresholdMs) { - LOG.warn("Slow PacketResponder send ack to upstream took " + duration - + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + myString - + ", replyAck=" + replyAck); - } else if (LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug(myString + ", replyAck=" + replyAck); } diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index 27dba1b..d72c3b6 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -79,8 +79,6 @@ final long deleteReportInterval; final long initialBlockReportDelay; final long cacheReportInterval; - final long dfsclientSlowIoWarningThresholdMs; - final long datanodeSlowIoWarningThresholdMs; final int writePacketSize; final String minimumNameNodeVersion; @@ -131,14 +129,7 @@ public DNConf(Configuration conf) { DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT); this.cacheReportInterval = conf.getLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT); - - this.dfsclientSlowIoWarningThresholdMs = conf.getLong( - DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY, - DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT); - this.datanodeSlowIoWarningThresholdMs = conf.getLong( - DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY, - DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT); - + long initBRDelay = conf.getLong( DFS_BLOCKREPORT_INITIAL_DELAY_KEY, DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT) * 1000L; @@ -177,7 +168,7 @@ public DNConf(Configuration conf) { DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY, DFS_DATANODE_RESTART_REPLICA_EXPIRY_DEFAULT) * 1000L; } - + // We get minimumNameNodeVersion via a method so it can be mocked out in tests. String getMinimumNameNodeVersion() { return this.minimumNameNodeVersion; diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index a118fcb..a0de9fa 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -42,7 +42,6 @@ import java.net.SocketException; import java.net.UnknownHostException; import java.nio.channels.ClosedChannelException; -import java.security.MessageDigest; import java.util.Arrays; import org.apache.commons.logging.Log; @@ -84,7 +83,6 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DataChecksum; -import com.google.common.base.Preconditions; import com.google.common.net.InetAddresses; import com.google.protobuf.ByteString; @@ -804,44 +802,7 @@ public void transferBlock(final ExtendedBlock blk, IOUtils.closeStream(out); } } - - private MD5Hash calcPartialBlockChecksum(ExtendedBlock block, - long requestLength, DataChecksum checksum, DataInputStream checksumIn) - throws IOException { - final int bytesPerCRC = checksum.getBytesPerChecksum(); - final int csize = checksum.getChecksumSize(); - final byte[] buffer = new byte[4*1024]; - MessageDigest digester = MD5Hash.getDigester(); - - long remaining = requestLength / bytesPerCRC * csize; - for (int toDigest = 0; remaining > 0; remaining -= toDigest) { - toDigest = checksumIn.read(buffer, 0, - (int) Math.min(remaining, buffer.length)); - if (toDigest < 0) { - break; - } - digester.update(buffer, 0, toDigest); - } - - int partialLength = (int) (requestLength % bytesPerCRC); - if (partialLength > 0) { - byte[] buf = new byte[partialLength]; - final InputStream blockIn = datanode.data.getBlockInputStream(block, - requestLength - partialLength); - try { - // Get the CRC of the partialLength. - IOUtils.readFully(blockIn, buf, 0, partialLength); - } finally { - IOUtils.closeStream(blockIn); - } - checksum.update(buf, 0, partialLength); - byte[] partialCrc = new byte[csize]; - checksum.writeValue(partialCrc, 0, true); - digester.update(partialCrc); - } - return new MD5Hash(digester.digest()); - } - + @Override public void blockChecksum(final ExtendedBlock block, final Token blockToken) throws IOException { @@ -849,32 +810,25 @@ public void blockChecksum(final ExtendedBlock block, getOutputStream()); checkAccess(out, true, block, blockToken, Op.BLOCK_CHECKSUM, BlockTokenSecretManager.AccessMode.READ); - // client side now can specify a range of the block for checksum - long requestLength = block.getNumBytes(); - Preconditions.checkArgument(requestLength >= 0); - long visibleLength = datanode.data.getReplicaVisibleLength(block); - boolean partialBlk = requestLength < visibleLength; - updateCurrentThreadName("Reading metadata for block " + block); - final LengthInputStream metadataIn = datanode.data - .getMetaDataInputStream(block); - - final DataInputStream checksumIn = new DataInputStream( - new BufferedInputStream(metadataIn, HdfsConstants.IO_FILE_BUFFER_SIZE)); + final LengthInputStream metadataIn = + datanode.data.getMetaDataInputStream(block); + final DataInputStream checksumIn = new DataInputStream(new BufferedInputStream( + metadataIn, HdfsConstants.IO_FILE_BUFFER_SIZE)); + updateCurrentThreadName("Getting checksum for block " + block); try { //read metadata file - final BlockMetadataHeader header = BlockMetadataHeader - .readHeader(checksumIn); - final DataChecksum checksum = header.getChecksum(); - final int csize = checksum.getChecksumSize(); + final BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn); + final DataChecksum checksum = header.getChecksum(); final int bytesPerCRC = checksum.getBytesPerChecksum(); - final long crcPerBlock = csize <= 0 ? 0 : - (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize()) / csize; + final long crcPerBlock = checksum.getChecksumSize() > 0 + ? (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize())/checksum.getChecksumSize() + : 0; + + //compute block checksum + final MD5Hash md5 = MD5Hash.digest(checksumIn); - final MD5Hash md5 = partialBlk && crcPerBlock > 0 ? - calcPartialBlockChecksum(block, requestLength, checksum, checksumIn) - : MD5Hash.digest(checksumIn); if (LOG.isDebugEnabled()) { LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC + ", crcPerBlock=" + crcPerBlock + ", md5=" + md5); @@ -887,7 +841,8 @@ public void blockChecksum(final ExtendedBlock block, .setBytesPerCrc(bytesPerCRC) .setCrcPerBlock(crcPerBlock) .setMd5(ByteString.copyFrom(md5.getDigest())) - .setCrcType(PBHelper.convert(checksum.getChecksumType()))) + .setCrcType(PBHelper.convert(checksum.getChecksumType())) + ) .build() .writeDelimitedTo(out); out.flush(); diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java index 83de6eb..fb144d3 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hdfs.web.resources.ReplicationParam; import org.apache.hadoop.hdfs.web.resources.UriFsPathParam; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -451,7 +452,7 @@ private Response get( MD5MD5CRC32FileChecksum checksum = null; DFSClient dfsclient = newDfsClient(nnId, conf); try { - checksum = dfsclient.getFileChecksum(fullpath, Long.MAX_VALUE); + checksum = dfsclient.getFileChecksum(fullpath); dfsclient.close(); dfsclient = null; } finally { diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index b163579..36028a0 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -227,7 +227,7 @@ boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target, + HdfsConstants.NAMENODE_LAYOUT_VERSION + " is required.\n" + "Please restart NameNode with the \"" + RollingUpgradeStartupOption.STARTED.getOptionString() - + "\" option if a rolling upgrade is already started;" + + "\" option if a rolling upgraded is already started;" + " or restart NameNode with the \"" + StartupOption.UPGRADE.getName() + "\" option to start" + " a new upgrade."); diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ce8cbee..6a9825a 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2424,12 +2424,6 @@ LocatedBlock prepareFileForWrite(String src, INodeFile file, .getClientName(), src); LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(cons); - if (ret != null) { - // update the quota: use the preferred block size for UC block - final long diff = file.getPreferredBlockSize() - ret.getBlockSize(); - dir.updateSpaceConsumed(src, 0, diff); - } - if (writeToEditLog) { getEditLog().logOpenFile(src, cons, logRetryCache); } diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java index deb3ada..288c809 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; -import org.apache.hadoop.hdfs.server.namenode.AclFeature; import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; /** @@ -113,25 +112,6 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix, out.println(); } - /** - * getAclFeature is not overridden because it is needed for resolving - * symlinks. - @Override - final AclFeature getAclFeature(int snapshotId) { - throw new UnsupportedOperationException("ACLs are not supported on symlinks"); - } - */ - - @Override - public void removeAclFeature() { - throw new UnsupportedOperationException("ACLs are not supported on symlinks"); - } - - @Override - public void addAclFeature(AclFeature f) { - throw new UnsupportedOperationException("ACLs are not supported on symlinks"); - } - @Override final XAttrFeature getXAttrFeature(int snapshotId) { throw new UnsupportedOperationException("XAttrs are not supported on symlinks"); diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 0ffb813..0daaa35 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -590,6 +590,10 @@ private Response put( org.apache.hadoop.fs.Path.class.getSimpleName(), snapshotPath); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } + case DELETESNAPSHOT: { + np.deleteSnapshot(fullpath, snapshotName.getValue()); + return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); + } case RENAMESNAPSHOT: { np.renameSnapshot(fullpath, oldSnapshotName.getValue(), snapshotName.getValue()); @@ -949,12 +953,9 @@ public Response deleteRoot( @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT) final DeleteOpParam op, @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT) - final RecursiveParam recursive, - @QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT) - final SnapshotNameParam snapshotName + final RecursiveParam recursive ) throws IOException, InterruptedException { - return delete(ugi, delegation, username, doAsUser, ROOT, op, recursive, - snapshotName); + return delete(ugi, delegation, username, doAsUser, ROOT, op, recursive); } /** Handle HTTP DELETE request. */ @@ -973,19 +974,17 @@ public Response delete( @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT) final DeleteOpParam op, @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT) - final RecursiveParam recursive, - @QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT) - final SnapshotNameParam snapshotName + final RecursiveParam recursive ) throws IOException, InterruptedException { - init(ugi, delegation, username, doAsUser, path, op, recursive, snapshotName); + init(ugi, delegation, username, doAsUser, path, op, recursive); return ugi.doAs(new PrivilegedExceptionAction() { @Override public Response run() throws IOException { try { return delete(ugi, delegation, username, doAsUser, - path.getAbsolutePath(), op, recursive, snapshotName); + path.getAbsolutePath(), op, recursive); } finally { reset(); } @@ -1000,22 +999,17 @@ private Response delete( final DoAsParam doAsUser, final String fullpath, final DeleteOpParam op, - final RecursiveParam recursive, - final SnapshotNameParam snapshotName + final RecursiveParam recursive ) throws IOException { final NameNode namenode = (NameNode)context.getAttribute("name.node"); - final NamenodeProtocols np = getRPCServer(namenode); switch(op.getValue()) { - case DELETE: { - final boolean b = np.delete(fullpath, recursive.getValue()); + case DELETE: + { + final boolean b = getRPCServer(namenode).delete(fullpath, recursive.getValue()); final String js = JsonUtil.toJsonString("boolean", b); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } - case DELETESNAPSHOT: { - np.deleteSnapshot(fullpath, snapshotName.getValue()); - return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); - } default: throw new UnsupportedOperationException(op + " is not supported"); } diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index 7df340a..1e7245d 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -437,22 +437,11 @@ private void ref(ShortCircuitReplica replica) { void unref(ShortCircuitReplica replica) { lock.lock(); try { - // If the replica is stale or unusable, but we haven't purged it yet, - // let's do that. It would be a shame to evict a non-stale replica so - // that we could put a stale or unusable one into the cache. - if (!replica.purged) { - String purgeReason = null; - if (!replica.getDataStream().getChannel().isOpen()) { - purgeReason = "purging replica because its data channel is closed."; - } else if (!replica.getMetaStream().getChannel().isOpen()) { - purgeReason = "purging replica because its meta channel is closed."; - } else if (replica.isStale()) { - purgeReason = "purging replica because it is stale."; - } - if (purgeReason != null) { - LOG.debug(this + ": " + purgeReason); - purge(replica); - } + // If the replica is stale, but we haven't purged it yet, let's do that. + // It would be a shame to evict a non-stale replica so that we could put + // a stale one into the cache. + if ((!replica.purged) && replica.isStale()) { + purge(replica); } String addedString = ""; boolean shouldTrimEvictionMaps = false; diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index bc791c6..522bbf8 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -957,7 +957,7 @@ Path decodeResponse(Map json) { public void deleteSnapshot(final Path path, final String snapshotName) throws IOException { statistics.incrementWriteOps(1); - final HttpOpParam.Op op = DeleteOpParam.Op.DELETESNAPSHOT; + final HttpOpParam.Op op = PutOpParam.Op.DELETESNAPSHOT; new FsPathRunner(op, path, new SnapshotNameParam(snapshotName)).run(); } diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java index 65275e0..710e2e8 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java @@ -24,7 +24,6 @@ /** Delete operations. */ public static enum Op implements HttpOpParam.Op { DELETE(HttpURLConnection.HTTP_OK), - DELETESNAPSHOT(HttpURLConnection.HTTP_OK), NULL(HttpURLConnection.HTTP_NOT_IMPLEMENTED); diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java index 7fd2b71..64b7f8c 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java +++ hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java @@ -47,6 +47,7 @@ REMOVEXATTR(false, HttpURLConnection.HTTP_OK), CREATESNAPSHOT(false, HttpURLConnection.HTTP_OK), + DELETESNAPSHOT(false, HttpURLConnection.HTTP_OK), RENAMESNAPSHOT(false, HttpURLConnection.HTTP_OK), NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED); diff --git hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 34b36a5..e1ca17d 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1930,22 +1930,4 @@ - - dfs.client.slow.io.warning.threshold.ms - 30000 - The threshold in milliseconds at which we will log a slow - io warning in a dfsclient. By default, this parameter is set to 30000 - milliseconds (30 seconds). - - - - - dfs.datanode.slow.io.warning.threshold.ms - 300 - The threshold in milliseconds at which we will log a slow - io warning in a datanode. By default, this parameter is set to 300 - milliseconds. - - - diff --git hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm index 7f8e821..8f5bef6 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm +++ hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm @@ -338,21 +338,8 @@ HDFS NFS Gateway The system administrator must ensure that the user on NFS client host has the same name and UID as that on the NFS gateway host. This is usually not a problem if the same user management system (e.g., LDAP/NIS) is used to create and deploy users on - HDFS nodes and NFS client node. In case the user account is created manually on different hosts, one might need to + HDFS nodes and NFS client node. In case the user account is created manually in different hosts, one might need to modify UID (e.g., do "usermod -u 123 myusername") on either NFS client or NFS gateway host in order to make it the same on both sides. More technical details of RPC AUTH_UNIX can be found in {{{http://tools.ietf.org/html/rfc1057}RPC specification}}. - Optionally, the system administrator can configure a custom static mapping - file in the event one wishes to access the HDFS NFS Gateway from a system with - a completely disparate set of UIDs/GIDs. By default this file is located at - "/etc/nfs.map", but a custom location can be configured by setting the - "dfs.nfs.static.mapping.file" property to the path of the static mapping file. - The format of the static mapping file is similar to what is described in the - exports(5) manual page, but roughly it is: - -------------------------- -# Mapping for clients accessing the NFS gateway -uid 10 100 # Map the remote UID 10 the local UID 100 -gid 11 101 # Map the remote GID 11 to the local GID 101 -------------------------- diff --git hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm index 94d5d0b..48bcca3 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm +++ hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm @@ -102,12 +102,6 @@ WebHDFS REST API * {{{Cancel Delegation Token}<<>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.cancelDelegationToken) - * {{{Create Snapshot}<<>>}} - (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.createSnapshot) - - * {{{Rename Snapshot}<<>>}} - (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.renameSnapshot) - * HTTP POST * {{{Append to a File}<<>>}} @@ -120,9 +114,6 @@ WebHDFS REST API * {{{Delete a File/Directory}<<>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.delete) - - * {{{Delete Snapshot}<<>>}} - (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.deleteSnapshot) ** {FileSystem URIs vs HTTP URLs} @@ -909,75 +900,6 @@ Transfer-Encoding: chunked {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.getAclStatus -* {Snapshot Operations} - -** {Create Snapshot} - - * Submit a HTTP PUT request. - -+--------------------------------- -curl -i -X PUT "http://:/webhdfs/v1/?op=CREATESNAPSHOT[&snapshotname=]" -+--------------------------------- - - The client receives a response with a {{{Path JSON Schema}<<>> JSON object}}: - -+--------------------------------- -HTTP/1.1 200 OK -Content-Type: application/json -Transfer-Encoding: chunked - -{"Path": "/user/szetszwo/.snapshot/s1"} -+--------------------------------- - - [] - - See also: - {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.createSnapshot - - -** {Delete Snapshot} - - * Submit a HTTP DELETE request. - -+--------------------------------- -curl -i -X DELETE "http://:/webhdfs/v1/?op=DELETESNAPSHOT&snapshotname=" -+--------------------------------- - - The client receives a response with zero content length: - -+--------------------------------- -HTTP/1.1 200 OK -Content-Length: 0 -+--------------------------------- - - [] - - See also: - {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.deleteSnapshot - - -** {Rename Snapshot} - - * Submit a HTTP PUT request. - -+--------------------------------- -curl -i -X PUT "http://:/webhdfs/v1/?op=RENAMESNAPSHOT - &oldsnapshotname=&snapshotname=" -+--------------------------------- - - The client receives a response with zero content length: - -+--------------------------------- -HTTP/1.1 200 OK -Content-Length: 0 -+--------------------------------- - - [] - - See also: - {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.renameSnapshot - - * {Delegation Token Operations} ** {Get Delegation Token} @@ -1917,26 +1839,6 @@ var tokenProperties = {{{Open and Read a File}<<>>}} -** {Old Snapshot Name} - -*----------------+-------------------------------------------------------------------+ -|| Name | <<>> | -*----------------+-------------------------------------------------------------------+ -|| Description | The old name of the snapshot to be renamed. | -*----------------+-------------------------------------------------------------------+ -|| Type | String | -*----------------+-------------------------------------------------------------------+ -|| Default Value | null | -*----------------+-------------------------------------------------------------------+ -|| Valid Values | An existing snapshot name. | -*----------------+-------------------------------------------------------------------+ -|| Syntax | Any string. | -*----------------+-------------------------------------------------------------------+ - - See also: - {{{Rename Snapshot}<<>>}} - - ** {Op} *----------------+-------------------------------------------------------------------+ @@ -2081,29 +1983,6 @@ var tokenProperties = {{{Set Replication Factor}<<>>}} -** {Snapshot Name} - -*----------------+-------------------------------------------------------------------+ -|| Name | <<>> | -*----------------+-------------------------------------------------------------------+ -|| Description | The name of the snapshot to be created/deleted. | -|| | Or the new name for snapshot rename. | -*----------------+-------------------------------------------------------------------+ -|| Type | String | -*----------------+-------------------------------------------------------------------+ -|| Default Value | null | -*----------------+-------------------------------------------------------------------+ -|| Valid Values | Any valid snapshot name. | -*----------------+-------------------------------------------------------------------+ -|| Syntax | Any string. | -*----------------+-------------------------------------------------------------------+ - - See also: - {{{Create Snapshot}<<>>}}, - {{{Delete Snapshot}<<>>}}, - {{{Rename Snapshot}<<>>}} - - ** {Sources} *----------------+-------------------------------------------------------------------+ @@ -2163,3 +2042,4 @@ var tokenProperties = See also: {{Authentication}} + diff --git hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java index a7dcbc1..09d4353 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java +++ hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java @@ -28,15 +28,12 @@ import java.io.File; import java.io.IOException; -import java.nio.channels.ClosedByInterruptException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -412,121 +409,4 @@ public void testShortCircuitCacheShutdown() throws Exception { getDomainSocketWatcher().isClosed()); cluster.shutdown(); } - - /** - * When an InterruptedException is sent to a thread calling - * FileChannel#read, the FileChannel is immediately closed and the - * thread gets an exception. This effectively means that we might have - * someone asynchronously calling close() on the file descriptors we use - * in BlockReaderLocal. So when unreferencing a ShortCircuitReplica in - * ShortCircuitCache#unref, we should check if the FileChannel objects - * are still open. If not, we should purge the replica to avoid giving - * it out to any future readers. - * - * This is a regression test for HDFS-6227: Short circuit read failed - * due to ClosedChannelException. - * - * Note that you may still get ClosedChannelException errors if two threads - * are reading from the same replica and an InterruptedException is delivered - * to one of them. - */ - @Test(timeout=120000) - public void testPurgingClosedReplicas() throws Exception { - BlockReaderTestUtil.enableBlockReaderFactoryTracing(); - final AtomicInteger replicasCreated = new AtomicInteger(0); - final AtomicBoolean testFailed = new AtomicBoolean(false); - DFSInputStream.tcpReadsDisabledForTesting = true; - BlockReaderFactory.createShortCircuitReplicaInfoCallback = - new ShortCircuitCache.ShortCircuitReplicaCreator() { - @Override - public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { - replicasCreated.incrementAndGet(); - return null; - } - }; - TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); - Configuration conf = createShortCircuitConf( - "testPurgingClosedReplicas", sockDir); - final MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - final DistributedFileSystem dfs = cluster.getFileSystem(); - final String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 4095; - final int SEED = 0xFADE0; - final DistributedFileSystem fs = - (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf); - DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - - final Semaphore sem = new Semaphore(0); - final List locatedBlocks = - cluster.getNameNode().getRpcServer().getBlockLocations( - TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks(); - final LocatedBlock lblock = locatedBlocks.get(0); // first block - final byte[] buf = new byte[TEST_FILE_LEN]; - Runnable readerRunnable = new Runnable() { - @Override - public void run() { - try { - while (true) { - BlockReader blockReader = null; - try { - blockReader = BlockReaderTestUtil. - getBlockReader(cluster, lblock, 0, TEST_FILE_LEN); - sem.release(); - try { - blockReader.readAll(buf, 0, TEST_FILE_LEN); - } finally { - sem.acquireUninterruptibly(); - } - } catch (ClosedByInterruptException e) { - LOG.info("got the expected ClosedByInterruptException", e); - sem.release(); - break; - } finally { - if (blockReader != null) blockReader.close(); - } - LOG.info("read another " + TEST_FILE_LEN + " bytes."); - } - } catch (Throwable t) { - LOG.error("getBlockReader failure", t); - testFailed.set(true); - sem.release(); - } - } - }; - Thread thread = new Thread(readerRunnable); - thread.start(); - - // While the thread is reading, send it interrupts. - // These should trigger a ClosedChannelException. - while (thread.isAlive()) { - sem.acquireUninterruptibly(); - thread.interrupt(); - sem.release(); - } - Assert.assertFalse(testFailed.get()); - - // We should be able to read from the file without - // getting a ClosedChannelException. - BlockReader blockReader = null; - try { - blockReader = BlockReaderTestUtil. - getBlockReader(cluster, lblock, 0, TEST_FILE_LEN); - blockReader.readFully(buf, 0, TEST_FILE_LEN); - } finally { - if (blockReader != null) blockReader.close(); - } - byte expected[] = DFSTestUtil. - calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); - Assert.assertTrue(Arrays.equals(buf, expected)); - - // Another ShortCircuitReplica object should have been created. - Assert.assertEquals(2, replicasCreated.get()); - - dfs.close(); - cluster.shutdown(); - sockDir.close(); - } } diff --git hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java deleted file mode 100644 index 0e56ba7..0000000 --- hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileChecksum; -import org.apache.hadoop.fs.Path; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestGetFileChecksum { - private static final int BLOCKSIZE = 1024; - private static final short REPLICATION = 3; - - private Configuration conf; - private MiniDFSCluster cluster; - private DistributedFileSystem dfs; - - @Before - public void setUp() throws Exception { - conf = new Configuration(); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) - .build(); - cluster.waitActive(); - dfs = cluster.getFileSystem(); - } - - @After - public void tearDown() throws Exception { - if (cluster != null) { - cluster.shutdown(); - } - } - - public void testGetFileChecksum(final Path foo, final int appendLength) - throws Exception { - final int appendRounds = 16; - FileChecksum[] fc = new FileChecksum[appendRounds + 1]; - DFSTestUtil.createFile(dfs, foo, appendLength, REPLICATION, 0L); - fc[0] = dfs.getFileChecksum(foo); - for (int i = 0; i < appendRounds; i++) { - DFSTestUtil.appendFile(dfs, foo, appendLength); - fc[i + 1] = dfs.getFileChecksum(foo); - } - - for (int i = 0; i < appendRounds + 1; i++) { - FileChecksum checksum = dfs.getFileChecksum(foo, appendLength * (i+1)); - Assert.assertTrue(checksum.equals(fc[i])); - } - } - - @Test - public void testGetFileChecksum() throws Exception { - testGetFileChecksum(new Path("/foo"), BLOCKSIZE / 4); - testGetFileChecksum(new Path("/bar"), BLOCKSIZE / 4 - 1); - } -} diff --git hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java index 3166ccc..e3b381c 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java +++ hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java @@ -22,12 +22,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import java.io.IOException; -import java.net.BindException; import java.net.URI; -import java.util.Random; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; @@ -41,13 +37,14 @@ private MiniDFSCluster cluster; private MiniJournalCluster journalCluster; private final Configuration conf; - private static final Log LOG = LogFactory.getLog(MiniQJMHACluster.class); public static final String NAMESERVICE = "ns1"; private static final String NN1 = "nn1"; private static final String NN2 = "nn2"; - private static final Random RANDOM = new Random(); - private int basePort = 10000; + private static final int NN1_IPC_PORT = 10000; + private static final int NN1_INFO_PORT = 10001; + private static final int NN2_IPC_PORT = 10002; + private static final int NN2_INFO_PORT = 10003; public static class Builder { private final Configuration conf; @@ -72,62 +69,51 @@ public void startupOption(StartupOption startOpt) { } } - public static MiniDFSNNTopology createDefaultTopology(int basePort) { + public static MiniDFSNNTopology createDefaultTopology() { return new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf(NAMESERVICE).addNN( - new MiniDFSNNTopology.NNConf("nn1").setIpcPort(basePort) - .setHttpPort(basePort + 1)).addNN( - new MiniDFSNNTopology.NNConf("nn2").setIpcPort(basePort + 2) - .setHttpPort(basePort + 3))); + new MiniDFSNNTopology.NNConf("nn1").setIpcPort(NN1_IPC_PORT) + .setHttpPort(NN1_INFO_PORT)).addNN( + new MiniDFSNNTopology.NNConf("nn2").setIpcPort(NN2_IPC_PORT) + .setHttpPort(NN2_INFO_PORT))); } - + private MiniQJMHACluster(Builder builder) throws IOException { this.conf = builder.conf; - int retryCount = 0; - while (true) { - try { - basePort = 10000 + RANDOM.nextInt(1000) * 4; - // start 3 journal nodes - journalCluster = new MiniJournalCluster.Builder(conf).format(true) - .build(); - URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE); - - // start cluster with 2 NameNodes - MiniDFSNNTopology topology = createDefaultTopology(basePort); - - initHAConf(journalURI, builder.conf); - - // First start up the NNs just to format the namespace. The MinIDFSCluster - // has no way to just format the NameNodes without also starting them. - cluster = builder.dfsBuilder.nnTopology(topology) - .manageNameDfsSharedDirs(false).build(); - cluster.waitActive(); - cluster.shutdown(); - - // initialize the journal nodes - Configuration confNN0 = cluster.getConfiguration(0); - NameNode.initializeSharedEdits(confNN0, true); - - cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt); - cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt); - - // restart the cluster - cluster.restartNameNodes(); - ++retryCount; - break; - } catch (BindException e) { - LOG.info("MiniQJMHACluster port conflicts, retried " + - retryCount + " times"); - } - } + // start 3 journal nodes + journalCluster = new MiniJournalCluster.Builder(conf).format(true) + .build(); + URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE); + + // start cluster with 2 NameNodes + MiniDFSNNTopology topology = createDefaultTopology(); + + initHAConf(journalURI, builder.conf); + + // First start up the NNs just to format the namespace. The MinIDFSCluster + // has no way to just format the NameNodes without also starting them. + cluster = builder.dfsBuilder.nnTopology(topology) + .manageNameDfsSharedDirs(false).build(); + cluster.waitActive(); + cluster.shutdown(); + + // initialize the journal nodes + Configuration confNN0 = cluster.getConfiguration(0); + NameNode.initializeSharedEdits(confNN0, true); + + cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt); + cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt); + + // restart the cluster + cluster.restartNameNodes(); } private Configuration initHAConf(URI journalURI, Configuration conf) { conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, journalURI.toString()); - String address1 = "127.0.0.1:" + basePort; - String address2 = "127.0.0.1:" + (basePort + 2); + String address1 = "127.0.0.1:" + NN1_IPC_PORT; + String address2 = "127.0.0.1:" + NN2_IPC_PORT; conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, NAMESERVICE, NN1), address1); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, diff --git hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java deleted file mode 100644 index 1c770b1..0000000 --- hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode; - -import static org.junit.Assert.assertEquals; - -import java.util.EnumSet; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSOutputStream; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class TestDiskspaceQuotaUpdate { - private static final int BLOCKSIZE = 1024; - private static final short REPLICATION = 1; - - private Configuration conf; - private MiniDFSCluster cluster; - private FSDirectory fsdir; - private DistributedFileSystem dfs; - - @Before - public void setUp() throws Exception { - conf = new Configuration(); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) - .build(); - cluster.waitActive(); - - fsdir = cluster.getNamesystem().getFSDirectory(); - dfs = cluster.getFileSystem(); - } - - @After - public void tearDown() throws Exception { - if (cluster != null) { - cluster.shutdown(); - } - } - - /** - * Test if the quota can be correctly updated for append - */ - @Test - public void testUpdateQuotaForAppend() throws Exception { - final Path foo = new Path("/foo"); - final Path bar = new Path(foo, "bar"); - DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L); - dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1); - - // append half of the block data - DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE / 2); - - INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); - Quota.Counts quota = fooNode.getDirectoryWithQuotaFeature() - .getSpaceConsumed(); - long ns = quota.get(Quota.NAMESPACE); - long ds = quota.get(Quota.DISKSPACE); - assertEquals(2, ns); // foo and bar - assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds); - - // append another block - DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE); - - quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); - ns = quota.get(Quota.NAMESPACE); - ds = quota.get(Quota.DISKSPACE); - assertEquals(2, ns); // foo and bar - assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds); - } - - /** - * Test if the quota can be correctly updated when file length is updated - * through fsync - */ - @Test - public void testUpdateQuotaForFSync() throws Exception { - final Path foo = new Path("/foo"); - final Path bar = new Path(foo, "bar"); - DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L); - dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1); - - FSDataOutputStream out = dfs.append(bar); - out.write(new byte[BLOCKSIZE / 4]); - ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet - .of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); - - INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); - Quota.Counts quota = fooNode.getDirectoryWithQuotaFeature() - .getSpaceConsumed(); - long ns = quota.get(Quota.NAMESPACE); - long ds = quota.get(Quota.DISKSPACE); - assertEquals(2, ns); // foo and bar - assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction - - out.write(new byte[BLOCKSIZE / 4]); - out.close(); - - fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); - quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); - ns = quota.get(Quota.NAMESPACE); - ds = quota.get(Quota.DISKSPACE); - assertEquals(2, ns); - assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds); - - // append another block - DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE); - - quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); - ns = quota.get(Quota.NAMESPACE); - ds = quota.get(Quota.DISKSPACE); - assertEquals(2, ns); // foo and bar - assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds); - } -} diff --git hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java index 5ddc635..8bcc6d7 100644 --- hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java +++ hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java @@ -104,7 +104,7 @@ public void setUpCluster() throws Exception { HAUtil.setAllowStandbyReads(conf, true); if (clusterType == TestType.SHARED_DIR_HA) { - MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(10000); + MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) .numDataNodes(0) diff --git hadoop-mapreduce-project/CHANGES.txt hadoop-mapreduce-project/CHANGES.txt index 82e9098..ce25750 100644 --- hadoop-mapreduce-project/CHANGES.txt +++ hadoop-mapreduce-project/CHANGES.txt @@ -202,8 +202,6 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5809. Enhance distcp to support preserving HDFS ACLs. (cnauroth) - MAPREDUCE-5899. Support incremental data copy in DistCp. (jing9) - OPTIMIZATIONS BUG FIXES diff --git hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java index 5fa26da..695d8bd 100644 --- hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java +++ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java @@ -50,7 +50,6 @@ public static final String CONF_LABEL_COPY_STRATEGY = "distcp.copy.strategy"; public static final String CONF_LABEL_SKIP_CRC = "distcp.skip.crc"; public static final String CONF_LABEL_OVERWRITE = "distcp.copy.overwrite"; - public static final String CONF_LABEL_APPEND = "distcp.copy.append"; public static final String CONF_LABEL_BANDWIDTH_MB = "distcp.map.bandwidth.mb"; public static final String CONF_LABEL_MAX_CHUNKS_TOLERABLE = diff --git hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index bfaba96..1639c1d 100644 --- hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -138,10 +138,6 @@ new Option("overwrite", false, "Choose to overwrite target files " + "unconditionally, even if they exist.")), - APPEND(DistCpConstants.CONF_LABEL_APPEND, - new Option("append", false, - "Reuse existing data in target files and append new data to them if possible")), - /** * Should DisctpExecution be blocking */ diff --git hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index 2d19c6a..5906266 100644 --- hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -39,7 +39,6 @@ private boolean deleteMissing = false; private boolean ignoreFailures = false; private boolean overwrite = false; - private boolean append = false; private boolean skipCRC = false; private boolean blocking = true; @@ -246,22 +245,6 @@ public void setOverwrite(boolean overwrite) { } /** - * @return whether we can append new data to target files - */ - public boolean shouldAppend() { - return append; - } - - /** - * Set if we want to append new data to target files. This is valid only with - * update option and CRC is not skipped. - */ - public void setAppend(boolean append) { - validate(DistCpOptionSwitch.APPEND, append); - this.append = append; - } - - /** * Should CRC/checksum check be skipped while checking files are identical * * @return true if checksum check should be skipped while checking files are @@ -489,7 +472,6 @@ public void validate(DistCpOptionSwitch option, boolean value) { value : this.atomicCommit); boolean skipCRC = (option == DistCpOptionSwitch.SKIP_CRC ? value : this.skipCRC); - boolean append = (option == DistCpOptionSwitch.APPEND ? value : this.append); if (syncFolder && atomicCommit) { throw new IllegalArgumentException("Atomic commit can't be used with " + @@ -510,14 +492,6 @@ public void validate(DistCpOptionSwitch option, boolean value) { throw new IllegalArgumentException("Skip CRC is valid only with update options"); } - if (!syncFolder && append) { - throw new IllegalArgumentException( - "Append is valid only with update options"); - } - if (skipCRC && append) { - throw new IllegalArgumentException( - "Append is disallowed when skipping CRC"); - } } /** @@ -536,8 +510,6 @@ public void appendToConf(Configuration conf) { String.valueOf(deleteMissing)); DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.OVERWRITE, String.valueOf(overwrite)); - DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.APPEND, - String.valueOf(append)); DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC, String.valueOf(skipCRC)); DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH, diff --git hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java index 09e8550..d36311d 100644 --- hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java +++ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java @@ -140,10 +140,6 @@ public static DistCpOptions parse(String args[]) throws IllegalArgumentException option.setOverwrite(true); } - if (command.hasOption(DistCpOptionSwitch.APPEND.getSwitch())) { - option.setAppend(true); - } - if (command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch())) { option.setDeleteMissing(true); } diff --git hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java index 02337f7..caf4057 100644 --- hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java +++ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java @@ -18,20 +18,13 @@ package org.apache.hadoop.tools.mapred; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.EnumSet; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.Mapper; @@ -43,6 +36,11 @@ import org.apache.hadoop.tools.util.DistCpUtils; import org.apache.hadoop.util.StringUtils; +import java.io.*; +import java.util.EnumSet; +import java.util.Arrays; +import java.util.List; + /** * Mapper class that executes the DistCp copy operation. * Implements the o.a.h.mapreduce.Mapper<> interface. @@ -64,15 +62,6 @@ BYTESSKIPPED, // Number of bytes that were skipped from copy. } - /** - * Indicate the action for each file - */ - static enum FileAction { - SKIP, // Skip copying the file since it's already in the target FS - APPEND, // Only need to append new data to the file in the target FS - OVERWRITE, // Overwrite the whole file - } - private static Log LOG = LogFactory.getLog(CopyMapper.class); private Configuration conf; @@ -81,7 +70,6 @@ private boolean ignoreFailures = false; private boolean skipCrc = false; private boolean overWrite = false; - private boolean append = false; private EnumSet preserve = EnumSet.noneOf(FileAttribute.class); private FileSystem targetFS = null; @@ -102,7 +90,6 @@ public void setup(Context context) throws IOException, InterruptedException { ignoreFailures = conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); skipCrc = conf.getBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false); overWrite = conf.getBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false); - append = conf.getBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), false); preserve = DistCpUtils.unpackAttributes(conf.get(DistCpOptionSwitch. PRESERVE_STATUS.getConfigLabel())); @@ -237,19 +224,20 @@ public void map(Text relPath, CopyListingFileStatus sourceFileStatus, return; } - FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target); - if (action == FileAction.SKIP) { + if (skipFile(sourceFS, sourceCurrStatus, target)) { LOG.info("Skipping copy of " + sourceCurrStatus.getPath() + " to " + target); updateSkipCounters(context, sourceCurrStatus); context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath())); - } else { + } + else { copyFileWithRetry(description, sourceCurrStatus, target, context, - action, fileAttributes); + fileAttributes); } DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus, fileAttributes); + } catch (IOException exception) { handleFailures(exception, sourceFileStatus, target, context); } @@ -266,14 +254,14 @@ private String getFileType(FileStatus fileStatus) { return DistCpUtils.unpackAttributes(attributeString); } - private void copyFileWithRetry(String description, - FileStatus sourceFileStatus, Path target, Context context, - FileAction action, EnumSet fileAttributes) - throws IOException { + private void copyFileWithRetry(String description, FileStatus sourceFileStatus, + Path target, Context context, + EnumSet fileAttributes) throws IOException { + long bytesCopied; try { - bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description, - action).execute(sourceFileStatus, target, context, fileAttributes); + bytesCopied = (Long)new RetriableFileCopyCommand(skipCrc, description) + .execute(sourceFileStatus, target, context, fileAttributes); } catch (Exception e) { context.setStatus("Copy Failure: " + sourceFileStatus.getPath()); throw new IOException("File copy failed: " + sourceFileStatus.getPath() + @@ -323,48 +311,25 @@ private static void incrementCounter(Context context, Counter counter, context.getCounter(counter).increment(value); } - private FileAction checkUpdate(FileSystem sourceFS, FileStatus source, - Path target) throws IOException { - final FileStatus targetFileStatus; - try { - targetFileStatus = targetFS.getFileStatus(target); - } catch (FileNotFoundException e) { - return FileAction.OVERWRITE; - } - if (targetFileStatus != null && !overWrite) { - if (canSkip(sourceFS, source, targetFileStatus)) { - return FileAction.SKIP; - } else if (append) { - long targetLen = targetFileStatus.getLen(); - if (targetLen < source.getLen()) { - FileChecksum sourceChecksum = sourceFS.getFileChecksum( - source.getPath(), targetLen); - if (sourceChecksum != null - && sourceChecksum.equals(targetFS.getFileChecksum(target))) { - // We require that the checksum is not null. Thus currently only - // DistributedFileSystem is supported - return FileAction.APPEND; - } - } - } - } - return FileAction.OVERWRITE; + private boolean skipFile(FileSystem sourceFS, FileStatus source, Path target) + throws IOException { + return targetFS.exists(target) + && !overWrite + && !mustUpdate(sourceFS, source, target); } - private boolean canSkip(FileSystem sourceFS, FileStatus source, - FileStatus target) throws IOException { - if (!syncFolders) { - return true; - } - boolean sameLength = target.getLen() == source.getLen(); - boolean sameBlockSize = source.getBlockSize() == target.getBlockSize() - || !preserve.contains(FileAttribute.BLOCKSIZE); - if (sameLength && sameBlockSize) { - return skipCrc || - DistCpUtils.checksumsAreEqual(sourceFS, source.getPath(), null, - targetFS, target.getPath()); - } else { - return false; - } + private boolean mustUpdate(FileSystem sourceFS, FileStatus source, Path target) + throws IOException { + final FileStatus targetFileStatus = targetFS.getFileStatus(target); + + return syncFolders + && ( + targetFileStatus.getLen() != source.getLen() + || (!skipCrc && + !DistCpUtils.checksumsAreEqual(sourceFS, + source.getPath(), null, targetFS, target)) + || (source.getBlockSize() != targetFileStatus.getBlockSize() && + preserve.contains(FileAttribute.BLOCKSIZE)) + ); } } diff --git hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index 1d61156..1b73526 100644 --- hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -18,8 +18,10 @@ package org.apache.hadoop.tools.mapred; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.EnumSet; @@ -27,8 +29,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CreateFlag; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -39,7 +39,6 @@ import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.tools.DistCpConstants; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; -import org.apache.hadoop.tools.mapred.CopyMapper.FileAction; import org.apache.hadoop.tools.util.DistCpUtils; import org.apache.hadoop.tools.util.RetriableCommand; import org.apache.hadoop.tools.util.ThrottledInputStream; @@ -55,15 +54,13 @@ private static Log LOG = LogFactory.getLog(RetriableFileCopyCommand.class); private static int BUFFER_SIZE = 8 * 1024; private boolean skipCrc = false; - private FileAction action; /** * Constructor, taking a description of the action. * @param description Verbose description of the copy operation. */ - public RetriableFileCopyCommand(String description, FileAction action) { + public RetriableFileCopyCommand(String description) { super(description); - this.action = action; } /** @@ -71,11 +68,9 @@ public RetriableFileCopyCommand(String description, FileAction action) { * * @param skipCrc Whether to skip the crc check. * @param description A verbose description of the copy operation. - * @param action We should overwrite the target file or append new data to it. */ - public RetriableFileCopyCommand(boolean skipCrc, String description, - FileAction action) { - this(description, action); + public RetriableFileCopyCommand(boolean skipCrc, String description) { + this(description); this.skipCrc = skipCrc; } @@ -101,17 +96,18 @@ protected Object doExecute(Object... arguments) throws Exception { } private long doCopy(FileStatus sourceFileStatus, Path target, - Mapper.Context context, EnumSet fileAttributes) - throws IOException { - final boolean toAppend = action == FileAction.APPEND; - Path targetPath = toAppend ? target : getTmpFile(target, context); + Mapper.Context context, + EnumSet fileAttributes) + throws IOException { + + Path tmpTargetPath = getTmpFile(target, context); final Configuration configuration = context.getConfiguration(); FileSystem targetFS = target.getFileSystem(configuration); try { if (LOG.isDebugEnabled()) { LOG.debug("Copying " + sourceFileStatus.getPath() + " to " + target); - LOG.debug("Target file path: " + targetPath); + LOG.debug("Tmp-file path: " + tmpTargetPath); } final Path sourcePath = sourceFileStatus.getPath(); final FileSystem sourceFS = sourcePath.getFileSystem(configuration); @@ -119,31 +115,22 @@ private long doCopy(FileStatus sourceFileStatus, Path target, .contains(FileAttribute.CHECKSUMTYPE) ? sourceFS .getFileChecksum(sourcePath) : null; - final long offset = action == FileAction.APPEND ? targetFS.getFileStatus( - target).getLen() : 0; - long bytesRead = copyToFile(targetPath, targetFS, sourceFileStatus, - offset, context, fileAttributes, sourceChecksum); + long bytesRead = copyToTmpFile(tmpTargetPath, targetFS, sourceFileStatus, + context, fileAttributes, sourceChecksum); - compareFileLengths(sourceFileStatus, targetPath, configuration, bytesRead - + offset); + compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, + bytesRead); //At this point, src&dest lengths are same. if length==0, we skip checksum if ((bytesRead != 0) && (!skipCrc)) { compareCheckSums(sourceFS, sourceFileStatus.getPath(), sourceChecksum, - targetFS, targetPath); - } - // it's not append case, thus we first write to a temporary file, rename - // it to the target path. - if (!toAppend) { - promoteTmpToTarget(targetPath, target, targetFS); + targetFS, tmpTargetPath); } + promoteTmpToTarget(tmpTargetPath, target, targetFS); return bytesRead; + } finally { - // note that for append case, it is possible that we append partial data - // and then fail. In that case, for the next retry, we either reuse the - // partial appended data if it is good or we overwrite the whole file - if (!toAppend && targetFS.exists(targetPath)) { - targetFS.delete(targetPath, false); - } + if (targetFS.exists(tmpTargetPath)) + targetFS.delete(tmpTargetPath, false); } } @@ -160,37 +147,29 @@ private ChecksumOpt getChecksumOpt(EnumSet fileAttributes, return null; } - private long copyToFile(Path targetPath, FileSystem targetFS, - FileStatus sourceFileStatus, long sourceOffset, Mapper.Context context, + private long copyToTmpFile(Path tmpTargetPath, FileSystem targetFS, + FileStatus sourceFileStatus, Mapper.Context context, EnumSet fileAttributes, final FileChecksum sourceChecksum) throws IOException { FsPermission permission = FsPermission.getFileDefault().applyUMask( FsPermission.getUMask(targetFS.getConf())); - final OutputStream outStream; - if (action == FileAction.OVERWRITE) { - final short repl = getReplicationFactor(fileAttributes, sourceFileStatus, - targetFS, targetPath); - final long blockSize = getBlockSize(fileAttributes, sourceFileStatus, - targetFS, targetPath); - FSDataOutputStream out = targetFS.create(targetPath, permission, - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), - BUFFER_SIZE, repl, blockSize, context, - getChecksumOpt(fileAttributes, sourceChecksum)); - outStream = new BufferedOutputStream(out); - } else { - outStream = new BufferedOutputStream(targetFS.append(targetPath, - BUFFER_SIZE)); - } - return copyBytes(sourceFileStatus, sourceOffset, outStream, BUFFER_SIZE, - context); + OutputStream outStream = new BufferedOutputStream( + targetFS.create(tmpTargetPath, permission, + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), BUFFER_SIZE, + getReplicationFactor(fileAttributes, sourceFileStatus, targetFS, + tmpTargetPath), + getBlockSize(fileAttributes, sourceFileStatus, targetFS, + tmpTargetPath), + context, getChecksumOpt(fileAttributes, sourceChecksum))); + return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, context); } private void compareFileLengths(FileStatus sourceFileStatus, Path target, - Configuration configuration, long targetLen) + Configuration configuration, long bytesRead) throws IOException { final Path sourcePath = sourceFileStatus.getPath(); FileSystem fs = sourcePath.getFileSystem(configuration); - if (fs.getFileStatus(sourcePath).getLen() != targetLen) + if (fs.getFileStatus(sourcePath).getLen() != bytesRead) throw new IOException("Mismatch in length of source:" + sourcePath + " and target:" + target); } @@ -236,8 +215,8 @@ private Path getTmpFile(Path target, Mapper.Context context) { } @VisibleForTesting - long copyBytes(FileStatus sourceFileStatus, long sourceOffset, - OutputStream outStream, int bufferSize, Mapper.Context context) + long copyBytes(FileStatus sourceFileStatus, OutputStream outStream, + int bufferSize, Mapper.Context context) throws IOException { Path source = sourceFileStatus.getPath(); byte buf[] = new byte[bufferSize]; @@ -246,21 +225,19 @@ long copyBytes(FileStatus sourceFileStatus, long sourceOffset, try { inStream = getInputStream(source, context.getConfiguration()); - int bytesRead = readBytes(inStream, buf, sourceOffset); + int bytesRead = readBytes(inStream, buf); while (bytesRead >= 0) { totalBytesRead += bytesRead; - if (action == FileAction.APPEND) { - sourceOffset += bytesRead; - } outStream.write(buf, 0, bytesRead); updateContextStatus(totalBytesRead, context, sourceFileStatus); - bytesRead = readBytes(inStream, buf, sourceOffset); + bytesRead = inStream.read(buf); } outStream.close(); outStream = null; } finally { IOUtils.cleanup(LOG, outStream, inStream); } + return totalBytesRead; } @@ -277,27 +254,24 @@ private void updateContextStatus(long totalBytesRead, Mapper.Context context, context.setStatus(message.toString()); } - private static int readBytes(ThrottledInputStream inStream, byte buf[], - long position) throws IOException { + private static int readBytes(InputStream inStream, byte buf[]) + throws IOException { try { - if (position == 0) { - return inStream.read(buf); - } else { - return inStream.read(position, buf, 0, buf.length); - } - } catch (IOException e) { + return inStream.read(buf); + } + catch (IOException e) { throw new CopyReadException(e); } } - private static ThrottledInputStream getInputStream(Path path, - Configuration conf) throws IOException { + private static ThrottledInputStream getInputStream(Path path, Configuration conf) + throws IOException { try { FileSystem fs = path.getFileSystem(conf); long bandwidthMB = conf.getInt(DistCpConstants.CONF_LABEL_BANDWIDTH_MB, DistCpConstants.DEFAULT_BANDWIDTH_MB); - FSDataInputStream in = fs.open(path); - return new ThrottledInputStream(in, bandwidthMB * 1024 * 1024); + return new ThrottledInputStream(new BufferedInputStream(fs.open(path)), + bandwidthMB * 1024 * 1024); } catch (IOException e) { throw new CopyReadException(e); diff --git hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java index f6fe118..75ae86a 100644 --- hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java +++ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java @@ -21,11 +21,6 @@ import java.io.IOException; import java.io.InputStream; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.PositionedReadable; - -import com.google.common.base.Preconditions; - /** * The ThrottleInputStream provides bandwidth throttling on a specified * InputStream. It is implemented as a wrapper on top of another InputStream @@ -95,25 +90,6 @@ public int read(byte[] b, int off, int len) throws IOException { return readLen; } - /** - * Read bytes starting from the specified position. This requires rawStream is - * an instance of {@link PositionedReadable}. - */ - public int read(long position, byte[] buffer, int offset, int length) - throws IOException { - if (!(rawStream instanceof PositionedReadable)) { - throw new UnsupportedOperationException( - "positioned read is not supported by the internal stream"); - } - throttle(); - int readLen = ((PositionedReadable) rawStream).read(position, buffer, - offset, length); - if (readLen != -1) { - bytesRead += readLen; - } - return readLen; - } - private void throttle() throws IOException { if (getBytesPerSec() > maxBytesPerSec) { try { diff --git hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index 8486aa1..296e994 100644 --- hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -18,12 +18,9 @@ package org.apache.hadoop.tools; -import static org.junit.Assert.fail; - import org.junit.Assert; import org.junit.Test; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.tools.DistCpOptions.*; import org.apache.hadoop.conf.Configuration; @@ -557,45 +554,4 @@ public void testOptionsAppendToConf() { Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), "U"); Assert.assertEquals(conf.getInt(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11); } - - @Test - public void testAppendOption() { - Configuration conf = new Configuration(); - Assert.assertFalse(conf.getBoolean( - DistCpOptionSwitch.APPEND.getConfigLabel(), false)); - Assert.assertFalse(conf.getBoolean( - DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); - - DistCpOptions options = OptionsParser.parse(new String[] { "-update", - "-append", "hdfs://localhost:8020/source/first", - "hdfs://localhost:8020/target/" }); - options.appendToConf(conf); - Assert.assertTrue(conf.getBoolean( - DistCpOptionSwitch.APPEND.getConfigLabel(), false)); - Assert.assertTrue(conf.getBoolean( - DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); - - // make sure -append is only valid when -update is specified - try { - options = OptionsParser.parse(new String[] { "-append", - "hdfs://localhost:8020/source/first", - "hdfs://localhost:8020/target/" }); - fail("Append should fail if update option is not specified"); - } catch (IllegalArgumentException e) { - GenericTestUtils.assertExceptionContains( - "Append is valid only with update options", e); - } - - // make sure -append is invalid when skipCrc is specified - try { - options = OptionsParser.parse(new String[] { - "-append", "-update", "-skipcrccheck", - "hdfs://localhost:8020/source/first", - "hdfs://localhost:8020/target/" }); - fail("Append should fail if skipCrc option is specified"); - } catch (IllegalArgumentException e) { - GenericTestUtils.assertExceptionContains( - "Append is disallowed when skipping CRC", e); - } - } } diff --git hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java index 2f16682..7eb1b68 100644 --- hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java +++ hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java @@ -25,13 +25,11 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; -import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CreateFlag; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options.ChecksumOpt; @@ -120,16 +118,6 @@ private static void createSourceData() throws Exception { touchFile(SOURCE_PATH + "/7/8/9"); } - private static void appendSourceData() throws Exception { - FileSystem fs = cluster.getFileSystem(); - for (Path source : pathList) { - if (fs.getFileStatus(source).isFile()) { - // append 2048 bytes per file - appendFile(source, DEFAULT_FILE_SIZE * 2); - } - } - } - private static void createSourceDataWithDifferentBlockSize() throws Exception { mkdirs(SOURCE_PATH + "/1"); mkdirs(SOURCE_PATH + "/2"); @@ -213,132 +201,85 @@ private static void touchFile(String path, boolean createMultipleBlocks, } } - /** - * Append specified length of bytes to a given file - */ - private static void appendFile(Path p, int length) throws IOException { - byte[] toAppend = new byte[length]; - Random random = new Random(); - random.nextBytes(toAppend); - FSDataOutputStream out = cluster.getFileSystem().append(p); - try { - out.write(toAppend); - } finally { - IOUtils.closeStream(out); - } - } - @Test public void testCopyWithDifferentChecksumType() throws Exception { testCopy(true); } @Test(timeout=40000) - public void testRun() throws Exception { + public void testRun() { testCopy(false); } - @Test - public void testCopyWithAppend() throws Exception { - final FileSystem fs = cluster.getFileSystem(); - // do the first distcp - testCopy(false); - // start appending data to source - appendSourceData(); - - // do the distcp again with -update and -append option - CopyMapper copyMapper = new CopyMapper(); - StubContext stubContext = new StubContext(getConfiguration(), null, 0); - Mapper.Context context = - stubContext.getContext(); - // Enable append - context.getConfiguration().setBoolean( - DistCpOptionSwitch.APPEND.getConfigLabel(), true); - copyMapper.setup(context); - for (Path path: pathList) { - copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), - new CopyListingFileStatus(cluster.getFileSystem().getFileStatus( - path)), context); - } - - verifyCopy(fs, false); - // verify that we only copied new appended data - Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE * 2, stubContext - .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) - .getValue()); - Assert.assertEquals(pathList.size(), stubContext.getReporter(). - getCounter(CopyMapper.Counter.COPY).getValue()); - } + private void testCopy(boolean preserveChecksum) { + try { + deleteState(); + if (preserveChecksum) { + createSourceDataWithDifferentChecksumType(); + } else { + createSourceData(); + } - private void testCopy(boolean preserveChecksum) throws Exception { - deleteState(); - if (preserveChecksum) { - createSourceDataWithDifferentChecksumType(); - } else { - createSourceData(); - } + FileSystem fs = cluster.getFileSystem(); + CopyMapper copyMapper = new CopyMapper(); + StubContext stubContext = new StubContext(getConfiguration(), null, 0); + Mapper.Context context + = stubContext.getContext(); - FileSystem fs = cluster.getFileSystem(); - CopyMapper copyMapper = new CopyMapper(); - StubContext stubContext = new StubContext(getConfiguration(), null, 0); - Mapper.Context context - = stubContext.getContext(); - - Configuration configuration = context.getConfiguration(); - EnumSet fileAttributes - = EnumSet.of(DistCpOptions.FileAttribute.REPLICATION); - if (preserveChecksum) { - fileAttributes.add(DistCpOptions.FileAttribute.CHECKSUMTYPE); - } - configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(), - DistCpUtils.packAttributes(fileAttributes)); + Configuration configuration = context.getConfiguration(); + EnumSet fileAttributes + = EnumSet.of(DistCpOptions.FileAttribute.REPLICATION); + if (preserveChecksum) { + fileAttributes.add(DistCpOptions.FileAttribute.CHECKSUMTYPE); + } + configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(), + DistCpUtils.packAttributes(fileAttributes)); - copyMapper.setup(context); + copyMapper.setup(context); - for (Path path: pathList) { - copyMapper.map( - new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), - new CopyListingFileStatus(fs.getFileStatus(path)), context); - } + for (Path path: pathList) { + copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), + new CopyListingFileStatus(fs.getFileStatus(path)), context); + } - // Check that the maps worked. - verifyCopy(fs, preserveChecksum); - Assert.assertEquals(pathList.size(), stubContext.getReporter() - .getCounter(CopyMapper.Counter.COPY).getValue()); - if (!preserveChecksum) { - Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE, stubContext - .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) - .getValue()); - } else { - Assert.assertEquals(nFiles * NON_DEFAULT_BLOCK_SIZE * 2, stubContext - .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) - .getValue()); - } + // Check that the maps worked. + for (Path path : pathList) { + final Path targetPath = new Path(path.toString() + .replaceAll(SOURCE_PATH, TARGET_PATH)); + Assert.assertTrue(fs.exists(targetPath)); + Assert.assertTrue(fs.isFile(targetPath) == fs.isFile(path)); + FileStatus sourceStatus = fs.getFileStatus(path); + FileStatus targetStatus = fs.getFileStatus(targetPath); + Assert.assertEquals(sourceStatus.getReplication(), + targetStatus.getReplication()); + if (preserveChecksum) { + Assert.assertEquals(sourceStatus.getBlockSize(), + targetStatus.getBlockSize()); + } + Assert.assertTrue(!fs.isFile(targetPath) + || fs.getFileChecksum(targetPath).equals(fs.getFileChecksum(path))); + } - testCopyingExistingFiles(fs, copyMapper, context); - for (Text value : stubContext.getWriter().values()) { - Assert.assertTrue(value.toString() + " is not skipped", value - .toString().startsWith("SKIP:")); - } - } + Assert.assertEquals(pathList.size(), + stubContext.getReporter().getCounter(CopyMapper.Counter.COPY).getValue()); + if (!preserveChecksum) { + Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE, stubContext + .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) + .getValue()); + } else { + Assert.assertEquals(nFiles * NON_DEFAULT_BLOCK_SIZE * 2, stubContext + .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) + .getValue()); + } - private void verifyCopy(FileSystem fs, boolean preserveChecksum) - throws Exception { - for (Path path : pathList) { - final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, - TARGET_PATH)); - Assert.assertTrue(fs.exists(targetPath)); - Assert.assertTrue(fs.isFile(targetPath) == fs.isFile(path)); - FileStatus sourceStatus = fs.getFileStatus(path); - FileStatus targetStatus = fs.getFileStatus(targetPath); - Assert.assertEquals(sourceStatus.getReplication(), - targetStatus.getReplication()); - if (preserveChecksum) { - Assert.assertEquals(sourceStatus.getBlockSize(), - targetStatus.getBlockSize()); + testCopyingExistingFiles(fs, copyMapper, context); + for (Text value : stubContext.getWriter().values()) { + Assert.assertTrue(value.toString() + " is not skipped", value.toString().startsWith("SKIP:")); } - Assert.assertTrue(!fs.isFile(targetPath) - || fs.getFileChecksum(targetPath).equals(fs.getFileChecksum(path))); + } + catch (Exception e) { + LOG.error("Unexpected exception: ", e); + Assert.assertTrue(false); } } diff --git hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java index f1b8532..c5ec513 100644 --- hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java +++ hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java @@ -22,7 +22,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.*; -import org.apache.hadoop.tools.mapred.CopyMapper.FileAction; import org.junit.Test; import static org.junit.Assert.*; import static org.mockito.Mockito.*; @@ -49,8 +48,8 @@ public void testFailOnCloseError() throws Exception { Exception actualEx = null; try { - new RetriableFileCopyCommand("testFailOnCloseError", FileAction.OVERWRITE) - .copyBytes(stat, 0, out, 512, context); + new RetriableFileCopyCommand("testFailOnCloseError") + .copyBytes(stat, out, 512, context); } catch (Exception e) { actualEx = e; } diff --git hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 046b213..f923733 100644 --- hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -36,8 +36,6 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -874,11 +872,4 @@ public String moveApplication(ApplicationId appId, String newQueue) throws YarnException { return scheduler.moveApplication(appId, newQueue); } - - @Override - @LimitedPrivate("yarn") - @Unstable - public Resource getClusterResource() { - return null; - } } diff --git hadoop-yarn-project/CHANGES.txt hadoop-yarn-project/CHANGES.txt index 81ff5bc..dd8aa12 100644 --- hadoop-yarn-project/CHANGES.txt +++ hadoop-yarn-project/CHANGES.txt @@ -85,32 +85,6 @@ Release 2.5.0 - UNRELEASED YARN-1981. Nodemanager version is not updated when a node reconnects (Jason Lowe via jeagles) - YARN-1938. Added kerberos login for the Timeline Server. (Zhijie Shen via - vinodkv) - - YARN-2017. Merged some of the common scheduler code. (Jian He via vinodkv) - - YARN-2049. Added delegation-token support for the Timeline Server. (Zhijie - Shen via vinodkv) - - YARN-1936. Added security support for the Timeline Client. (Zhijie Shen via - vinodkv) - - YARN-1937. Added owner-only ACLs support for Timeline Client and server. - (Zhijie Shen via vinodkv) - - YARN-2012. Fair Scheduler: allow default queue placement rule to take an - arbitrary queue (Ashwin Shankar via Sandy Ryza) - - YARN-2059. Added admin ACLs support to Timeline Server. (Zhijie Shen via - vinodkv) - - YARN-2073. Fair Scheduler: Add a utilization threshold to prevent preempting - resources when cluster is free (Karthik Kambatla via Sandy Ryza) - - YARN-2071. Modified levelDB store permissions to be readable only by the - server user. (Zhijie Shen via vinodkv) - OPTIMIZATIONS BUG FIXES @@ -143,15 +117,6 @@ Release 2.5.0 - UNRELEASED YARN-2050. Fix LogCLIHelpers to create the correct FileContext (Ming Ma via jlowe) - YARN-2089. FairScheduler: QueuePlacementPolicy and QueuePlacementRule - are missing audience annotations. (Zhihai Xu via kasha) - - YARN-2096. Race in TestRMRestart#testQueueMetricsOnRMRestart. - (Anubhav Dhoot via kasha) - - YARN-2105. Fix TestFairScheduler after YARN-2012. (Ashwin Shankar via - Sandy Ryza) - Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES @@ -272,9 +237,6 @@ Release 2.4.1 - UNRELEASED YARN-2066. Wrong field is referenced in GetApplicationsRequestPBImpl#mergeLocalToBuilder() (Hong Zhiguo via junping_du) - YARN-2081. Fixed TestDistributedShell failure after YARN-1962. (Zhiguo Hong - via zjshen) - Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES diff --git hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 79ce746..0fa1a9e 100644 --- hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -142,17 +142,6 @@ - - - - - - - - - - - @@ -189,6 +178,12 @@ + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java deleted file mode 100644 index 90d13a2..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records.timeline; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -/** - * The response of delegation token related request - */ -@XmlRootElement(name = "delegationtoken") -@XmlAccessorType(XmlAccessType.NONE) -@Public -@Unstable -public class TimelineDelegationTokenResponse { - - private String type; - private Object content; - - public TimelineDelegationTokenResponse() { - - } - - @XmlElement(name = "type") - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - @XmlElement(name = "content") - public Object getContent() { - return content; - } - - public void setContent(Object content) { - this.content = content; - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java index 77a97ba..37c0046 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java @@ -107,17 +107,6 @@ public void setErrors(List errors) { */ public static final int IO_EXCEPTION = 2; - /** - * Error code returned if the user specifies the timeline system reserved - * filter key - */ - public static final int SYSTEM_FILTER_CONFLICT = 3; - - /** - * Error code returned if the user is denied to access the timeline data - */ - public static final int ACCESS_DENIED = 4; - private String entityId; private String entityType; private int errorCode; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 48763b3..cbf6552 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -324,11 +324,11 @@ public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address"; public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries"; - public static final int DEFAULT_ZK_RM_NUM_RETRIES = 500; + public static final int DEFAULT_ZK_RM_NUM_RETRIES = 1000; public static final String RM_ZK_RETRY_INTERVAL_MS = RM_ZK_PREFIX + "retry-interval-ms"; - public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 2000; + public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 1000; public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms"; public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000; @@ -1136,6 +1136,14 @@ public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; + /**The kerberos principal to be used for spnego filter for timeline service.*/ + public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY = + TIMELINE_SERVICE_PREFIX + "webapp.spnego-principal"; + + /**The kerberos keytab to be used for spnego filter for timeline service.*/ + public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = + TIMELINE_SERVICE_PREFIX + "webapp.spnego-keytab-file"; + /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; @@ -1188,14 +1196,6 @@ public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS = 1000 * 60 * 5; - /** The Kerberos principal for the timeline server.*/ - public static final String TIMELINE_SERVICE_PRINCIPAL = - TIMELINE_SERVICE_PREFIX + "principal"; - - /** The Kerberos keytab for the timeline server.*/ - public static final String TIMELINE_SERVICE_KEYTAB = - TIMELINE_SERVICE_PREFIX + "keytab"; - //////////////////////////////// // Other Configs //////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 3445e27..2907e4b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -73,7 +73,6 @@ public void setup() throws Exception { conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); conf.set("yarn.log.dir", "target"); - conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster( TestDistributedShell.class.getSimpleName(), 1, 1, 1, 1, true); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java index de1d3e2..fda0a4f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java @@ -23,17 +23,17 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; /** * A client library that can be used to post some information in terms of a * number of conceptual entities. + * + * @See Entity */ @Public @Unstable @@ -67,22 +67,4 @@ protected TimelineClient(String name) { public abstract TimelinePutResponse putEntities( TimelineEntity... entities) throws IOException, YarnException; - /** - *

- * Get a delegation token so as to be able to talk to the timeline server in a - * secure way. - *

- * - * @param renewer - * Address of the renewer who can renew these tokens when needed by - * securely talking to the timeline server - * @return a delegation token ({@link Token}) that can be used to talk to the - * timeline server - * @throws IOException - * @throws YarnException - */ - @Public - public abstract Token getDelegationToken( - String renewer) throws IOException, YarnException; - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java deleted file mode 100644 index f4f1507..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java +++ /dev/null @@ -1,252 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.client.api.impl; - -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.client.Authenticator; -import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenOperation; -import org.apache.hadoop.yarn.security.client.TimelineAuthenticationConsts; -import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.map.ObjectMapper; - -/** - * A KerberosAuthenticator subclass that fallback to - * {@link TimelineAuthenticationConsts}. - */ -@Private -@Unstable -public class TimelineAuthenticator extends KerberosAuthenticator { - - private static ObjectMapper mapper; - - static { - mapper = new ObjectMapper(); - YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); - } - - /** - * Returns the fallback authenticator if the server does not use Kerberos - * SPNEGO HTTP authentication. - * - * @return a {@link TimelineAuthenticationConsts} instance. - */ - @Override - protected Authenticator getFallBackAuthenticator() { - return new TimelineAuthenticator(); - } - - public static void injectDelegationToken(Map params, - Token dtToken) - throws IOException { - if (dtToken != null) { - params.put(TimelineAuthenticationConsts.DELEGATION_PARAM, - dtToken.encodeToUrlString()); - } - } - - private boolean hasDelegationToken(URL url) { - return url.getQuery().contains( - TimelineAuthenticationConsts.DELEGATION_PARAM + "="); - } - - @Override - public void authenticate(URL url, AuthenticatedURL.Token token) - throws IOException, AuthenticationException { - if (!hasDelegationToken(url)) { - super.authenticate(url, token); - } - } - - public static Token getDelegationToken( - URL url, AuthenticatedURL.Token token, String renewer) throws IOException { - TimelineDelegationTokenOperation op = - TimelineDelegationTokenOperation.GETDELEGATIONTOKEN; - Map params = new HashMap(); - params.put(TimelineAuthenticationConsts.OP_PARAM, op.toString()); - params.put(TimelineAuthenticationConsts.RENEWER_PARAM, renewer); - url = appendParams(url, params); - AuthenticatedURL aUrl = - new AuthenticatedURL(new TimelineAuthenticator()); - try { - HttpURLConnection conn = aUrl.openConnection(url, token); - conn.setRequestMethod(op.getHttpMethod()); - TimelineDelegationTokenResponse dtRes = validateAndParseResponse(conn); - if (!dtRes.getType().equals( - TimelineAuthenticationConsts.DELEGATION_TOKEN_URL)) { - throw new IOException("The response content is not expected: " - + dtRes.getContent()); - } - String tokenStr = dtRes.getContent().toString(); - Token dToken = - new Token(); - dToken.decodeFromUrlString(tokenStr); - return dToken; - } catch (AuthenticationException ex) { - throw new IOException(ex.toString(), ex); - } - } - - public static long renewDelegationToken(URL url, - AuthenticatedURL.Token token, - Token dToken) throws IOException { - Map params = new HashMap(); - params.put(TimelineAuthenticationConsts.OP_PARAM, - TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); - params.put(TimelineAuthenticationConsts.TOKEN_PARAM, - dToken.encodeToUrlString()); - url = appendParams(url, params); - AuthenticatedURL aUrl = - new AuthenticatedURL(new TimelineAuthenticator()); - try { - HttpURLConnection conn = aUrl.openConnection(url, token); - conn.setRequestMethod( - TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.getHttpMethod()); - TimelineDelegationTokenResponse dtRes = validateAndParseResponse(conn); - if (!dtRes.getType().equals( - TimelineAuthenticationConsts.DELEGATION_TOKEN_EXPIRATION_TIME)) { - throw new IOException("The response content is not expected: " - + dtRes.getContent()); - } - return Long.valueOf(dtRes.getContent().toString()); - } catch (AuthenticationException ex) { - throw new IOException(ex.toString(), ex); - } - } - - public static void cancelDelegationToken(URL url, - AuthenticatedURL.Token token, - Token dToken) throws IOException { - Map params = new HashMap(); - params.put(TimelineAuthenticationConsts.OP_PARAM, - TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); - params.put(TimelineAuthenticationConsts.TOKEN_PARAM, - dToken.encodeToUrlString()); - url = appendParams(url, params); - AuthenticatedURL aUrl = - new AuthenticatedURL(new TimelineAuthenticator()); - try { - HttpURLConnection conn = aUrl.openConnection(url, token); - conn.setRequestMethod(TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN - .getHttpMethod()); - validateAndParseResponse(conn); - } catch (AuthenticationException ex) { - throw new IOException(ex.toString(), ex); - } - } - - /** - * Convenience method that appends parameters an HTTP URL. - * - * @param url - * the url. - * @param params - * the query string parameters. - * - * @return a URL - * - * @throws IOException - * thrown if an IO error occurs. - */ - public static URL appendParams(URL url, Map params) - throws IOException { - StringBuilder sb = new StringBuilder(); - sb.append(url); - String separator = url.toString().contains("?") ? "&" : "?"; - for (Map.Entry entry : params.entrySet()) { - sb.append(separator).append(entry.getKey()).append("="). - append(URLEncoder.encode(entry.getValue(), "UTF8")); - separator = "&"; - } - return new URL(sb.toString()); - } - - /** - * Validates the response of an HttpURLConnection. If the current - * status code is not 200, it will throw an exception with a detail message - * using Server side error messages if available. Otherwise, - * {@link TimelineDelegationTokenResponse} will be parsed and returned. - * - * @param conn - * the HttpURLConnection. - * @return - * @throws IOException - * thrown if the current status code is not 200 or the JSON response - * cannot be parsed correctly - */ - private static TimelineDelegationTokenResponse validateAndParseResponse( - HttpURLConnection conn) throws IOException { - int status = conn.getResponseCode(); - JsonNode json = mapper.readTree(conn.getInputStream()); - if (status == HttpURLConnection.HTTP_OK) { - return mapper.readValue(json, TimelineDelegationTokenResponse.class); - } else { - // If the status code is not 200, some thing wrong should happen at the - // server side, the JSON content is going to contain exception details. - // We can use the JSON content to reconstruct the exception object. - try { - String message = - json.get(TimelineAuthenticationConsts.ERROR_MESSAGE_JSON) - .getTextValue(); - String exception = - json.get(TimelineAuthenticationConsts.ERROR_EXCEPTION_JSON) - .getTextValue(); - String className = - json.get(TimelineAuthenticationConsts.ERROR_CLASSNAME_JSON) - .getTextValue(); - - try { - ClassLoader cl = TimelineAuthenticator.class.getClassLoader(); - Class klass = cl.loadClass(className); - Constructor constr = klass.getConstructor(String.class); - throw (IOException) constr.newInstance(message); - } catch (IOException ex) { - throw ex; - } catch (Exception ex) { - throw new IOException(MessageFormat.format("{0} - {1}", exception, - message)); - } - } catch (IOException ex) { - if (ex.getCause() instanceof IOException) { - throw (IOException) ex.getCause(); - } - throw new IOException( - MessageFormat.format("HTTP status [{0}], {1}", - status, conn.getResponseMessage())); - } - } - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 5ffe17a..64cc041 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -18,43 +18,24 @@ package org.apache.hadoop.yarn.client.api.impl; -import java.io.File; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URI; -import java.net.URL; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import javax.ws.rs.core.MediaType; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector; -import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -63,8 +44,6 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; @Private @Unstable @@ -73,29 +52,16 @@ private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class); private static final String RESOURCE_URI_STR = "/ws/v1/timeline/"; private static final Joiner JOINER = Joiner.on(""); - private static Options opts; - static { - opts = new Options(); - opts.addOption("put", true, "Put the TimelineEntities in a JSON file"); - opts.getOption("put").setArgName("Path to the JSON file"); - opts.addOption("help", false, "Print usage"); - } private Client client; private URI resURI; private boolean isEnabled; - private TimelineAuthenticatedURLConnectionFactory urlFactory; public TimelineClientImpl() { super(TimelineClientImpl.class.getName()); ClientConfig cc = new DefaultClientConfig(); cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); - if (UserGroupInformation.isSecurityEnabled()) { - urlFactory = new TimelineAuthenticatedURLConnectionFactory(); - client = new Client(new URLConnectionClientHandler(urlFactory), cc); - } else { - client = Client.create(cc); - } + client = Client.create(cc); } protected void serviceInit(Configuration conf) throws Exception { @@ -117,9 +83,6 @@ protected void serviceInit(Configuration conf) throws Exception { YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), RESOURCE_URI_STR)); } - if (UserGroupInformation.isSecurityEnabled()) { - urlFactory.setService(TimelineUtils.buildTimelineTokenService(conf)); - } LOG.info("Timeline service address: " + resURI); } super.serviceInit(conf); @@ -161,13 +124,6 @@ public TimelinePutResponse putEntities( return resp.getEntity(TimelinePutResponse.class); } - @Override - public Token getDelegationToken( - String renewer) throws IOException, YarnException { - return TimelineAuthenticator.getDelegationToken(resURI.toURL(), - urlFactory.token, renewer); - } - @Private @VisibleForTesting public ClientResponse doPostingEntities(TimelineEntities entities) { @@ -177,138 +133,4 @@ public ClientResponse doPostingEntities(TimelineEntities entities) { .post(ClientResponse.class, entities); } - private static class TimelineAuthenticatedURLConnectionFactory - implements HttpURLConnectionFactory { - - private AuthenticatedURL.Token token; - private TimelineAuthenticator authenticator; - private Token dToken; - private Text service; - - public TimelineAuthenticatedURLConnectionFactory() { - token = new AuthenticatedURL.Token(); - authenticator = new TimelineAuthenticator(); - } - - @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { - try { - if (dToken == null) { - //TODO: need to take care of the renew case - dToken = selectToken(); - if (LOG.isDebugEnabled()) { - LOG.debug("Timeline delegation token: " + dToken.toString()); - } - } - if (dToken != null) { - Map params = new HashMap(); - TimelineAuthenticator.injectDelegationToken(params, dToken); - url = TimelineAuthenticator.appendParams(url, params); - if (LOG.isDebugEnabled()) { - LOG.debug("URL with delegation token: " + url); - } - } - return new AuthenticatedURL(authenticator).openConnection(url, token); - } catch (AuthenticationException e) { - LOG.error("Authentication failed when openning connection [" + url - + "] with token [" + token + "].", e); - throw new IOException(e); - } - } - - private Token selectToken() { - UserGroupInformation ugi; - try { - ugi = UserGroupInformation.getCurrentUser(); - } catch (IOException e) { - String msg = "Error when getting the current user"; - LOG.error(msg, e); - throw new YarnRuntimeException(msg, e); - } - TimelineDelegationTokenSelector tokenSelector = - new TimelineDelegationTokenSelector(); - return tokenSelector.selectToken( - service, ugi.getCredentials().getAllTokens()); - } - - public void setService(Text service) { - this.service = service; - } - - } - - public static void main(String[] argv) throws Exception { - CommandLine cliParser = new GnuParser().parse(opts, argv); - if (cliParser.hasOption("put")) { - String path = cliParser.getOptionValue("put"); - if (path != null && path.length() > 0) { - putTimelineEntitiesInJSONFile(path); - return; - } - } - printUsage(); - } - - /** - * Put timeline data in a JSON file via command line. - * - * @param path - * path to the {@link TimelineEntities} JSON file - */ - private static void putTimelineEntitiesInJSONFile(String path) { - File jsonFile = new File(path); - if (!jsonFile.exists()) { - System.out.println("Error: File [" + jsonFile.getAbsolutePath() - + "] doesn't exist"); - return; - } - ObjectMapper mapper = new ObjectMapper(); - YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); - TimelineEntities entities = null; - try { - entities = mapper.readValue(jsonFile, TimelineEntities.class); - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - e.printStackTrace(System.err); - return; - } - Configuration conf = new YarnConfiguration(); - TimelineClient client = TimelineClient.createTimelineClient(); - client.init(conf); - client.start(); - try { - if (UserGroupInformation.isSecurityEnabled() - && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) { - Token token = - client.getDelegationToken( - UserGroupInformation.getCurrentUser().getUserName()); - UserGroupInformation.getCurrentUser().addToken(token); - } - TimelinePutResponse response = client.putEntities( - entities.getEntities().toArray( - new TimelineEntity[entities.getEntities().size()])); - if (response.getErrors().size() == 0) { - System.out.println("Timeline data is successfully put"); - } else { - for (TimelinePutResponse.TimelinePutError error : response.getErrors()) { - System.out.println("TimelineEntity [" + error.getEntityType() + ":" + - error.getEntityId() + "] is not successfully put. Error code: " + - error.getErrorCode()); - } - } - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - e.printStackTrace(System.err); - } finally { - client.stop(); - } - } - - /** - * Helper function to print out usage - */ - private static void printUsage() { - new HelpFormatter().printHelp("TimelineClient", opts); - } - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index f1a3b6e..8a0348b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -30,13 +29,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.DataInputByteBuffer; -import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; @@ -70,7 +64,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; @@ -81,7 +74,6 @@ import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.api.AHSClient; -import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -90,10 +82,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; -import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; @@ -107,11 +97,8 @@ protected long submitPollIntervalMillis; private long asyncApiPollIntervalMillis; private long asyncApiPollTimeoutMillis; - private AHSClient historyClient; + protected AHSClient historyClient; private boolean historyServiceEnabled; - protected TimelineClient timelineClient; - protected Text timelineService; - protected boolean timelineServiceEnabled; private static final String ROOT = "root"; @@ -139,17 +126,10 @@ protected void serviceInit(Configuration conf) throws Exception { if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { historyServiceEnabled = true; - historyClient = AHSClient.createAHSClient(); - historyClient.init(conf); + historyClient = AHSClientImpl.createAHSClient(); + historyClient.init(getConfig()); } - if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { - timelineServiceEnabled = true; - timelineClient = TimelineClient.createTimelineClient(); - timelineClient.init(conf); - timelineService = TimelineUtils.buildTimelineTokenService(conf); - } super.serviceInit(conf); } @@ -161,9 +141,6 @@ protected void serviceStart() throws Exception { if (historyServiceEnabled) { historyClient.start(); } - if (timelineServiceEnabled) { - timelineClient.start(); - } } catch (IOException e) { throw new YarnRuntimeException(e); } @@ -178,9 +155,6 @@ protected void serviceStop() throws Exception { if (historyServiceEnabled) { historyClient.stop(); } - if (timelineServiceEnabled) { - timelineClient.stop(); - } super.serviceStop(); } @@ -215,12 +189,6 @@ public YarnClientApplication createApplication() Records.newRecord(SubmitApplicationRequest.class); request.setApplicationSubmissionContext(appContext); - // Automatically add the timeline DT into the CLC - // Only when the security and the timeline service are both enabled - if (isSecurityEnabled() && timelineServiceEnabled) { - addTimelineDelegationToken(appContext.getAMContainerSpec()); - } - //TODO: YARN-1763:Handle RM failovers during the submitApplication call. rmClient.submitApplication(request); @@ -270,48 +238,6 @@ public YarnClientApplication createApplication() return applicationId; } - private void addTimelineDelegationToken( - ContainerLaunchContext clc) throws YarnException, IOException { - org.apache.hadoop.security.token.Token timelineDelegationToken = - timelineClient.getDelegationToken( - UserGroupInformation.getCurrentUser().getUserName()); - if (timelineDelegationToken == null) { - return; - } - Credentials credentials = new Credentials(); - DataInputByteBuffer dibb = new DataInputByteBuffer(); - ByteBuffer tokens = clc.getTokens(); - if (tokens != null) { - dibb.reset(tokens); - credentials.readTokenStorageStream(dibb); - tokens.rewind(); - } - // If the timeline delegation token is already in the CLC, no need to add - // one more - for (org.apache.hadoop.security.token.Token token : credentials - .getAllTokens()) { - TokenIdentifier tokenIdentifier = token.decodeIdentifier(); - if (tokenIdentifier instanceof TimelineDelegationTokenIdentifier) { - return; - } - } - credentials.addToken(timelineService, timelineDelegationToken); - if (LOG.isDebugEnabled()) { - LOG.debug("Add timline delegation token into credentials: " - + timelineDelegationToken); - } - DataOutputBuffer dob = new DataOutputBuffer(); - credentials.writeTokenStorageToStream(dob); - tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - clc.setTokens(tokens); - } - - @Private - @VisibleForTesting - protected boolean isSecurityEnabled() { - return UserGroupInformation.isSecurityEnabled(); - } - @Override public void killApplication(ApplicationId applicationId) throws YarnException, IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 6407f7a..cfee6f7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -25,26 +25,19 @@ import static org.mockito.Mockito.when; import java.io.IOException; -import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; -import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.junit.Assert; + import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.DataInputByteBuffer; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; @@ -76,23 +69,19 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.util.Records; -import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; -import org.junit.Assert; import org.junit.Test; public class TestYarnClient { @@ -736,80 +725,4 @@ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout, IOUtils.closeQuietly(client); } } - - @Test - public void testAutomaticTimelineDelegationTokenLoading() - throws Exception { - Configuration conf = new YarnConfiguration(); - conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); - SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); - final Token dToken = - new Token(); - // crate a mock client - YarnClientImpl client = new YarnClientImpl() { - @Override - protected void serviceInit(Configuration conf) throws Exception { - if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { - timelineServiceEnabled = true; - timelineClient = mock(TimelineClient.class); - when(timelineClient.getDelegationToken(any(String.class))) - .thenReturn(dToken); - timelineClient.init(getConfig()); - timelineService = TimelineUtils.buildTimelineTokenService(getConfig()); - } - this.setConfig(conf); - } - - @Override - protected void serviceStart() throws Exception { - rmClient = mock(ApplicationClientProtocol.class); - } - - @Override - protected void serviceStop() throws Exception { - } - - @Override - public ApplicationReport getApplicationReport(ApplicationId appId) { - ApplicationReport report = mock(ApplicationReport.class); - when(report.getYarnApplicationState()) - .thenReturn(YarnApplicationState.SUBMITTED); - return report; - } - - @Override - public boolean isSecurityEnabled() { - return true; - } - }; - client.init(conf); - client.start(); - ApplicationSubmissionContext context = - mock(ApplicationSubmissionContext.class); - ApplicationId applicationId = ApplicationId.newInstance(0, 1); - when(context.getApplicationId()).thenReturn(applicationId); - DataOutputBuffer dob = new DataOutputBuffer(); - Credentials credentials = new Credentials(); - credentials.writeTokenStorageToStream(dob); - ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - ContainerLaunchContext clc = ContainerLaunchContext.newInstance( - null, null, null, null, tokens, null); - when(context.getAMContainerSpec()).thenReturn(clc); - client.submitApplication(context); - // Check whether token is added or not - credentials = new Credentials(); - DataInputByteBuffer dibb = new DataInputByteBuffer(); - tokens = clc.getTokens(); - if (tokens != null) { - dibb.reset(tokens); - credentials.readTokenStorageStream(dibb); - tokens.rewind(); - } - Collection> dTokens = - credentials.getAllTokens(); - Assert.assertEquals(1, dTokens.size()); - Assert.assertEquals(dToken, dTokens.iterator().next()); - client.stop(); - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java deleted file mode 100644 index d2b1358..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.security.client; - -import java.lang.annotation.Annotation; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.KerberosInfo; -import org.apache.hadoop.security.SecurityInfo; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenInfo; -import org.apache.hadoop.security.token.TokenSelector; -import org.apache.hadoop.yarn.api.ApplicationHistoryProtocolPB; -import org.apache.hadoop.yarn.conf.YarnConfiguration; - - -@Public -@Unstable -public class ClientTimelineSecurityInfo extends SecurityInfo { - @Override - public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { - if (!protocol - .equals(ApplicationHistoryProtocolPB.class)) { - return null; - } - return new KerberosInfo() { - - @Override - public Class annotationType() { - return null; - } - - @Override - public String serverPrincipal() { - return YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL; - } - - @Override - public String clientPrincipal() { - return null; - } - }; - } - - @Override - public TokenInfo getTokenInfo(Class protocol, Configuration conf) { - if (!protocol - .equals(ApplicationHistoryProtocolPB.class)) { - return null; - } - return new TokenInfo() { - - @Override - public Class annotationType() { - return null; - } - - @Override - public Class> - value() { - return TimelineDelegationTokenSelector.class; - } - }; - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java deleted file mode 100644 index e89e1a6..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.security.client; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -/** - * The constants that are going to be used by the timeline Kerberos + delegation - * token authentication. - */ - -@Private -@Unstable -public class TimelineAuthenticationConsts { - - public static final String ERROR_EXCEPTION_JSON = "exception"; - public static final String ERROR_CLASSNAME_JSON = "javaClassName"; - public static final String ERROR_MESSAGE_JSON = "message"; - - public static final String OP_PARAM = "op"; - public static final String DELEGATION_PARAM = "delegation"; - public static final String TOKEN_PARAM = "token"; - public static final String RENEWER_PARAM = "renewer"; - public static final String DELEGATION_TOKEN_URL = "url"; - public static final String DELEGATION_TOKEN_EXPIRATION_TIME = - "expirationTime"; -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java deleted file mode 100644 index 82e0d69..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.security.client; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; - -@Public -@Unstable -public class TimelineDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier { - - public static final Text KIND_NAME = new Text("TIMELINE_DELEGATION_TOKEN"); - - public TimelineDelegationTokenIdentifier() { - - } - - /** - * Create a new timeline delegation token identifier - * - * @param owner the effective username of the token owner - * @param renewer the username of the renewer - * @param realUser the real username of the token owner - */ - public TimelineDelegationTokenIdentifier(Text owner, Text renewer, - Text realUser) { - super(owner, renewer, realUser); - } - - @Override - public Text getKind() { - return KIND_NAME; - } - - @InterfaceAudience.Private - public static class Renewer extends Token.TrivialRenewer { - @Override - protected Text getKind() { - return KIND_NAME; - } - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java deleted file mode 100644 index 33ab178..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.security.client; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPut; - -/** - * DelegationToken operations. - */ -@Unstable -@Private -public enum TimelineDelegationTokenOperation { - // TODO: need think about which ops can be done without kerberos - // credentials, for safety, we enforces all need kerberos credentials now. - GETDELEGATIONTOKEN(HttpGet.METHOD_NAME, true), - RENEWDELEGATIONTOKEN(HttpPut.METHOD_NAME, true), - CANCELDELEGATIONTOKEN(HttpPut.METHOD_NAME, true); - - private String httpMethod; - private boolean requiresKerberosCredentials; - - private TimelineDelegationTokenOperation(String httpMethod, - boolean requiresKerberosCredentials) { - this.httpMethod = httpMethod; - this.requiresKerberosCredentials = requiresKerberosCredentials; - } - - public String getHttpMethod() { - return httpMethod; - } - - public boolean requiresKerberosCredentials() { - return requiresKerberosCredentials; - } - -} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java deleted file mode 100644 index df1e84d..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.security.client; - -import java.util.Collection; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenSelector; - -@Public -@Unstable -public class TimelineDelegationTokenSelector - implements TokenSelector { - - private static final Log LOG = LogFactory - .getLog(TimelineDelegationTokenSelector.class); - - @SuppressWarnings("unchecked") - public Token selectToken(Text service, - Collection> tokens) { - if (service == null) { - return null; - } - if (LOG.isDebugEnabled()) { - LOG.debug("Looking for a token with service " + service.toString()); - } - for (Token token : tokens) { - if (LOG.isDebugEnabled()) { - LOG.debug("Token kind is " + token.getKind().toString() - + " and the token's service name is " + token.getService()); - } - if (TimelineDelegationTokenIdentifier.KIND_NAME.equals(token.getKind()) - && service.equals(token.getService())) { - return (Token) token; - } - } - return null; - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java index 02b5eb4..324c6f6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java @@ -19,18 +19,15 @@ package org.apache.hadoop.yarn.util.timeline; import java.io.IOException; -import java.net.InetSocketAddress; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.map.AnnotationIntrospector; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; +import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; /** * The helper class for the timeline module. @@ -44,7 +41,9 @@ static { mapper = new ObjectMapper(); - YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); + AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); + mapper.setAnnotationIntrospector(introspector); + mapper.setSerializationInclusion(Inclusion.NON_NULL); } /** @@ -83,26 +82,4 @@ public static String dumpTimelineRecordtoJSON(Object o, boolean pretty) } } - public static InetSocketAddress getTimelineTokenServiceAddress( - Configuration conf) { - InetSocketAddress timelineServiceAddr = null; - if (YarnConfiguration.useHttps(conf)) { - timelineServiceAddr = conf.getSocketAddr( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT); - } else { - timelineServiceAddr = conf.getSocketAddr( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT); - } - return timelineServiceAddr; - } - - public static Text buildTimelineTokenService(Configuration conf) { - InetSocketAddress timelineServiceAddr = - getTimelineTokenServiceAddress(conf); - return SecurityUtil.buildTokenService(timelineServiceAddr); - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java index 3cc1aec..ac0ea7f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java @@ -49,14 +49,9 @@ public YarnJacksonJaxbJsonProvider() { @Override public ObjectMapper locateMapper(Class type, MediaType mediaType) { ObjectMapper mapper = super.locateMapper(type, mediaType); - configObjectMapper(mapper); - return mapper; - } - - public static void configObjectMapper(ObjectMapper mapper) { AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); mapper.setAnnotationIntrospector(introspector); mapper.setSerializationInclusion(Inclusion.NON_NULL); + return mapper; } - } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo index d90a267..babc2fb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -12,7 +12,6 @@ # limitations under the License. # org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo org.apache.hadoop.yarn.security.SchedulerSecurityInfo org.apache.hadoop.yarn.security.admin.AdminSecurityInfo diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index a4ad548..d860461 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -15,5 +15,4 @@ org.apache.hadoop.yarn.security.ContainerTokenIdentifier org.apache.hadoop.yarn.security.AMRMTokenIdentifier org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier org.apache.hadoop.yarn.security.NMTokenIdentifier diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index 9fcfa19..dd0b2c4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -14,4 +14,3 @@ org.apache.hadoop.yarn.security.AMRMTokenIdentifier$Renewer org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer -org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier$Renewer diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a528095..be831ad 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -309,14 +309,17 @@ Number of times RM tries to connect to ZooKeeper. yarn.resourcemanager.zk-num-retries - 500 + 1000 Retry interval in milliseconds when connecting to ZooKeeper. + When HA is enabled, the value here is NOT used. It is generated + automatically from yarn.resourcemanager.zk-timeout-ms and + yarn.resourcemanager.zk-num-retries. yarn.resourcemanager.zk-retry-interval-ms - 2000 + 1000 @@ -1195,18 +1198,6 @@ - The Kerberos principal for the timeline server. - yarn.timeline-service.principal - - - - - The Kerberos keytab for the timeline server. - yarn.timeline-service.keytab - /etc/krb5.keytab - - - Indicate to ResourceManager as well as clients whether history-service is enabled or not. If enabled, ResourceManager starts recording historical data that ApplicationHistory service can consume. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 5e1277f..731ae14 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -18,17 +18,12 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; -import java.io.IOException; -import java.net.InetSocketAddress; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; import org.apache.hadoop.util.ExitUtil; @@ -38,11 +33,8 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineAuthenticationFilterInitializer; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -60,12 +52,10 @@ private static final Log LOG = LogFactory .getLog(ApplicationHistoryServer.class); - protected ApplicationHistoryClientService ahsClientService; - protected ApplicationHistoryManager historyManager; - protected TimelineStore timelineStore; - protected TimelineDelegationTokenSecretManagerService secretManagerService; - protected TimelineACLsManager timelineACLsManager; - protected WebApp webApp; + ApplicationHistoryClientService ahsClientService; + ApplicationHistoryManager historyManager; + TimelineStore timelineStore; + private WebApp webApp; public ApplicationHistoryServer() { super(ApplicationHistoryServer.class.getName()); @@ -79,22 +69,13 @@ protected void serviceInit(Configuration conf) throws Exception { addService((Service) historyManager); timelineStore = createTimelineStore(conf); addIfService(timelineStore); - secretManagerService = createTimelineDelegationTokenSecretManagerService(conf); - addService(secretManagerService); - timelineACLsManager = createTimelineACLsManager(conf); - - DefaultMetricsSystem.initialize("ApplicationHistoryServer"); - JvmMetrics.initSingleton("ApplicationHistoryServer", null); super.serviceInit(conf); } @Override protected void serviceStart() throws Exception { - try { - doSecureLogin(getConfig()); - } catch(IOException ie) { - throw new YarnRuntimeException("Failed to login", ie); - } + DefaultMetricsSystem.initialize("ApplicationHistoryServer"); + JvmMetrics.initSingleton("ApplicationHistoryServer", null); startWebApp(); super.serviceStart(); @@ -167,48 +148,21 @@ protected TimelineStore createTimelineStore( TimelineStore.class), conf); } - protected TimelineDelegationTokenSecretManagerService - createTimelineDelegationTokenSecretManagerService(Configuration conf) { - return new TimelineDelegationTokenSecretManagerService(); - } - - protected TimelineACLsManager createTimelineACLsManager(Configuration conf) { - return new TimelineACLsManager(conf); - } - protected void startWebApp() { - Configuration conf = getConfig(); - // Play trick to make the customized filter will only be loaded by the - // timeline server when security is enabled and Kerberos authentication - // is used. - if (UserGroupInformation.isSecurityEnabled() - && conf - .get(TimelineAuthenticationFilterInitializer.PREFIX + "type", "") - .equals("kerberos")) { - String initializers = conf.get("hadoop.http.filter.initializers"); - initializers = - initializers == null || initializers.length() == 0 ? "" : "," - + initializers; - if (!initializers.contains( - TimelineAuthenticationFilterInitializer.class.getName())) { - conf.set("hadoop.http.filter.initializers", - TimelineAuthenticationFilterInitializer.class.getName() - + initializers); - } - } - String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(conf); + String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); LOG.info("Instantiating AHSWebApp at " + bindAddress); try { - AHSWebApp ahsWebApp = AHSWebApp.getInstance(); - ahsWebApp.setApplicationHistoryManager(historyManager); - ahsWebApp.setTimelineStore(timelineStore); - ahsWebApp.setTimelineDelegationTokenSecretManagerService(secretManagerService); - ahsWebApp.setTimelineACLsManager(timelineACLsManager); webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, - ahsClientService, "ws") - .with(conf).at(bindAddress).start(ahsWebApp); + ahsClientService, "ws") + .with(getConfig()) + .withHttpSpnegoPrincipalKey( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY) + .withHttpSpnegoKeytabKey( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) + .at(bindAddress) + .start(new AHSWebApp(historyManager, timelineStore)); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); @@ -223,22 +177,4 @@ protected void startWebApp() { public TimelineStore getTimelineStore() { return timelineStore; } - - private void doSecureLogin(Configuration conf) throws IOException { - InetSocketAddress socAddr = getBindAddress(conf); - SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB, - YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, socAddr.getHostName()); - } - - /** - * Retrieve the timeline server bind address from configuration - * - * @param conf - * @return InetSocketAddress - */ - private static InetSocketAddress getBindAddress(Configuration conf) { - return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java index 51bffda..edd4842 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java @@ -18,9 +18,6 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -39,16 +36,13 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.collections.map.LRUMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.service.AbstractService; @@ -56,8 +50,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.fusesource.leveldbjni.JniDBFactory; @@ -68,7 +62,8 @@ import org.iq80.leveldb.WriteBatch; import org.iq80.leveldb.WriteOptions; -import com.google.common.annotations.VisibleForTesting; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; /** *

An implementation of an application timeline store backed by leveldb.

@@ -125,9 +120,7 @@ private static final Log LOG = LogFactory .getLog(LeveldbTimelineStore.class); - @Private - @VisibleForTesting - static final String FILENAME = "leveldb-timeline-store.ldb"; + private static final String FILENAME = "leveldb-timeline-store.ldb"; private static final byte[] START_TIME_LOOKUP_PREFIX = "k".getBytes(); private static final byte[] ENTITY_ENTRY_PREFIX = "e".getBytes(); @@ -142,11 +135,6 @@ private static final byte[] EMPTY_BYTES = new byte[0]; - @Private - @VisibleForTesting - static final FsPermission LEVELDB_DIR_UMASK = FsPermission - .createImmutable((short) 0700); - private Map startTimeWriteCache; private Map startTimeReadCache; @@ -176,23 +164,16 @@ protected void serviceInit(Configuration conf) throws Exception { YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); JniDBFactory factory = new JniDBFactory(); - Path dbPath = new Path( - conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH), FILENAME); - FileSystem localFS = null; - try { - localFS = FileSystem.getLocal(conf); - if (!localFS.exists(dbPath)) { - if (!localFS.mkdirs(dbPath)) { - throw new IOException("Couldn't create directory for leveldb " + - "timeline store " + dbPath); - } - localFS.setPermission(dbPath, LEVELDB_DIR_UMASK); + String path = conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH); + File p = new File(path); + if (!p.exists()) { + if (!p.mkdirs()) { + throw new IOException("Couldn't create directory for leveldb " + + "timeline store " + path); } - } finally { - IOUtils.cleanup(LOG, localFS); } - LOG.info("Using leveldb path " + dbPath); - db = factory.open(new File(dbPath.toString()), options); + LOG.info("Using leveldb path " + path); + db = factory.open(new File(path, FILENAME), options); startTimeWriteCache = Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize( conf))); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java index 06f3d60..86ac1f8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -39,8 +40,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; /** @@ -313,29 +314,15 @@ private static TimelineEntity maskFields( entityToReturn.setEntityId(entity.getEntityId()); entityToReturn.setEntityType(entity.getEntityType()); entityToReturn.setStartTime(entity.getStartTime()); - // Deep copy - if (fields.contains(Field.EVENTS)) { - entityToReturn.addEvents(entity.getEvents()); - } else if (fields.contains(Field.LAST_EVENT_ONLY)) { - entityToReturn.addEvent(entity.getEvents().get(0)); - } else { - entityToReturn.setEvents(null); - } - if (fields.contains(Field.RELATED_ENTITIES)) { - entityToReturn.addRelatedEntities(entity.getRelatedEntities()); - } else { - entityToReturn.setRelatedEntities(null); - } - if (fields.contains(Field.PRIMARY_FILTERS)) { - entityToReturn.addPrimaryFilters(entity.getPrimaryFilters()); - } else { - entityToReturn.setPrimaryFilters(null); - } - if (fields.contains(Field.OTHER_INFO)) { - entityToReturn.addOtherInfo(entity.getOtherInfo()); - } else { - entityToReturn.setOtherInfo(null); - } + entityToReturn.setEvents(fields.contains(Field.EVENTS) ? + entity.getEvents() : fields.contains(Field.LAST_EVENT_ONLY) ? + Arrays.asList(entity.getEvents().get(0)) : null); + entityToReturn.setRelatedEntities(fields.contains(Field.RELATED_ENTITIES) ? + entity.getRelatedEntities() : null); + entityToReturn.setPrimaryFilters(fields.contains(Field.PRIMARY_FILTERS) ? + entity.getPrimaryFilters() : null); + entityToReturn.setOtherInfo(fields.contains(Field.OTHER_INFO) ? + entity.getOtherInfo() : null); return entityToReturn; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java index fc02873b..6b50d83 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java @@ -18,25 +18,12 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -@Private -@Unstable +@InterfaceAudience.Private +@InterfaceStability.Unstable public interface TimelineStore extends Service, TimelineReader, TimelineWriter { - - /** - * The system filter which will be automatically added to a - * {@link TimelineEntity}'s primary filter section when storing the entity. - * The filter key is case sensitive. Users are supposed not to use the key - * reserved by the timeline system. - */ - @Private - enum SystemFilter { - ENTITY_OWNER - } - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java deleted file mode 100644 index 8009b39..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; - -import java.io.IOException; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.security.AdminACLsManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore.SystemFilter; - -import com.google.common.annotations.VisibleForTesting; - -/** - * TimelineACLsManager check the entity level timeline data access. - */ -@Private -public class TimelineACLsManager { - - private static final Log LOG = LogFactory.getLog(TimelineACLsManager.class); - - private AdminACLsManager adminAclsManager; - - public TimelineACLsManager(Configuration conf) { - this.adminAclsManager = new AdminACLsManager(conf); - } - - public boolean checkAccess(UserGroupInformation callerUGI, - TimelineEntity entity) throws YarnException, IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Verifying the access of " + callerUGI.getShortUserName() - + " on the timeline entity " - + new EntityIdentifier(entity.getEntityId(), entity.getEntityType())); - } - - if (!adminAclsManager.areACLsEnabled()) { - return true; - } - - Set values = - entity.getPrimaryFilters().get( - SystemFilter.ENTITY_OWNER.toString()); - if (values == null || values.size() != 1) { - throw new YarnException("Owner information of the timeline entity " - + new EntityIdentifier(entity.getEntityId(), entity.getEntityType()) - + " is corrupted."); - } - String owner = values.iterator().next().toString(); - // TODO: Currently we just check the user is the admin or the timeline - // entity owner. In the future, we need to check whether the user is in the - // allowed user/group list - if (callerUGI != null - && (adminAclsManager.isAdmin(callerUGI) || - callerUGI.getShortUserName().equals(owner))) { - return true; - } - return false; - } - - @Private - @VisibleForTesting - public AdminACLsManager - setAdminACLsManager(AdminACLsManager adminAclsManager) { - AdminACLsManager oldAdminACLsManager = this.adminAclsManager; - this.adminAclsManager = adminAclsManager; - return oldAdminACLsManager; - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java deleted file mode 100644 index 53ef1ed..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; - -import java.util.Properties; - -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; - -@Private -@Unstable -public class TimelineAuthenticationFilter extends AuthenticationFilter { - - @Override - protected Properties getConfiguration(String configPrefix, - FilterConfig filterConfig) throws ServletException { - // In yarn-site.xml, we can simply set type to "kerberos". However, we need - // to replace the name here to use the customized Kerberos + DT service - // instead of the standard Kerberos handler. - Properties properties = super.getConfiguration(configPrefix, filterConfig); - if (properties.getProperty(AUTH_TYPE).equals("kerberos")) { - properties.setProperty( - AUTH_TYPE, TimelineClientAuthenticationService.class.getName()); - } - return properties; - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java deleted file mode 100644 index e3c3032..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; - -import java.io.FileReader; -import java.io.IOException; -import java.io.Reader; -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.FilterContainer; -import org.apache.hadoop.http.FilterInitializer; -import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.security.SecurityUtil; - -/** - *

- * Initializes {@link TimelineAuthenticationFilter} which provides support for - * Kerberos HTTP SPNEGO authentication. - *

- *

- * It enables Kerberos HTTP SPNEGO plus delegation token authentication for the - * timeline server. - *

- * Refer to the core-default.xml file, after the comment 'HTTP - * Authentication' for details on the configuration options. All related - * configuration properties have 'hadoop.http.authentication.' as prefix. - */ -public class TimelineAuthenticationFilterInitializer extends FilterInitializer { - - /** - * The configuration prefix of timeline Kerberos + DT authentication - */ - public static final String PREFIX = "yarn.timeline-service.http.authentication."; - - private static final String SIGNATURE_SECRET_FILE = - TimelineAuthenticationFilter.SIGNATURE_SECRET + ".file"; - - /** - *

- * Initializes {@link TimelineAuthenticationFilter} - *

- *

- * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN - * configuration properties prefixed with - * "yarn.timeline-service.authentication." - *

- * - * @param container - * The filter container - * @param conf - * Configuration for run-time parameters - */ - @Override - public void initFilter(FilterContainer container, Configuration conf) { - Map filterConfig = new HashMap(); - - // setting the cookie path to root '/' so it is used for all resources. - filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/"); - - for (Map.Entry entry : conf) { - String name = entry.getKey(); - if (name.startsWith(PREFIX)) { - String value = conf.get(name); - name = name.substring(PREFIX.length()); - filterConfig.put(name, value); - } - } - - String signatureSecretFile = filterConfig.get(SIGNATURE_SECRET_FILE); - if (signatureSecretFile != null) { - try { - StringBuilder secret = new StringBuilder(); - Reader reader = new FileReader(signatureSecretFile); - int c = reader.read(); - while (c > -1) { - secret.append((char) c); - c = reader.read(); - } - reader.close(); - filterConfig - .put(TimelineAuthenticationFilter.SIGNATURE_SECRET, - secret.toString()); - } catch (IOException ex) { - throw new RuntimeException( - "Could not read HTTP signature secret file: " - + signatureSecretFile); - } - } - - // Resolve _HOST into bind address - String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); - String principal = - filterConfig.get(TimelineClientAuthenticationService.PRINCIPAL); - if (principal != null) { - try { - principal = SecurityUtil.getServerPrincipal(principal, bindAddress); - } catch (IOException ex) { - throw new RuntimeException( - "Could not resolve Kerberos principal name: " + ex.toString(), ex); - } - filterConfig.put(TimelineClientAuthenticationService.PRINCIPAL, - principal); - } - - container.addGlobalFilter("Timeline Authentication Filter", - TimelineAuthenticationFilter.class.getName(), - filterConfig); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java deleted file mode 100644 index f11633d..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; - -import java.io.IOException; -import java.io.Writer; -import java.text.MessageFormat; -import java.util.HashSet; -import java.util.Set; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.core.MediaType; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.server.AuthenticationToken; -import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenOperation; -import org.apache.hadoop.yarn.security.client.TimelineAuthenticationConsts; -import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; -import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.codehaus.jackson.map.ObjectMapper; - -/** - * Server side AuthenticationHandler that authenticates requests - * using the incoming delegation token as a 'delegation' query string parameter. - *

- * If not delegation token is present in the request it delegates to the - * {@link KerberosAuthenticationHandler} - */ -@Private -@Unstable -public class TimelineClientAuthenticationService - extends KerberosAuthenticationHandler { - - public static final String TYPE = "kerberos-dt"; - private static final Set DELEGATION_TOKEN_OPS = new HashSet(); - private static final String OP_PARAM = "op"; - private static final String ENTER = System.getProperty("line.separator"); - - private ObjectMapper mapper; - - static { - DELEGATION_TOKEN_OPS.add( - TimelineDelegationTokenOperation.GETDELEGATIONTOKEN.toString()); - DELEGATION_TOKEN_OPS.add( - TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); - DELEGATION_TOKEN_OPS.add( - TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); - } - - public TimelineClientAuthenticationService() { - super(); - mapper = new ObjectMapper(); - YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); - } - - /** - * Returns authentication type of the handler. - * - * @return delegationtoken-kerberos - */ - @Override - public String getType() { - return TYPE; - } - - @Override - public boolean managementOperation(AuthenticationToken token, - HttpServletRequest request, HttpServletResponse response) - throws IOException, AuthenticationException { - boolean requestContinues = true; - String op = request.getParameter(OP_PARAM); - op = (op != null) ? op.toUpperCase() : null; - if (DELEGATION_TOKEN_OPS.contains(op) && - !request.getMethod().equals("OPTIONS")) { - TimelineDelegationTokenOperation dtOp = - TimelineDelegationTokenOperation.valueOf(op); - if (dtOp.getHttpMethod().equals(request.getMethod())) { - if (dtOp.requiresKerberosCredentials() && token == null) { - response.sendError(HttpServletResponse.SC_UNAUTHORIZED, - MessageFormat.format( - "Operation [{0}] requires SPNEGO authentication established", - dtOp)); - requestContinues = false; - } else { - TimelineDelegationTokenSecretManagerService secretManager = - AHSWebApp.getInstance() - .getTimelineDelegationTokenSecretManagerService(); - try { - TimelineDelegationTokenResponse res = null; - switch (dtOp) { - case GETDELEGATIONTOKEN: - UserGroupInformation ownerUGI = - UserGroupInformation.createRemoteUser(token.getUserName()); - String renewerParam = - request - .getParameter(TimelineAuthenticationConsts.RENEWER_PARAM); - if (renewerParam == null) { - renewerParam = token.getUserName(); - } - Token dToken = - secretManager.createToken(ownerUGI, renewerParam); - res = new TimelineDelegationTokenResponse(); - res.setType(TimelineAuthenticationConsts.DELEGATION_TOKEN_URL); - res.setContent(dToken.encodeToUrlString()); - break; - case RENEWDELEGATIONTOKEN: - case CANCELDELEGATIONTOKEN: - String tokenParam = - request - .getParameter(TimelineAuthenticationConsts.TOKEN_PARAM); - if (tokenParam == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST, - MessageFormat - .format( - "Operation [{0}] requires the parameter [{1}]", - dtOp, - TimelineAuthenticationConsts.TOKEN_PARAM)); - requestContinues = false; - } else { - if (dtOp == TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN) { - Token dt = - new Token(); - dt.decodeFromUrlString(tokenParam); - secretManager.cancelToken(dt, token.getUserName()); - } else { - Token dt = - new Token(); - dt.decodeFromUrlString(tokenParam); - long expirationTime = - secretManager.renewToken(dt, token.getUserName()); - res = new TimelineDelegationTokenResponse(); - res.setType(TimelineAuthenticationConsts.DELEGATION_TOKEN_EXPIRATION_TIME); - res.setContent(expirationTime); - } - } - break; - } - if (requestContinues) { - response.setStatus(HttpServletResponse.SC_OK); - if (res != null) { - response.setContentType(MediaType.APPLICATION_JSON); - Writer writer = response.getWriter(); - mapper.writeValue(writer, res); - writer.write(ENTER); - writer.flush(); - - } - requestContinues = false; - } - } catch (IOException e) { - throw new AuthenticationException(e.toString(), e); - } - } - } else { - response - .sendError( - HttpServletResponse.SC_BAD_REQUEST, - MessageFormat - .format( - "Wrong HTTP method [{0}] for operation [{1}], it should be [{2}]", - request.getMethod(), dtOp, dtOp.getHttpMethod())); - requestContinues = false; - } - } - return requestContinues; - } - - /** - * Authenticates a request looking for the delegation - * query-string parameter and verifying it is a valid token. If there is not - * delegation query-string parameter, it delegates the - * authentication to the {@link KerberosAuthenticationHandler} unless it is - * disabled. - * - * @param request - * the HTTP client request. - * @param response - * the HTTP client response. - * - * @return the authentication token for the authenticated request. - * @throws IOException - * thrown if an IO error occurred. - * @throws AuthenticationException - * thrown if the authentication failed. - */ - @Override - public AuthenticationToken authenticate(HttpServletRequest request, - HttpServletResponse response) - throws IOException, AuthenticationException { - AuthenticationToken token; - String delegationParam = - request - .getParameter(TimelineAuthenticationConsts.DELEGATION_PARAM); - if (delegationParam != null) { - Token dt = - new Token(); - dt.decodeFromUrlString(delegationParam); - TimelineDelegationTokenSecretManagerService secretManager = - AHSWebApp.getInstance() - .getTimelineDelegationTokenSecretManagerService(); - UserGroupInformation ugi = secretManager.verifyToken(dt); - final String shortName = ugi.getShortUserName(); - // creating a ephemeral token - token = new AuthenticationToken(shortName, ugi.getUserName(), getType()); - token.setExpires(0); - } else { - token = super.authenticate(request, response); - } - return token; - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java deleted file mode 100644 index 2808dac..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.net.InetSocketAddress; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; -import org.apache.hadoop.yarn.util.timeline.TimelineUtils; - -/** - * The service wrapper of {@link TimelineDelegationTokenSecretManager} - */ -@Private -@Unstable -public class TimelineDelegationTokenSecretManagerService extends AbstractService { - - private TimelineDelegationTokenSecretManager secretManager = null; - private InetSocketAddress serviceAddr = null; - - public TimelineDelegationTokenSecretManagerService() { - super(TimelineDelegationTokenSecretManagerService.class.getName()); - } - - @Override - protected void serviceInit(Configuration conf) throws Exception { - long secretKeyInterval = - conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY, - YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); - long tokenMaxLifetime = - conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, - YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); - long tokenRenewInterval = - conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, - YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); - secretManager = new TimelineDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, - 3600000); - secretManager.startThreads(); - - serviceAddr = TimelineUtils.getTimelineTokenServiceAddress(getConfig()); - super.init(conf); - } - - @Override - protected void serviceStop() throws Exception { - secretManager.stopThreads(); - super.stop(); - } - - /** - * Creates a delegation token. - * - * @param ugi UGI creating the token. - * @param renewer token renewer. - * @return new delegation token. - * @throws IOException thrown if the token could not be created. - */ - public Token createToken( - UserGroupInformation ugi, String renewer) throws IOException { - renewer = (renewer == null) ? ugi.getShortUserName() : renewer; - String user = ugi.getUserName(); - Text owner = new Text(user); - Text realUser = null; - if (ugi.getRealUser() != null) { - realUser = new Text(ugi.getRealUser().getUserName()); - } - TimelineDelegationTokenIdentifier tokenIdentifier = - new TimelineDelegationTokenIdentifier(owner, new Text(renewer), realUser); - Token token = - new Token(tokenIdentifier, secretManager); - SecurityUtil.setTokenService(token, serviceAddr); - return token; - } - - /** - * Renews a delegation token. - * - * @param token delegation token to renew. - * @param renewer token renewer. - * @throws IOException thrown if the token could not be renewed. - */ - public long renewToken(Token token, - String renewer) throws IOException { - return secretManager.renewToken(token, renewer); - } - - /** - * Cancels a delegation token. - * - * @param token delegation token to cancel. - * @param canceler token canceler. - * @throws IOException thrown if the token could not be canceled. - */ - public void cancelToken(Token token, - String canceler) throws IOException { - secretManager.cancelToken(token, canceler); - } - - /** - * Verifies a delegation token. - * - * @param token delegation token to verify. - * @return the UGI for the token. - * @throws IOException thrown if the token could not be verified. - */ - public UserGroupInformation verifyToken(Token token) - throws IOException { - ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); - DataInputStream dis = new DataInputStream(buf); - TimelineDelegationTokenIdentifier id = new TimelineDelegationTokenIdentifier(); - try { - id.readFields(dis); - secretManager.verifyToken(id, token.getPassword()); - } finally { - dis.close(); - } - return id.getUser(); - } - - /** - * Create a timeline secret manager - * - * @param delegationKeyUpdateInterval - * the number of seconds for rolling new secret keys. - * @param delegationTokenMaxLifetime - * the maximum lifetime of the delegation tokens - * @param delegationTokenRenewInterval - * how often the tokens must be renewed - * @param delegationTokenRemoverScanInterval - * how often the tokens are scanned for expired tokens - */ - @Private - @Unstable - public static class TimelineDelegationTokenSecretManager extends - AbstractDelegationTokenSecretManager { - - public TimelineDelegationTokenSecretManager(long delegationKeyUpdateInterval, - long delegationTokenMaxLifetime, long delegationTokenRenewInterval, - long delegationTokenRemoverScanInterval) { - super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, - delegationTokenRenewInterval, delegationTokenRemoverScanInterval); - } - - @Override - public TimelineDelegationTokenIdentifier createIdentifier() { - return new TimelineDelegationTokenIdentifier(); - } - - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 17b1e62..93065b3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -19,80 +19,25 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnWebParams; -import com.google.common.annotations.VisibleForTesting; - public class AHSWebApp extends WebApp implements YarnWebParams { - private ApplicationHistoryManager applicationHistoryManager; - private TimelineStore timelineStore; - private TimelineDelegationTokenSecretManagerService secretManagerService; - private TimelineACLsManager timelineACLsManager; - - private static AHSWebApp instance = null; - - public static AHSWebApp getInstance() { - if (instance == null) { - instance = new AHSWebApp(); - } - return instance; - } - - @Private - @VisibleForTesting - public static void resetInstance() { - instance = null; - } - - private AHSWebApp() { - - } - - public ApplicationHistoryManager getApplicationHistoryManager() { - return applicationHistoryManager; - } + private final ApplicationHistoryManager applicationHistoryManager; + private final TimelineStore timelineStore; - public void setApplicationHistoryManager( - ApplicationHistoryManager applicationHistoryManager) { + public AHSWebApp(ApplicationHistoryManager applicationHistoryManager, + TimelineStore timelineStore) { this.applicationHistoryManager = applicationHistoryManager; - } - - public TimelineStore getTimelineStore() { - return timelineStore; - } - - public void setTimelineStore(TimelineStore timelineStore) { this.timelineStore = timelineStore; } - public TimelineDelegationTokenSecretManagerService - getTimelineDelegationTokenSecretManagerService() { - return secretManagerService; - } - - public void setTimelineDelegationTokenSecretManagerService( - TimelineDelegationTokenSecretManagerService secretManagerService) { - this.secretManagerService = secretManagerService; - } - - public TimelineACLsManager getTimelineACLsManager() { - return timelineACLsManager; - } - - public void setTimelineACLsManager(TimelineACLsManager timelineACLsManager) { - this.timelineACLsManager = timelineACLsManager; - } - @Override public void setup() { bind(YarnJacksonJaxbJsonProvider.class); @@ -101,9 +46,6 @@ public void setup() { bind(GenericExceptionHandler.class); bind(ApplicationContext.class).toInstance(applicationHistoryManager); bind(TimelineStore.class).toInstance(timelineStore); - bind(TimelineDelegationTokenSecretManagerService.class).toInstance( - secretManagerService); - bind(TimelineACLsManager.class).toInstance(timelineACLsManager); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 5d749fa..44567ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -25,7 +25,6 @@ import java.util.Collection; import java.util.EnumSet; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.SortedSet; @@ -53,21 +52,17 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.NameValuePair; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; -import org.apache.hadoop.yarn.webapp.NotFoundException; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -80,13 +75,10 @@ private static final Log LOG = LogFactory.getLog(TimelineWebServices.class); private TimelineStore store; - private TimelineACLsManager timelineACLsManager; @Inject - public TimelineWebServices(TimelineStore store, - TimelineACLsManager timelineACLsManager) { + public TimelineWebServices(TimelineStore store) { this.store = store; - this.timelineACLsManager = timelineACLsManager; } @XmlRootElement(name = "about") @@ -149,9 +141,6 @@ public TimelineEntities getEntities( init(res); TimelineEntities entities = null; try { - EnumSet fieldEnums = parseFieldsStr(fields, ","); - boolean modified = extendFields(fieldEnums); - UserGroupInformation callerUGI = getUser(req); entities = store.getEntities( parseStr(entityType), parseLongStr(limit), @@ -161,33 +150,7 @@ public TimelineEntities getEntities( parseLongStr(fromTs), parsePairStr(primaryFilter, ":"), parsePairsStr(secondaryFilter, ",", ":"), - fieldEnums); - if (entities != null) { - Iterator entitiesItr = - entities.getEntities().iterator(); - while (entitiesItr.hasNext()) { - TimelineEntity entity = entitiesItr.next(); - try { - // check ACLs - if (!timelineACLsManager.checkAccess(callerUGI, entity)) { - entitiesItr.remove(); - } else { - // clean up system data - if (modified) { - entity.setPrimaryFilters(null); - } else { - cleanupOwnerInfo(entity); - } - } - } catch (YarnException e) { - LOG.error("Error when verifying access for user " + callerUGI - + " on the events of the timeline entity " - + new EntityIdentifier(entity.getEntityId(), - entity.getEntityType()), e); - entitiesItr.remove(); - } - } - } + parseFieldsStr(fields, ",")); } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); @@ -219,25 +182,9 @@ public TimelineEntity getEntity( init(res); TimelineEntity entity = null; try { - EnumSet fieldEnums = parseFieldsStr(fields, ","); - boolean modified = extendFields(fieldEnums); entity = store.getEntity(parseStr(entityId), parseStr(entityType), - fieldEnums); - if (entity != null) { - // check ACLs - UserGroupInformation callerUGI = getUser(req); - if (!timelineACLsManager.checkAccess(callerUGI, entity)) { - entity = null; - } else { - // clean up the system data - if (modified) { - entity.setPrimaryFilters(null); - } else { - cleanupOwnerInfo(entity); - } - } - } + parseFieldsStr(fields, ",")); } catch (IllegalArgumentException e) { throw new BadRequestException( "requested invalid field."); @@ -245,15 +192,9 @@ public TimelineEntity getEntity( LOG.error("Error getting entity", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); - } catch (YarnException e) { - LOG.error("Error getting entity", e); - throw new WebApplicationException(e, - Response.Status.INTERNAL_SERVER_ERROR); } if (entity == null) { - throw new NotFoundException("Timeline entity " - + new EntityIdentifier(parseStr(entityId), parseStr(entityType)) - + " is not found"); + throw new WebApplicationException(Response.Status.NOT_FOUND); } return entity; } @@ -276,7 +217,6 @@ public TimelineEvents getEvents( init(res); TimelineEvents events = null; try { - UserGroupInformation callerUGI = getUser(req); events = store.getEntityTimelines( parseStr(entityType), parseArrayStr(entityId, ","), @@ -284,29 +224,6 @@ public TimelineEvents getEvents( parseLongStr(windowStart), parseLongStr(windowEnd), parseArrayStr(eventType, ",")); - if (events != null) { - Iterator eventsItr = - events.getAllEvents().iterator(); - while (eventsItr.hasNext()) { - TimelineEvents.EventsOfOneEntity eventsOfOneEntity = eventsItr.next(); - try { - TimelineEntity entity = store.getEntity( - eventsOfOneEntity.getEntityId(), - eventsOfOneEntity.getEntityType(), - EnumSet.of(Field.PRIMARY_FILTERS)); - // check ACLs - if (!timelineACLsManager.checkAccess(callerUGI, entity)) { - eventsItr.remove(); - } - } catch (Exception e) { - LOG.error("Error when verifying access for user " + callerUGI - + " on the events of the timeline entity " - + new EntityIdentifier(eventsOfOneEntity.getEntityId(), - eventsOfOneEntity.getEntityType()), e); - eventsItr.remove(); - } - } - } } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); @@ -335,66 +252,12 @@ public TimelinePutResponse postEntities( if (entities == null) { return new TimelinePutResponse(); } - UserGroupInformation callerUGI = getUser(req); try { List entityIDs = new ArrayList(); - TimelineEntities entitiesToPut = new TimelineEntities(); - List errors = - new ArrayList(); for (TimelineEntity entity : entities.getEntities()) { EntityIdentifier entityID = new EntityIdentifier(entity.getEntityId(), entity.getEntityType()); - - // check if there is existing entity - TimelineEntity existingEntity = null; - try { - existingEntity = - store.getEntity(entityID.getId(), entityID.getType(), - EnumSet.of(Field.PRIMARY_FILTERS)); - if (existingEntity != null - && !timelineACLsManager.checkAccess(callerUGI, existingEntity)) { - throw new YarnException("The timeline entity " + entityID - + " was not put by " + callerUGI + " before"); - } - } catch (Exception e) { - // Skip the entity which already exists and was put by others - LOG.warn("Skip the timeline entity: " + entityID + ", because " - + e.getMessage()); - TimelinePutResponse.TimelinePutError error = - new TimelinePutResponse.TimelinePutError(); - error.setEntityId(entityID.getId()); - error.setEntityType(entityID.getType()); - error.setErrorCode( - TimelinePutResponse.TimelinePutError.ACCESS_DENIED); - errors.add(error); - continue; - } - - // inject owner information for the access check if this is the first - // time to post the entity, in case it's the admin who is updating - // the timeline data. - try { - if (existingEntity == null) { - injectOwnerInfo(entity, - callerUGI == null ? "" : callerUGI.getShortUserName()); - } - } catch (YarnException e) { - // Skip the entity which messes up the primary filter and record the - // error - LOG.warn("Skip the timeline entity: " + entityID + ", because " - + e.getMessage()); - TimelinePutResponse.TimelinePutError error = - new TimelinePutResponse.TimelinePutError(); - error.setEntityId(entityID.getId()); - error.setEntityType(entityID.getType()); - error.setErrorCode( - TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT); - errors.add(error); - continue; - } - entityIDs.add(entityID); - entitiesToPut.addEntity(entity); if (LOG.isDebugEnabled()) { LOG.debug("Storing the entity " + entityID + ", JSON-style content: " + TimelineUtils.dumpTimelineRecordtoJSON(entity)); @@ -403,10 +266,7 @@ public TimelinePutResponse postEntities( if (LOG.isDebugEnabled()) { LOG.debug("Storing entities: " + CSV_JOINER.join(entityIDs)); } - TimelinePutResponse response = store.put(entitiesToPut); - // add the errors of timeline system filter key conflict - response.addErrors(errors); - return response; + return store.put(entities); } catch (IOException e) { LOG.error("Error putting entities", e); throw new WebApplicationException(e, @@ -490,14 +350,6 @@ private static NameValuePair parsePairStr(String str, String delimiter) { } } - private static boolean extendFields(EnumSet fieldEnums) { - boolean modified = false; - if (fieldEnums != null && !fieldEnums.contains(Field.PRIMARY_FILTERS)) { - fieldEnums.add(Field.PRIMARY_FILTERS); - modified = true; - } - return modified; - } private static Long parseLongStr(String str) { return str == null ? null : Long.parseLong(str.trim()); } @@ -506,34 +358,4 @@ private static String parseStr(String str) { return str == null ? null : str.trim(); } - private static UserGroupInformation getUser(HttpServletRequest req) { - String remoteUser = req.getRemoteUser(); - UserGroupInformation callerUGI = null; - if (remoteUser != null) { - callerUGI = UserGroupInformation.createRemoteUser(remoteUser); - } - return callerUGI; - } - - private static void injectOwnerInfo(TimelineEntity timelineEntity, - String owner) throws YarnException { - if (timelineEntity.getPrimaryFilters() != null && - timelineEntity.getPrimaryFilters().containsKey( - TimelineStore.SystemFilter.ENTITY_OWNER)) { - throw new YarnException( - "User should not use the timeline system filter key: " - + TimelineStore.SystemFilter.ENTITY_OWNER); - } - timelineEntity.addPrimaryFilter( - TimelineStore.SystemFilter.ENTITY_OWNER - .toString(), owner); - } - - private static void cleanupOwnerInfo(TimelineEntity timelineEntity) { - if (timelineEntity.getPrimaryFilters() != null) { - timelineEntity.getPrimaryFilters().remove( - TimelineStore.SystemFilter.ENTITY_OWNER.toString()); - } - } - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index 3f3c08a..a35fe46 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; import org.junit.Before; @@ -75,7 +74,6 @@ public void setup() { @After public void tearDown() throws Exception { - AHSWebApp.resetInstance(); historyServer.stop(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index 5c55bec..d6d20af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -26,7 +26,6 @@ import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.junit.After; import org.junit.Test; @@ -41,7 +40,7 @@ public void testStartStopServer() throws Exception { Configuration config = new YarnConfiguration(); historyServer.init(config); assertEquals(STATE.INITED, historyServer.getServiceState()); - assertEquals(4, historyServer.getServices().size()); + assertEquals(3, historyServer.getServices().size()); ApplicationHistoryClientService historyService = historyServer.getClientService(); assertNotNull(historyServer.getClientService()); @@ -74,6 +73,5 @@ public void stop() { if (historyServer != null) { historyServer.stop(); } - AHSWebApp.resetInstance(); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java index 59449f8..9b27309 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java @@ -17,10 +17,6 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - import java.io.File; import java.io.IOException; import java.util.Collections; @@ -32,8 +28,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -45,6 +39,9 @@ import org.junit.Before; import org.junit.Test; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import static org.junit.Assert.assertEquals; + @InterfaceAudience.Private @InterfaceStability.Unstable public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { @@ -54,7 +51,7 @@ @Before public void setup() throws Exception { fsContext = FileContext.getLocalFSFileContext(); - Configuration conf = new YarnConfiguration(); + Configuration conf = new Configuration(); fsPath = new File("target", this.getClass().getSimpleName() + "-tmpDir").getAbsoluteFile(); fsContext.delete(new Path(fsPath.getAbsolutePath()), true); @@ -75,15 +72,6 @@ public void tearDown() throws Exception { } @Test - public void testRootDirPermission() throws IOException { - FileSystem fs = FileSystem.getLocal(new YarnConfiguration()); - FileStatus file = fs.getFileStatus( - new Path(fsPath.getAbsolutePath(), LeveldbTimelineStore.FILENAME)); - assertNotNull(file); - assertEquals(LeveldbTimelineStore.LEVELDB_DIR_UMASK, file.getPermission()); - } - - @Test public void testGetSingleEntity() throws IOException { super.testGetSingleEntity(); ((LeveldbTimelineStore)store).clearStartTimeCache(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java deleted file mode 100644 index 39102b4..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.junit.Assert; -import org.junit.Test; - -public class TestTimelineACLsManager { - - @Test - public void testYarnACLsNotEnabled() throws Exception { - Configuration conf = new YarnConfiguration(); - conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); - TimelineACLsManager timelineACLsManager = - new TimelineACLsManager(conf); - TimelineEntity entity = new TimelineEntity(); - entity.addPrimaryFilter( - TimelineStore.SystemFilter.ENTITY_OWNER - .toString(), "owner"); - Assert.assertTrue( - "Always true when ACLs are not enabled", - timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("user"), entity)); - } - - @Test - public void testYarnACLsEnabled() throws Exception { - Configuration conf = new YarnConfiguration(); - conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); - conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); - TimelineACLsManager timelineACLsManager = - new TimelineACLsManager(conf); - TimelineEntity entity = new TimelineEntity(); - entity.addPrimaryFilter( - TimelineStore.SystemFilter.ENTITY_OWNER - .toString(), "owner"); - Assert.assertTrue( - "Owner should be allowed to access", - timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("owner"), entity)); - Assert.assertFalse( - "Other shouldn't be allowed to access", - timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("other"), entity)); - Assert.assertTrue( - "Admin should be allowed to access", - timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("admin"), entity)); - } - - @Test - public void testCorruptedOwnerInfo() throws Exception { - Configuration conf = new YarnConfiguration(); - conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); - conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner"); - TimelineACLsManager timelineACLsManager = - new TimelineACLsManager(conf); - TimelineEntity entity = new TimelineEntity(); - try { - timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("owner"), entity); - Assert.fail("Exception is expected"); - } catch (YarnException e) { - Assert.assertTrue("It's not the exact expected exception", e.getMessage() - .contains("is corrupted.")); - } - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java index 7e3e409..650184b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java @@ -20,33 +20,19 @@ import static org.junit.Assert.assertEquals; -import java.io.IOException; - -import javax.inject.Singleton; -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletRequestWrapper; import javax.ws.rs.core.MediaType; -import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; + import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.security.AdminACLsManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TestMemoryTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TestMemoryTimelineStore; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.junit.Assert; import org.junit.Test; import com.google.inject.Guice; @@ -64,9 +50,6 @@ public class TestTimelineWebServices extends JerseyTest { private static TimelineStore store; - private static TimelineACLsManager timelineACLsManager; - private static AdminACLsManager adminACLsManager; - private static String remoteUser; private long beforeTime; private Injector injector = Guice.createInjector(new ServletModule() { @@ -82,15 +65,7 @@ protected void configureServlets() { Assert.fail(); } bind(TimelineStore.class).toInstance(store); - Configuration conf = new YarnConfiguration(); - conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); - timelineACLsManager = new TimelineACLsManager(conf); - conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); - conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); - adminACLsManager = new AdminACLsManager(conf); - bind(TimelineACLsManager.class).toInstance(timelineACLsManager); serve("/*").with(GuiceContainer.class); - filter("/*").through(TestFilter.class); } }); @@ -365,8 +340,8 @@ public void testGetEvents() throws Exception { public void testPostEntities() throws Exception { TimelineEntities entities = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); - entity.setEntityId("test id 1"); - entity.setEntityType("test type 1"); + entity.setEntityId("test id"); + entity.setEntityType("test type"); entity.setStartTime(System.currentTimeMillis()); entities.addEntity(entity); WebResource r = resource(); @@ -380,252 +355,14 @@ public void testPostEntities() throws Exception { Assert.assertEquals(0, putResposne.getErrors().size()); // verify the entity exists in the store response = r.path("ws").path("v1").path("timeline") - .path("test type 1").path("test id 1") + .path("test type").path("test id") .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); entity = response.getEntity(TimelineEntity.class); Assert.assertNotNull(entity); - Assert.assertEquals("test id 1", entity.getEntityId()); - Assert.assertEquals("test type 1", entity.getEntityType()); - } - - @Test - public void testPostEntitiesWithYarnACLsEnabled() throws Exception { - AdminACLsManager oldAdminACLsManager = - timelineACLsManager.setAdminACLsManager(adminACLsManager); - remoteUser = "tester"; - try { - TimelineEntities entities = new TimelineEntities(); - TimelineEntity entity = new TimelineEntity(); - entity.setEntityId("test id 2"); - entity.setEntityType("test type 2"); - entity.setStartTime(System.currentTimeMillis()); - entities.addEntity(entity); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("timeline") - .accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, entities); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertNotNull(putResponse); - Assert.assertEquals(0, putResponse.getErrors().size()); - - // override/append timeline data in the same entity with different user - remoteUser = "other"; - response = r.path("ws").path("v1").path("timeline") - .accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, entities); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertNotNull(putResponse); - Assert.assertEquals(1, putResponse.getErrors().size()); - Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, - putResponse.getErrors().get(0).getErrorCode()); - } finally { - timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); - remoteUser = null; - } - } - - @Test - public void testGetEntityWithYarnACLsEnabled() throws Exception { - AdminACLsManager oldAdminACLsManager = - timelineACLsManager.setAdminACLsManager(adminACLsManager); - remoteUser = "tester"; - try { - TimelineEntities entities = new TimelineEntities(); - TimelineEntity entity = new TimelineEntity(); - entity.setEntityId("test id 3"); - entity.setEntityType("test type 3"); - entity.setStartTime(System.currentTimeMillis()); - entities.addEntity(entity); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("timeline") - .accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, entities); - // verify the system data will not be exposed - // 1. No field specification - response = r.path("ws").path("v1").path("timeline") - .path("test type 3").path("test id 3") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - entity = response.getEntity(TimelineEntity.class); - Assert.assertNull(entity.getPrimaryFilters().get( - TimelineStore.SystemFilter.ENTITY_OWNER.toString())); - // 2. other field - response = r.path("ws").path("v1").path("timeline") - .path("test type 3").path("test id 3") - .queryParam("fields", "relatedentities") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - entity = response.getEntity(TimelineEntity.class); - Assert.assertNull(entity.getPrimaryFilters().get( - TimelineStore.SystemFilter.ENTITY_OWNER.toString())); - // 3. primaryfilters field - response = r.path("ws").path("v1").path("timeline") - .path("test type 3").path("test id 3") - .queryParam("fields", "primaryfilters") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - entity = response.getEntity(TimelineEntity.class); - Assert.assertNull(entity.getPrimaryFilters().get( - TimelineStore.SystemFilter.ENTITY_OWNER.toString())); - - // get entity with other user - remoteUser = "other"; - response = r.path("ws").path("v1").path("timeline") - .path("test type 3").path("test id 3") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - assertEquals(ClientResponse.Status.NOT_FOUND, - response.getClientResponseStatus()); - } finally { - timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); - remoteUser = null; - } - } - - @Test - public void testGetEntitiesWithYarnACLsEnabled() { - AdminACLsManager oldAdminACLsManager = - timelineACLsManager.setAdminACLsManager(adminACLsManager); - remoteUser = "tester"; - try { - TimelineEntities entities = new TimelineEntities(); - TimelineEntity entity = new TimelineEntity(); - entity.setEntityId("test id 4"); - entity.setEntityType("test type 4"); - entity.setStartTime(System.currentTimeMillis()); - entities.addEntity(entity); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("timeline") - .accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, entities); - - remoteUser = "other"; - entities = new TimelineEntities(); - entity = new TimelineEntity(); - entity.setEntityId("test id 5"); - entity.setEntityType("test type 4"); - entity.setStartTime(System.currentTimeMillis()); - entities.addEntity(entity); - r = resource(); - response = r.path("ws").path("v1").path("timeline") - .accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, entities); - - response = r.path("ws").path("v1").path("timeline") - .path("test type 4") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - entities = response.getEntity(TimelineEntities.class); - assertEquals(1, entities.getEntities().size()); - assertEquals("test type 4", entities.getEntities().get(0).getEntityType()); - assertEquals("test id 5", entities.getEntities().get(0).getEntityId()); - } finally { - timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); - remoteUser = null; - } - } - - @Test - public void testGetEventsWithYarnACLsEnabled() { - AdminACLsManager oldAdminACLsManager = - timelineACLsManager.setAdminACLsManager(adminACLsManager); - remoteUser = "tester"; - try { - TimelineEntities entities = new TimelineEntities(); - TimelineEntity entity = new TimelineEntity(); - entity.setEntityId("test id 5"); - entity.setEntityType("test type 5"); - entity.setStartTime(System.currentTimeMillis()); - TimelineEvent event = new TimelineEvent(); - event.setEventType("event type 1"); - event.setTimestamp(System.currentTimeMillis()); - entity.addEvent(event); - entities.addEntity(entity); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("timeline") - .accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, entities); - - remoteUser = "other"; - entities = new TimelineEntities(); - entity = new TimelineEntity(); - entity.setEntityId("test id 6"); - entity.setEntityType("test type 5"); - entity.setStartTime(System.currentTimeMillis()); - event = new TimelineEvent(); - event.setEventType("event type 2"); - event.setTimestamp(System.currentTimeMillis()); - entity.addEvent(event); - entities.addEntity(entity); - r = resource(); - response = r.path("ws").path("v1").path("timeline") - .accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, entities); - - response = r.path("ws").path("v1").path("timeline") - .path("test type 5").path("events") - .queryParam("entityId", "test id 5,test id 6") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - TimelineEvents events = response.getEntity(TimelineEvents.class); - assertEquals(1, events.getAllEvents().size()); - assertEquals("test id 6", events.getAllEvents().get(0).getEntityId()); - } finally { - timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); - remoteUser = null; - } + Assert.assertEquals("test id", entity.getEntityId()); + Assert.assertEquals("test type", entity.getEntityType()); } - @Singleton - private static class TestFilter implements Filter { - - @Override - public void init(FilterConfig filterConfig) throws ServletException { - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, - FilterChain chain) throws IOException, ServletException { - if (request instanceof HttpServletRequest) { - request = - new TestHttpServletRequestWrapper((HttpServletRequest) request); - } - chain.doFilter(request, response); - } - - @Override - public void destroy() { - } - - } - - private static class TestHttpServletRequestWrapper extends HttpServletRequestWrapper { - - public TestHttpServletRequestWrapper(HttpServletRequest request) { - super(request); - } - - @Override - public String getRemoteUser() { - return TestTimelineWebServices.remoteUser; - } - - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index f94aedbf..684c82b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -170,7 +170,7 @@ public void init(Configuration config, public void editSchedule(){ CSQueue root = scheduler.getRootQueue(); Resource clusterResources = - Resources.clone(scheduler.getClusterResource()); + Resources.clone(scheduler.getClusterResources()); containerBasedPreemptOrKill(root, clusterResources); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 9b15bb2..6026e94 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -90,7 +90,9 @@ private String zkHostPort = null; private int zkSessionTimeout; - private long zkRetryInterval; + + @VisibleForTesting + long zkRetryInterval; private List zkAcl; private List zkAuths; @@ -199,9 +201,14 @@ public synchronized void initInternal(Configuration conf) throws Exception { zkSessionTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS); - zkRetryInterval = - conf.getLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS); + + if (HAUtil.isHAEnabled(conf)) { + zkRetryInterval = zkSessionTimeout / numRetries; + } else { + zkRetryInterval = + conf.getLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS); + } zkAcl = RMZKUtils.getZKAcls(conf); zkAuths = RMZKUtils.getZKAuths(conf); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 642cd31..0f3af41 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -22,41 +22,21 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.resource.Resources; -public abstract class AbstractYarnScheduler - - implements ResourceScheduler { - - private static final Log LOG = LogFactory.getLog(AbstractYarnScheduler.class); - - // Nodes in the cluster, indexed by NodeId - protected Map nodes = - new ConcurrentHashMap(); - - // Whole capacity of the cluster - protected Resource clusterResource = Resource.newInstance(0, 0); - - protected Resource minimumAllocation; - protected Resource maximumAllocation; +public abstract class AbstractYarnScheduler implements ResourceScheduler { protected RMContext rmContext; - protected Map> applications; + protected Map applications; protected final static List EMPTY_CONTAINER_LIST = new ArrayList(); protected static final Allocation EMPTY_ALLOCATION = new Allocation( @@ -65,7 +45,7 @@ public synchronized List getTransferredContainers( ApplicationAttemptId currentAttempt) { ApplicationId appId = currentAttempt.getApplicationId(); - SchedulerApplication app = applications.get(appId); + SchedulerApplication app = applications.get(appId); List containerList = new ArrayList(); RMApp appImpl = this.rmContext.getRMApps().get(appId); if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) { @@ -84,75 +64,10 @@ return containerList; } - public Map> - getSchedulerApplications() { + public Map getSchedulerApplications() { return applications; } - - @Override - public Resource getClusterResource() { - return clusterResource; - } - - @Override - public Resource getMinimumResourceCapability() { - return minimumAllocation; - } - - @Override - public Resource getMaximumResourceCapability() { - return maximumAllocation; - } - - public T getApplicationAttempt(ApplicationAttemptId applicationAttemptId) { - SchedulerApplication app = - applications.get(applicationAttemptId.getApplicationId()); - return app == null ? null : app.getCurrentAppAttempt(); - } - - @Override - public SchedulerAppReport getSchedulerAppInfo( - ApplicationAttemptId appAttemptId) { - SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId); - if (attempt == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Request for appInfo of unknown attempt " + appAttemptId); - } - return null; - } - return new SchedulerAppReport(attempt); - } - - @Override - public ApplicationResourceUsageReport getAppResourceUsageReport( - ApplicationAttemptId appAttemptId) { - SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId); - if (attempt == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Request for appInfo of unknown attempt " + appAttemptId); - } - return null; - } - return attempt.getResourceUsageReport(); - } - - public T getCurrentAttemptForContainer(ContainerId containerId) { - return getApplicationAttempt(containerId.getApplicationAttemptId()); - } - - @Override - public RMContainer getRMContainer(ContainerId containerId) { - SchedulerApplicationAttempt attempt = - getCurrentAttemptForContainer(containerId); - return (attempt == null) ? null : attempt.getRMContainer(containerId); - } - - @Override - public SchedulerNodeReport getNodeReport(NodeId nodeId) { - N node = nodes.get(nodeId); - return node == null ? null : new SchedulerNodeReport(node); - } - + @Override public String moveApplication(ApplicationId appId, String newQueue) throws YarnException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index 2c788aa..4d6ca0e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -23,11 +23,11 @@ @Private @Unstable -public class SchedulerApplication { +public class SchedulerApplication { private Queue queue; private final String user; - private T currentAttempt; + private SchedulerApplicationAttempt currentAttempt; public SchedulerApplication(Queue queue, String user) { this.queue = queue; @@ -46,11 +46,11 @@ public String getUser() { return user; } - public T getCurrentAppAttempt() { + public SchedulerApplicationAttempt getCurrentAppAttempt() { return currentAttempt; } - public void setCurrentAppAttempt(T currentAttempt) { + public void setCurrentAppAttempt(SchedulerApplicationAttempt currentAttempt) { this.currentAttempt = currentAttempt; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 85d016b..524b1ab 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -18,27 +18,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.util.resource.Resources; - -import com.google.common.base.Preconditions; /** * Represents a YARN Cluster Node from the viewpoint of the scheduler. @@ -47,231 +31,59 @@ @Unstable public abstract class SchedulerNode { - private static final Log LOG = LogFactory.getLog(SchedulerNode.class); - - private Resource availableResource = Resource.newInstance(0, 0); - private Resource usedResource = Resource.newInstance(0, 0); - private Resource totalResourceCapability; - private RMContainer reservedContainer; - private volatile int numContainers; - - - /* set of containers that are allocated containers */ - private final Map launchedContainers = - new HashMap(); - - private final RMNode rmNode; - private final String nodeName; - - public SchedulerNode(RMNode node, boolean usePortForNodeName) { - this.rmNode = node; - this.availableResource = Resources.clone(node.getTotalCapability()); - this.totalResourceCapability = Resources.clone(node.getTotalCapability()); - if (usePortForNodeName) { - nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort(); - } else { - nodeName = rmNode.getHostName(); - } - } - - public RMNode getRMNode() { - return this.rmNode; - } - - /** - * Get the ID of the node which contains both its hostname and port. - * - * @return the ID of the node - */ - public NodeId getNodeID() { - return this.rmNode.getNodeID(); - } - - public String getHttpAddress() { - return this.rmNode.getHttpAddress(); - } - /** * Get the name of the node for scheduling matching decisions. *

- * Typically this is the 'hostname' reported by the node, but it could be - * configured to be 'hostname:port' reported by the node via the + * Typically this is the 'hostname' reported by the node, but it could be + * configured to be 'hostname:port' reported by the node via the * {@link YarnConfiguration#RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME} constant. * The main usecase of this is Yarn minicluster to be able to differentiate * node manager instances by their port number. * * @return name of the node for scheduling matching decisions. */ - public String getNodeName() { - return nodeName; - } - + public abstract String getNodeName(); + /** * Get rackname. - * * @return rackname */ - public String getRackName() { - return this.rmNode.getRackName(); - } - - /** - * The Scheduler has allocated containers on this node to the given - * application. - * - * @param applicationId - * application - * @param rmContainer - * allocated container - */ - public synchronized void allocateContainer(ApplicationId applicationId, - RMContainer rmContainer) { - Container container = rmContainer.getContainer(); - deductAvailableResource(container.getResource()); - ++numContainers; - - launchedContainers.put(container.getId(), rmContainer); - - LOG.info("Assigned container " + container.getId() + " of capacity " - + container.getResource() + " on host " + rmNode.getNodeAddress() - + ", which currently has " + numContainers + " containers, " - + getUsedResource() + " used and " + getAvailableResource() - + " available"); - } - - /** - * Get available resources on the node. - * - * @return available resources on the node - */ - public synchronized Resource getAvailableResource() { - return this.availableResource; - } - + public abstract String getRackName(); + /** * Get used resources on the node. - * * @return used resources on the node */ - public synchronized Resource getUsedResource() { - return this.usedResource; - } + public abstract Resource getUsedResource(); /** - * Get total resources on the node. - * - * @return total resources on the node. + * Get available resources on the node. + * @return available resources on the node */ - public Resource getTotalResource() { - return this.totalResourceCapability; - } - - private synchronized boolean isValidContainer(Container c) { - if (launchedContainers.containsKey(c.getId())) { - return true; - } - return false; - } - - private synchronized void updateResource(Container container) { - addAvailableResource(container.getResource()); - --numContainers; - } + public abstract Resource getAvailableResource(); /** - * Release an allocated container on this node. - * - * @param container - * container to be released + * Get number of active containers on the node. + * @return number of active containers on the node */ - public synchronized void releaseContainer(Container container) { - if (!isValidContainer(container)) { - LOG.error("Invalid container released " + container); - return; - } - - /* remove the containers from the nodemanger */ - if (null != launchedContainers.remove(container.getId())) { - updateResource(container); - } - - LOG.info("Released container " + container.getId() + " of capacity " - + container.getResource() + " on host " + rmNode.getNodeAddress() - + ", which currently has " + numContainers + " containers, " - + getUsedResource() + " used and " + getAvailableResource() - + " available" + ", release resources=" + true); - } - - private synchronized void addAvailableResource(Resource resource) { - if (resource == null) { - LOG.error("Invalid resource addition of null resource for " - + rmNode.getNodeAddress()); - return; - } - Resources.addTo(availableResource, resource); - Resources.subtractFrom(usedResource, resource); - } - - private synchronized void deductAvailableResource(Resource resource) { - if (resource == null) { - LOG.error("Invalid deduction of null resource for " - + rmNode.getNodeAddress()); - return; - } - Resources.subtractFrom(availableResource, resource); - Resources.addTo(usedResource, resource); - } - + public abstract int getNumContainers(); + /** - * Reserve container for the attempt on this node. + * Apply delta resource on node's available resource. + * @param deltaResource the delta of resource need to apply to node */ - public abstract void reserveResource(SchedulerApplicationAttempt attempt, - Priority priority, RMContainer container); + public abstract void applyDeltaOnAvailableResource(Resource deltaResource); /** - * Unreserve resources on this node. + * Get total resources on the node. + * @return total resources on the node. */ - public abstract void unreserveResource(SchedulerApplicationAttempt attempt); - - @Override - public String toString() { - return "host: " + rmNode.getNodeAddress() + " #containers=" - + getNumContainers() + " available=" - + getAvailableResource().getMemory() + " used=" - + getUsedResource().getMemory(); - } - + public abstract Resource getTotalResource(); + /** - * Get number of active containers on the node. - * - * @return number of active containers on the node + * Get the ID of the node which contains both its hostname and port. + * @return the ID of the node */ - public int getNumContainers() { - return numContainers; - } - - public synchronized List getRunningContainers() { - return new ArrayList(launchedContainers.values()); - } - - public synchronized RMContainer getReservedContainer() { - return reservedContainer; - } - - protected synchronized void - setReservedContainer(RMContainer reservedContainer) { - this.reservedContainer = reservedContainer; - } + public abstract NodeId getNodeID(); - /** - * Apply delta resource on node's available resource. - * - * @param deltaResource - * the delta of resource need to apply to node - */ - public synchronized void - applyDeltaOnAvailableResource(Resource deltaResource) { - // we can only adjust available resource if total resource is changed. - Resources.addTo(this.availableResource, deltaResource); - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 21eba39..2348603 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -69,15 +69,7 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, @Public @Stable public List getQueueUserAclInfo(); - - /** - * Get the whole resource capacity of the cluster. - * @return the whole resource capacity of the cluster. - */ - @LimitedPrivate("yarn") - @Unstable - public Resource getClusterResource(); - + /** * Get minimum allocatable {@link Resource}. * @return minimum allocatable resource @@ -190,7 +182,7 @@ boolean checkAccess(UserGroupInformation callerUGI, @LimitedPrivate("yarn") @Unstable public RMContainer getRMContainer(ContainerId containerId); - + /** * Moves the given application to the given queue * @param appId diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 9eed61f..e28c18c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -41,6 +41,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -48,9 +49,11 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -72,7 +75,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; @@ -95,9 +100,9 @@ @LimitedPrivate("yarn") @Evolving @SuppressWarnings("unchecked") -public class CapacityScheduler extends - AbstractYarnScheduler implements - PreemptableResourceScheduler, CapacitySchedulerContext, Configurable { +public class CapacityScheduler extends AbstractYarnScheduler + implements PreemptableResourceScheduler, CapacitySchedulerContext, + Configurable { private static final Log LOG = LogFactory.getLog(CapacityScheduler.class); @@ -177,8 +182,16 @@ public Configuration getConf() { private Map queues = new ConcurrentHashMap(); + private Map nodes = + new ConcurrentHashMap(); + + private Resource clusterResource = + RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Resource.class); private int numNodeManagers = 0; + private Resource minimumAllocation; + private Resource maximumAllocation; + private boolean initialized = false; private ResourceCalculator calculator; @@ -218,6 +231,16 @@ public RMContainerTokenSecretManager getContainerTokenSecretManager() { } @Override + public Resource getMinimumResourceCapability() { + return minimumAllocation; + } + + @Override + public Resource getMaximumResourceCapability() { + return maximumAllocation; + } + + @Override public Comparator getApplicationComparator() { return applicationComparator; } @@ -241,6 +264,11 @@ public synchronized int getNumClusterNodes() { public RMContext getRMContext() { return this.rmContext; } + + @Override + public Resource getClusterResources() { + return clusterResource; + } @Override public synchronized void @@ -255,7 +283,7 @@ public RMContext getRMContext() { this.calculator = this.conf.getResourceCalculator(); this.usePortForNodeName = this.conf.getUsePortForNodeName(); this.applications = - new ConcurrentHashMap>(); + new ConcurrentHashMap(); initializeQueues(this.conf); @@ -508,8 +536,8 @@ private synchronized void addApplication(ApplicationId applicationId, .handle(new RMAppRejectedEvent(applicationId, ace.toString())); return; } - SchedulerApplication application = - new SchedulerApplication(queue, user); + SchedulerApplication application = + new SchedulerApplication(queue, user); applications.put(applicationId, application); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName); @@ -520,7 +548,7 @@ private synchronized void addApplication(ApplicationId applicationId, private synchronized void addApplicationAttempt( ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt) { - SchedulerApplication application = + SchedulerApplication application = applications.get(applicationAttemptId.getApplicationId()); CSQueue queue = (CSQueue) application.getQueue(); @@ -544,8 +572,7 @@ private synchronized void addApplicationAttempt( private synchronized void doneApplication(ApplicationId applicationId, RMAppState finalState) { - SchedulerApplication application = - applications.get(applicationId); + SchedulerApplication application = applications.get(applicationId); if (application == null){ // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps, // ignore it. @@ -570,7 +597,7 @@ private synchronized void doneApplicationAttempt( " finalState=" + rmAppAttemptFinalState); FiCaSchedulerApp attempt = getApplicationAttempt(applicationAttemptId); - SchedulerApplication application = + SchedulerApplication application = applications.get(applicationAttemptId.getApplicationId()); if (application == null || attempt == null) { @@ -632,7 +659,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, // Sanity check SchedulerUtils.normalizeRequests( - ask, getResourceCalculator(), getClusterResource(), + ask, getResourceCalculator(), getClusterResources(), getMinimumResourceCapability(), maximumAllocation); // Release containers @@ -795,7 +822,7 @@ private synchronized void allocateContainersToNode(FiCaSchedulerNode node) { // Try to schedule more if there are no reservations to fulfill if (node.getReservedContainer() == null) { - if (Resources.greaterThanOrEqual(calculator, getClusterResource(), + if (Resources.greaterThanOrEqual(calculator, getClusterResources(), node.getAvailableResource(), minimumAllocation)) { if (LOG.isDebugEnabled()) { LOG.debug("Trying to schedule on node: " + node.getNodeName() + @@ -915,7 +942,7 @@ private synchronized void addNode(RMNode nodeManager) { } private synchronized void removeNode(RMNode nodeInfo) { - FiCaSchedulerNode node = nodes.get(nodeInfo.getNodeID()); + FiCaSchedulerNode node = this.nodes.get(nodeInfo.getNodeID()); if (node == null) { return; } @@ -988,10 +1015,28 @@ private synchronized void completedContainer(RMContainer rmContainer, @Lock(Lock.NoLock.class) @VisibleForTesting - @Override public FiCaSchedulerApp getApplicationAttempt( ApplicationAttemptId applicationAttemptId) { - return super.getApplicationAttempt(applicationAttemptId); + SchedulerApplication app = + applications.get(applicationAttemptId.getApplicationId()); + if (app != null) { + return (FiCaSchedulerApp) app.getCurrentAppAttempt(); + } + return null; + } + + @Override + public SchedulerAppReport getSchedulerAppInfo( + ApplicationAttemptId applicationAttemptId) { + FiCaSchedulerApp app = getApplicationAttempt(applicationAttemptId); + return app == null ? null : new SchedulerAppReport(app); + } + + @Override + public ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId applicationAttemptId) { + FiCaSchedulerApp app = getApplicationAttempt(applicationAttemptId); + return app == null ? null : app.getResourceUsageReport(); } @Lock(Lock.NoLock.class) @@ -1003,6 +1048,24 @@ FiCaSchedulerNode getNode(NodeId nodeId) { Map getAllNodes() { return nodes; } + + @Override + public RMContainer getRMContainer(ContainerId containerId) { + FiCaSchedulerApp attempt = getCurrentAttemptForContainer(containerId); + return (attempt == null) ? null : attempt.getRMContainer(containerId); + } + + @VisibleForTesting + public FiCaSchedulerApp getCurrentAttemptForContainer( + ContainerId containerId) { + SchedulerApplication app = + applications.get(containerId.getApplicationAttemptId() + .getApplicationId()); + if (app != null) { + return (FiCaSchedulerApp) app.getCurrentAppAttempt(); + } + return null; + } @Override @Lock(Lock.NoLock.class) @@ -1011,6 +1074,12 @@ public void recover(RMState state) throws Exception { } @Override + public SchedulerNodeReport getNodeReport(NodeId nodeId) { + FiCaSchedulerNode node = getNode(nodeId); + return node == null ? null : new SchedulerNodeReport(node); + } + + @Override public void dropContainerReservation(RMContainer container) { if(LOG.isDebugEnabled()){ LOG.debug("DROP_RESERVATION:" + container.toString()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java index a3dbc35..3dd4c6d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java @@ -43,7 +43,7 @@ RMContext getRMContext(); - Resource getClusterResource(); + Resource getClusterResources(); /** * Get the yarn configuration. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 5ddb9a4..971edb8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -174,12 +174,12 @@ public LeafQueue(CapacitySchedulerContext cs, int maxActiveApplications = CSQueueUtils.computeMaxActiveApplications( resourceCalculator, - cs.getClusterResource(), this.minimumAllocation, + cs.getClusterResources(), this.minimumAllocation, maxAMResourcePerQueuePercent, absoluteMaxCapacity); this.maxActiveAppsUsingAbsCap = CSQueueUtils.computeMaxActiveApplications( resourceCalculator, - cs.getClusterResource(), this.minimumAllocation, + cs.getClusterResources(), this.minimumAllocation, maxAMResourcePerQueuePercent, absoluteCapacity); int maxActiveApplicationsPerUser = CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit, @@ -195,7 +195,7 @@ public LeafQueue(CapacitySchedulerContext cs, cs.getConfiguration().getAcls(getQueuePath()); setupQueueConfigs( - cs.getClusterResource(), + cs.getClusterResources(), capacity, absoluteCapacity, maximumCapacity, absoluteMaxCapacity, userLimit, userLimitFactor, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index dba92a6..1f09475 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -143,7 +143,7 @@ public ParentQueue(CapacitySchedulerContext cs, this.queueInfo.setQueueName(queueName); this.queueInfo.setChildQueues(new ArrayList()); - setupQueueConfigs(cs.getClusterResource(), + setupQueueConfigs(cs.getClusterResources(), capacity, absoluteCapacity, maximumCapacity, absoluteMaxCapacity, state, acls); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java index 7bab760..58f12d0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java @@ -18,84 +18,248 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.util.resource.Resources; public class FiCaSchedulerNode extends SchedulerNode { private static final Log LOG = LogFactory.getLog(FiCaSchedulerNode.class); + private static final RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + + private Resource availableResource = recordFactory.newRecordInstance(Resource.class); + private Resource usedResource = recordFactory.newRecordInstance(Resource.class); + private Resource totalResourceCapability; + + private volatile int numContainers; + + private RMContainer reservedContainer; + + /* set of containers that are allocated containers */ + private final Map launchedContainers = + new HashMap(); + + private final RMNode rmNode; + private final String nodeName; + public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName) { - super(node, usePortForNodeName); + this.rmNode = node; + this.availableResource.setMemory(node.getTotalCapability().getMemory()); + this.availableResource.setVirtualCores(node.getTotalCapability().getVirtualCores()); + totalResourceCapability = + Resource.newInstance(node.getTotalCapability().getMemory(), node + .getTotalCapability().getVirtualCores()); + if (usePortForNodeName) { + nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort(); + } else { + nodeName = rmNode.getHostName(); + } + } + + public RMNode getRMNode() { + return this.rmNode; + } + + public NodeId getNodeID() { + return this.rmNode.getNodeID(); + } + + public String getHttpAddress() { + return this.rmNode.getHttpAddress(); } @Override + public String getNodeName() { + return nodeName; + } + + @Override + public String getRackName() { + return this.rmNode.getRackName(); + } + + /** + * The Scheduler has allocated containers on this node to the + * given application. + * + * @param applicationId application + * @param rmContainer allocated container + */ + public synchronized void allocateContainer(ApplicationId applicationId, + RMContainer rmContainer) { + Container container = rmContainer.getContainer(); + deductAvailableResource(container.getResource()); + ++numContainers; + + launchedContainers.put(container.getId(), rmContainer); + + LOG.info("Assigned container " + container.getId() + + " of capacity " + container.getResource() + " on host " + rmNode.getNodeAddress() + + ", which currently has " + numContainers + " containers, " + + getUsedResource() + " used and " + + getAvailableResource() + " available"); + } + + @Override + public synchronized Resource getAvailableResource() { + return this.availableResource; + } + + @Override + public synchronized Resource getUsedResource() { + return this.usedResource; + } + + @Override + public Resource getTotalResource() { + return this.totalResourceCapability; + } + + private synchronized boolean isValidContainer(Container c) { + if (launchedContainers.containsKey(c.getId())) + return true; + return false; + } + + private synchronized void updateResource(Container container) { + addAvailableResource(container.getResource()); + --numContainers; + } + + /** + * Release an allocated container on this node. + * @param container container to be released + */ + public synchronized void releaseContainer(Container container) { + if (!isValidContainer(container)) { + LOG.error("Invalid container released " + container); + return; + } + + /* remove the containers from the nodemanger */ + if (null != launchedContainers.remove(container.getId())) { + updateResource(container); + } + + LOG.info("Released container " + container.getId() + + " of capacity " + container.getResource() + " on host " + rmNode.getNodeAddress() + + ", which currently has " + numContainers + " containers, " + + getUsedResource() + " used and " + getAvailableResource() + + " available" + ", release resources=" + true); + } + + + private synchronized void addAvailableResource(Resource resource) { + if (resource == null) { + LOG.error("Invalid resource addition of null resource for " + + rmNode.getNodeAddress()); + return; + } + Resources.addTo(availableResource, resource); + Resources.subtractFrom(usedResource, resource); + } + + private synchronized void deductAvailableResource(Resource resource) { + if (resource == null) { + LOG.error("Invalid deduction of null resource for " + + rmNode.getNodeAddress()); + return; + } + Resources.subtractFrom(availableResource, resource); + Resources.addTo(usedResource, resource); + } + + @Override + public String toString() { + return "host: " + rmNode.getNodeAddress() + " #containers=" + getNumContainers() + + " available=" + getAvailableResource().getMemory() + + " used=" + getUsedResource().getMemory(); + } + + @Override + public int getNumContainers() { + return numContainers; + } + + public synchronized List getRunningContainers() { + return new ArrayList(launchedContainers.values()); + } + public synchronized void reserveResource( - SchedulerApplicationAttempt application, Priority priority, - RMContainer container) { + SchedulerApplicationAttempt application, Priority priority, + RMContainer reservedContainer) { // Check if it's already reserved - RMContainer reservedContainer = getReservedContainer(); - if (reservedContainer != null) { + if (this.reservedContainer != null) { // Sanity check - if (!container.getContainer().getNodeId().equals(getNodeID())) { + if (!reservedContainer.getContainer().getNodeId().equals(getNodeID())) { throw new IllegalStateException("Trying to reserve" + - " container " + container + - " on node " + container.getReservedNode() + - " when currently" + " reserved resource " + reservedContainer + - " on node " + reservedContainer.getReservedNode()); + " container " + reservedContainer + + " on node " + reservedContainer.getReservedNode() + + " when currently" + " reserved resource " + this.reservedContainer + + " on node " + this.reservedContainer.getReservedNode()); } // Cannot reserve more than one application attempt on a given node! // Reservation is still against attempt. - if (!reservedContainer.getContainer().getId().getApplicationAttemptId() - .equals(container.getContainer().getId().getApplicationAttemptId())) { + if (!this.reservedContainer.getContainer().getId().getApplicationAttemptId().equals( + reservedContainer.getContainer().getId().getApplicationAttemptId())) { throw new IllegalStateException("Trying to reserve" + - " container " + container + + " container " + reservedContainer + " for application " + application.getApplicationAttemptId() + " when currently" + - " reserved container " + reservedContainer + + " reserved container " + this.reservedContainer + " on node " + this); } if (LOG.isDebugEnabled()) { LOG.debug("Updated reserved container " - + container.getContainer().getId() + " on node " + this + + reservedContainer.getContainer().getId() + " on node " + this + " for application attempt " + application.getApplicationAttemptId()); } } else { if (LOG.isDebugEnabled()) { LOG.debug("Reserved container " - + container.getContainer().getId() + " on node " + this + + reservedContainer.getContainer().getId() + " on node " + this + " for application attempt " + application.getApplicationAttemptId()); } } - setReservedContainer(container); + this.reservedContainer = reservedContainer; } - @Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { - + // adding NP checks as this can now be called for preemption - if (getReservedContainer() != null - && getReservedContainer().getContainer() != null - && getReservedContainer().getContainer().getId() != null - && getReservedContainer().getContainer().getId() - .getApplicationAttemptId() != null) { + if (reservedContainer != null + && reservedContainer.getContainer() != null + && reservedContainer.getContainer().getId() != null + && reservedContainer.getContainer().getId().getApplicationAttemptId() != null) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = - getReservedContainer().getContainer().getId() - .getApplicationAttemptId(); + reservedContainer.getContainer().getId().getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + @@ -105,6 +269,17 @@ public synchronized void unreserveResource( " on node " + this); } } - setReservedContainer(null); + reservedContainer = null; + } + + public synchronized RMContainer getReservedContainer() { + return reservedContainer; } + + @Override + public synchronized void applyDeltaOnAvailableResource(Resource deltaResource) { + // we can only adjust available resource if total resource is changed. + Resources.addTo(this.availableResource, deltaResource); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 1e94046..c265fcf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -119,9 +119,9 @@ public QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive) { // TODO: we might change these queue metrics around a little bit // to match the semantics of the fair scheduler. queueInfo.setCapacity((float) getFairShare().getMemory() / - scheduler.getClusterResource().getMemory()); + scheduler.getClusterCapacity().getMemory()); queueInfo.setCapacity((float) getResourceUsage().getMemory() / - scheduler.getClusterResource().getMemory()); + scheduler.getClusterCapacity().getMemory()); ArrayList childQueueInfos = new ArrayList(); if (includeChildQueues) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java index 69f2ab3..97ea6d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java @@ -18,16 +18,28 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.util.resource.Resources; @Private @Unstable @@ -35,56 +47,208 @@ private static final Log LOG = LogFactory.getLog(FSSchedulerNode.class); + private static final RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + + private Resource availableResource; + private Resource usedResource = recordFactory.newRecordInstance(Resource.class); + private Resource totalResourceCapability; + + private volatile int numContainers; + + private RMContainer reservedContainer; private AppSchedulable reservedAppSchedulable; + + /* set of containers that are allocated containers */ + private final Map launchedContainers = + new HashMap(); + + private final RMNode rmNode; + private final String nodeName; public FSSchedulerNode(RMNode node, boolean usePortForNodeName) { - super(node, usePortForNodeName); + this.rmNode = node; + this.availableResource = Resources.clone(node.getTotalCapability()); + totalResourceCapability = + Resource.newInstance(node.getTotalCapability().getMemory(), node + .getTotalCapability().getVirtualCores()); + if (usePortForNodeName) { + nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort(); + } else { + nodeName = rmNode.getHostName(); + } + } + + public RMNode getRMNode() { + return rmNode; + } + + public NodeId getNodeID() { + return rmNode.getNodeID(); + } + + public String getHttpAddress() { + return rmNode.getHttpAddress(); + } + + @Override + public String getNodeName() { + return nodeName; } @Override + public String getRackName() { + return rmNode.getRackName(); + } + + /** + * The Scheduler has allocated containers on this node to the + * given application. + * + * @param applicationId application + * @param rmContainer allocated container + */ + public synchronized void allocateContainer(ApplicationId applicationId, + RMContainer rmContainer) { + Container container = rmContainer.getContainer(); + deductAvailableResource(container.getResource()); + ++numContainers; + + launchedContainers.put(container.getId(), rmContainer); + + LOG.info("Assigned container " + container.getId() + + " of capacity " + container.getResource() + " on host " + rmNode.getNodeAddress() + + ", which currently has " + numContainers + " containers, " + + getUsedResource() + " used and " + + getAvailableResource() + " available"); + } + + @Override + public synchronized Resource getAvailableResource() { + return availableResource; + } + + @Override + public synchronized Resource getUsedResource() { + return usedResource; + } + + private synchronized boolean isValidContainer(Container c) { + if (launchedContainers.containsKey(c.getId())) { + return true; + } + return false; + } + + private synchronized void updateResource(Container container) { + addAvailableResource(container.getResource()); + --numContainers; + } + + /** + * Release an allocated container on this node. + * @param container container to be released + */ + public synchronized void releaseContainer(Container container) { + if (!isValidContainer(container)) { + LOG.error("Invalid container released " + container); + return; + } + + /* remove the containers from the nodemanger */ + launchedContainers.remove(container.getId()); + updateResource(container); + + LOG.info("Released container " + container.getId() + + " of capacity " + container.getResource() + " on host " + rmNode.getNodeAddress() + + ", which currently has " + numContainers + " containers, " + + getUsedResource() + " used and " + getAvailableResource() + + " available" + ", release resources=" + true); + } + + + private synchronized void addAvailableResource(Resource resource) { + if (resource == null) { + LOG.error("Invalid resource addition of null resource for " + + rmNode.getNodeAddress()); + return; + } + Resources.addTo(availableResource, resource); + Resources.subtractFrom(usedResource, resource); + } + + @Override + public Resource getTotalResource() { + return this.totalResourceCapability; + } + + private synchronized void deductAvailableResource(Resource resource) { + if (resource == null) { + LOG.error("Invalid deduction of null resource for " + + rmNode.getNodeAddress()); + return; + } + Resources.subtractFrom(availableResource, resource); + Resources.addTo(usedResource, resource); + } + + @Override + public String toString() { + return "host: " + rmNode.getNodeAddress() + " #containers=" + getNumContainers() + + " available=" + getAvailableResource() + + " used=" + getUsedResource(); + } + + @Override + public int getNumContainers() { + return numContainers; + } + + public synchronized List getRunningContainers() { + return new ArrayList(launchedContainers.values()); + } + public synchronized void reserveResource( - SchedulerApplicationAttempt application, Priority priority, - RMContainer container) { + FSSchedulerApp application, Priority priority, + RMContainer reservedContainer) { // Check if it's already reserved - RMContainer reservedContainer = getReservedContainer(); - if (reservedContainer != null) { + if (this.reservedContainer != null) { // Sanity check - if (!container.getContainer().getNodeId().equals(getNodeID())) { + if (!reservedContainer.getContainer().getNodeId().equals(getNodeID())) { throw new IllegalStateException("Trying to reserve" + - " container " + container + - " on node " + container.getReservedNode() + - " when currently" + " reserved resource " + reservedContainer + - " on node " + reservedContainer.getReservedNode()); + " container " + reservedContainer + + " on node " + reservedContainer.getReservedNode() + + " when currently" + " reserved resource " + this.reservedContainer + + " on node " + this.reservedContainer.getReservedNode()); } // Cannot reserve more than one application on a given node! - if (!reservedContainer.getContainer().getId().getApplicationAttemptId() - .equals(container.getContainer().getId().getApplicationAttemptId())) { + if (!this.reservedContainer.getContainer().getId().getApplicationAttemptId().equals( + reservedContainer.getContainer().getId().getApplicationAttemptId())) { throw new IllegalStateException("Trying to reserve" + - " container " + container + + " container " + reservedContainer + " for application " + application.getApplicationId() + " when currently" + - " reserved container " + reservedContainer + + " reserved container " + this.reservedContainer + " on node " + this); } LOG.info("Updated reserved container " + - container.getContainer().getId() + " on node " + + reservedContainer.getContainer().getId() + " on node " + this + " for application " + application); } else { - LOG.info("Reserved container " + container.getContainer().getId() + + LOG.info("Reserved container " + reservedContainer.getContainer().getId() + " on node " + this + " for application " + application); } - setReservedContainer(container); - this.reservedAppSchedulable = ((FSSchedulerApp) application).getAppSchedulable(); + this.reservedContainer = reservedContainer; + this.reservedAppSchedulable = application.getAppSchedulable(); } - @Override public synchronized void unreserveResource( - SchedulerApplicationAttempt application) { + FSSchedulerApp application) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = - getReservedContainer().getContainer().getId().getApplicationAttemptId(); + reservedContainer.getContainer().getId().getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + @@ -94,11 +258,22 @@ public synchronized void unreserveResource( " on node " + this); } - setReservedContainer(null); + this.reservedContainer = null; this.reservedAppSchedulable = null; } + public synchronized RMContainer getReservedContainer() { + return reservedContainer; + } + public synchronized AppSchedulable getReservedAppSchedulable() { return reservedAppSchedulable; } + + @Override + public synchronized void applyDeltaOnAvailableResource(Resource deltaResource) { + // we can only adjust available resource if total resource is changed. + Resources.addTo(this.availableResource, deltaResource); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 830f6f7..fab9ebe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -39,6 +39,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -74,8 +76,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.ContainersAndNMTokensAllocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -118,11 +122,11 @@ @LimitedPrivate("yarn") @Unstable @SuppressWarnings("unchecked") -public class FairScheduler extends - AbstractYarnScheduler { +public class FairScheduler extends AbstractYarnScheduler { private boolean initialized; private FairSchedulerConfiguration conf; - + private Resource minimumAllocation; + private Resource maximumAllocation; private Resource incrAllocation; private QueueManager queueMgr; private Clock clock; @@ -148,11 +152,15 @@ // Time we last ran preemptTasksIfNecessary private long lastPreemptCheckTime; - // Preemption related variables - protected boolean preemptionEnabled; - protected float preemptionUtilizationThreshold; + // Nodes in the cluster, indexed by NodeId + private Map nodes = + new ConcurrentHashMap(); + + // Aggregate capacity of the cluster + private Resource clusterCapacity = + RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Resource.class); - // How often tasks are preempted + // How often tasks are preempted protected long preemptionInterval; // ms to wait before force killing stuff (must be longer than a couple @@ -162,6 +170,7 @@ // Containers whose AMs have been warned that they will be preempted soon. private List warnedContainers = new ArrayList(); + protected boolean preemptionEnabled; protected boolean sizeBasedWeight; // Give larger weights to larger jobs protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not @@ -237,6 +246,23 @@ public QueueManager getQueueManager() { return queueMgr; } + @Override + public RMContainer getRMContainer(ContainerId containerId) { + FSSchedulerApp attempt = getCurrentAttemptForContainer(containerId); + return (attempt == null) ? null : attempt.getRMContainer(containerId); + } + + private FSSchedulerApp getCurrentAttemptForContainer( + ContainerId containerId) { + SchedulerApplication app = + applications.get(containerId.getApplicationAttemptId() + .getApplicationId()); + if (app != null) { + return (FSSchedulerApp) app.getCurrentAppAttempt(); + } + return null; + } + /** * A runnable which calls {@link FairScheduler#update()} every * UPDATE_INTERVAL milliseconds. @@ -268,7 +294,7 @@ protected synchronized void update() { // Recursively update demands for all queues rootQueue.updateDemand(); - rootQueue.setFairShare(clusterResource); + rootQueue.setFairShare(clusterCapacity); // Recursively compute fair shares for all queues // and update metrics rootQueue.recomputeShares(); @@ -296,9 +322,9 @@ private void updatePreemptionVariables() { * Is a queue below its min share for the given task type? */ boolean isStarvedForMinShare(FSLeafQueue sched) { - Resource desiredShare = Resources.min(RESOURCE_CALCULATOR, clusterResource, + Resource desiredShare = Resources.min(RESOURCE_CALCULATOR, clusterCapacity, sched.getMinShare(), sched.getDemand()); - return Resources.lessThan(RESOURCE_CALCULATOR, clusterResource, + return Resources.lessThan(RESOURCE_CALCULATOR, clusterCapacity, sched.getResourceUsage(), desiredShare); } @@ -307,9 +333,9 @@ boolean isStarvedForMinShare(FSLeafQueue sched) { * defined as being below half its fair share. */ boolean isStarvedForFairShare(FSLeafQueue sched) { - Resource desiredFairShare = Resources.min(RESOURCE_CALCULATOR, clusterResource, + Resource desiredFairShare = Resources.min(RESOURCE_CALCULATOR, clusterCapacity, Resources.multiply(sched.getFairShare(), .5), sched.getDemand()); - return Resources.lessThan(RESOURCE_CALCULATOR, clusterResource, + return Resources.lessThan(RESOURCE_CALCULATOR, clusterCapacity, sched.getResourceUsage(), desiredFairShare); } @@ -321,7 +347,7 @@ boolean isStarvedForFairShare(FSLeafQueue sched) { * and then select the right ones using preemptTasks. */ protected synchronized void preemptTasksIfNecessary() { - if (!shouldAttemptPreemption()) { + if (!preemptionEnabled) { return; } @@ -331,11 +357,12 @@ protected synchronized void preemptTasksIfNecessary() { } lastPreemptCheckTime = curTime; - Resource resToPreempt = Resources.clone(Resources.none()); + Resource resToPreempt = Resources.none(); + for (FSLeafQueue sched : queueMgr.getLeafQueues()) { - Resources.addTo(resToPreempt, resToPreempt(sched, curTime)); + resToPreempt = Resources.add(resToPreempt, resToPreempt(sched, curTime)); } - if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, resToPreempt, + if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity, resToPreempt, Resources.none())) { preemptResources(queueMgr.getLeafQueues(), resToPreempt); } @@ -362,7 +389,7 @@ protected void preemptResources(Collection scheds, // Collect running containers from over-scheduled queues List runningContainers = new ArrayList(); for (FSLeafQueue sched : scheds) { - if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, + if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity, sched.getResourceUsage(), sched.getFairShare())) { for (AppSchedulable as : sched.getRunnableAppSchedulables()) { for (RMContainer c : as.getApp().getLiveContainers()) { @@ -394,7 +421,7 @@ public int compare(RMContainer c1, RMContainer c2) { while (warnedIter.hasNext()) { RMContainer container = warnedIter.next(); if (container.getState() == RMContainerState.RUNNING && - Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, + Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity, toPreempt, Resources.none())) { warnOrKillContainer(container, apps.get(container), queues.get(container)); preemptedThisRound.add(container); @@ -408,12 +435,12 @@ public int compare(RMContainer c1, RMContainer c2) { // sure we don't preempt too many from any queue Iterator runningIter = runningContainers.iterator(); while (runningIter.hasNext() && - Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, + Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity, toPreempt, Resources.none())) { RMContainer container = runningIter.next(); FSLeafQueue sched = queues.get(container); if (!preemptedThisRound.contains(container) && - Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, + Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity, sched.getResourceUsage(), sched.getFairShare())) { warnOrKillContainer(container, apps.get(container), sched); @@ -469,20 +496,20 @@ protected Resource resToPreempt(FSLeafQueue sched, long curTime) { Resource resDueToMinShare = Resources.none(); Resource resDueToFairShare = Resources.none(); if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) { - Resource target = Resources.min(RESOURCE_CALCULATOR, clusterResource, + Resource target = Resources.min(RESOURCE_CALCULATOR, clusterCapacity, sched.getMinShare(), sched.getDemand()); - resDueToMinShare = Resources.max(RESOURCE_CALCULATOR, clusterResource, + resDueToMinShare = Resources.max(RESOURCE_CALCULATOR, clusterCapacity, Resources.none(), Resources.subtract(target, sched.getResourceUsage())); } if (curTime - sched.getLastTimeAtHalfFairShare() > fairShareTimeout) { - Resource target = Resources.min(RESOURCE_CALCULATOR, clusterResource, + Resource target = Resources.min(RESOURCE_CALCULATOR, clusterCapacity, sched.getFairShare(), sched.getDemand()); - resDueToFairShare = Resources.max(RESOURCE_CALCULATOR, clusterResource, + resDueToFairShare = Resources.max(RESOURCE_CALCULATOR, clusterCapacity, Resources.none(), Resources.subtract(target, sched.getResourceUsage())); } - Resource resToPreempt = Resources.max(RESOURCE_CALCULATOR, clusterResource, + Resource resToPreempt = Resources.max(RESOURCE_CALCULATOR, clusterCapacity, resDueToMinShare, resDueToFairShare); - if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, + if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity, resToPreempt, Resources.none())) { String message = "Should preempt " + resToPreempt + " res for queue " + sched.getName() + ": resDueToMinShare = " + resDueToMinShare @@ -513,12 +540,18 @@ public synchronized ResourceWeights getAppWeight(AppSchedulable app) { return resourceWeights; } + @Override + public Resource getMinimumResourceCapability() { + return minimumAllocation; + } + public Resource getIncrementResourceCapability() { return incrAllocation; } - private FSSchedulerNode getFSSchedulerNode(NodeId nodeId) { - return nodes.get(nodeId); + @Override + public Resource getMaximumResourceCapability() { + return maximumAllocation; } public double getNodeLocalityThreshold() { @@ -545,6 +578,10 @@ public synchronized int getContinuousSchedulingSleepMs() { return continuousSchedulingSleepMs; } + public Resource getClusterCapacity() { + return clusterCapacity; + } + public synchronized Clock getClock() { return clock; } @@ -592,8 +629,8 @@ protected synchronized void addApplication(ApplicationId applicationId, return; } - SchedulerApplication application = - new SchedulerApplication(queue, user); + SchedulerApplication application = + new SchedulerApplication(queue, user); applications.put(applicationId, application); queue.getMetrics().submitApp(user); @@ -610,7 +647,7 @@ protected synchronized void addApplication(ApplicationId applicationId, protected synchronized void addApplicationAttempt( ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt) { - SchedulerApplication application = + SchedulerApplication application = applications.get(applicationAttemptId.getApplicationId()); String user = application.getUser(); FSLeafQueue queue = (FSLeafQueue) application.getQueue(); @@ -683,8 +720,7 @@ FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { private synchronized void removeApplication(ApplicationId applicationId, RMAppState finalState) { - SchedulerApplication application = - applications.get(applicationId); + SchedulerApplication application = applications.get(applicationId); if (application == null){ LOG.warn("Couldn't find application " + applicationId); return; @@ -698,7 +734,7 @@ private synchronized void removeApplicationAttempt( RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { LOG.info("Application " + applicationAttemptId + " is done." + " finalState=" + rmAppAttemptFinalState); - SchedulerApplication application = + SchedulerApplication application = applications.get(applicationAttemptId.getApplicationId()); FSSchedulerApp attempt = getSchedulerApp(applicationAttemptId); @@ -773,7 +809,7 @@ private synchronized void completedContainer(RMContainer rmContainer, } // Get the node on which the container was allocated - FSSchedulerNode node = getFSSchedulerNode(container.getNodeId()); + FSSchedulerNode node = nodes.get(container.getNodeId()); if (rmContainer.getState() == RMContainerState.RESERVED) { application.unreserve(node, rmContainer.getReservedPriority()); @@ -791,20 +827,20 @@ private synchronized void completedContainer(RMContainer rmContainer, private synchronized void addNode(RMNode node) { nodes.put(node.getNodeID(), new FSSchedulerNode(node, usePortForNodeName)); - Resources.addTo(clusterResource, node.getTotalCapability()); + Resources.addTo(clusterCapacity, node.getTotalCapability()); updateRootQueueMetrics(); LOG.info("Added node " + node.getNodeAddress() + - " cluster capacity: " + clusterResource); + " cluster capacity: " + clusterCapacity); } private synchronized void removeNode(RMNode rmNode) { - FSSchedulerNode node = getFSSchedulerNode(rmNode.getNodeID()); + FSSchedulerNode node = nodes.get(rmNode.getNodeID()); // This can occur when an UNHEALTHY node reconnects if (node == null) { return; } - Resources.subtractFrom(clusterResource, rmNode.getTotalCapability()); + Resources.subtractFrom(clusterCapacity, rmNode.getTotalCapability()); updateRootQueueMetrics(); // Remove running containers @@ -829,7 +865,7 @@ private synchronized void removeNode(RMNode rmNode) { nodes.remove(rmNode.getNodeID()); LOG.info("Removed node " + rmNode.getNodeAddress() + - " cluster capacity: " + clusterResource); + " cluster capacity: " + clusterCapacity); } @Override @@ -846,7 +882,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, // Sanity check SchedulerUtils.normalizeRequests(ask, new DominantResourceCalculator(), - clusterResource, minimumAllocation, maximumAllocation, incrAllocation); + clusterCapacity, minimumAllocation, maximumAllocation, incrAllocation); // Release containers for (ContainerId releasedContainerId : release) { @@ -925,13 +961,13 @@ private void containerLaunchedOnNode(ContainerId containerId, FSSchedulerNode no */ private synchronized void nodeUpdate(RMNode nm) { if (LOG.isDebugEnabled()) { - LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + clusterResource); + LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + clusterCapacity); } eventLog.log("HEARTBEAT", nm.getHostName()); - FSSchedulerNode node = getFSSchedulerNode(nm.getNodeID()); + FSSchedulerNode node = nodes.get(nm.getNodeID()); // Update resource if any change - SchedulerUtils.updateResourceIfChanged(node, nm, clusterResource, LOG); + SchedulerUtils.updateResourceIfChanged(node, nm, clusterCapacity, LOG); List containerInfoList = nm.pullContainerUpdates(); List newlyLaunchedContainers = new ArrayList(); @@ -976,7 +1012,7 @@ private void continuousScheduling() { // iterate all nodes for (NodeId nodeId : nodeIdList) { if (nodes.containsKey(nodeId)) { - FSSchedulerNode node = getFSSchedulerNode(nodeId); + FSSchedulerNode node = nodes.get(nodeId); try { if (Resources.fitsIn(minimumAllocation, node.getAvailableResource())) { @@ -1002,7 +1038,7 @@ private void continuousScheduling() { @Override public int compare(NodeId n1, NodeId n2) { - return RESOURCE_CALCULATOR.compare(clusterResource, + return RESOURCE_CALCULATOR.compare(clusterCapacity, nodes.get(n2).getAvailableResource(), nodes.get(n1).getAvailableResource()); } @@ -1039,7 +1075,7 @@ private synchronized void attemptScheduling(FSSchedulerNode node) { int assignedContainers = 0; while (node.getReservedContainer() == null) { boolean assignedContainer = false; - if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, + if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterCapacity, queueMgr.getRootQueue().assignContainer(node), Resources.none())) { assignedContainers++; @@ -1053,8 +1089,45 @@ private synchronized void attemptScheduling(FSSchedulerNode node) { updateRootQueueMetrics(); } + @Override + public SchedulerNodeReport getNodeReport(NodeId nodeId) { + FSSchedulerNode node = nodes.get(nodeId); + return node == null ? null : new SchedulerNodeReport(node); + } + public FSSchedulerApp getSchedulerApp(ApplicationAttemptId appAttemptId) { - return (FSSchedulerApp) super.getApplicationAttempt(appAttemptId); + SchedulerApplication app = + applications.get(appAttemptId.getApplicationId()); + if (app != null) { + return (FSSchedulerApp) app.getCurrentAppAttempt(); + } + return null; + } + + @Override + public SchedulerAppReport getSchedulerAppInfo( + ApplicationAttemptId appAttemptId) { + FSSchedulerApp attempt = getSchedulerApp(appAttemptId); + if (attempt == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Request for appInfo of unknown attempt " + appAttemptId); + } + return null; + } + return new SchedulerAppReport(attempt); + } + + @Override + public ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId appAttemptId) { + FSSchedulerApp attempt = getSchedulerApp(appAttemptId); + if (attempt == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Request for appInfo of unknown attempt " + appAttemptId); + } + return null; + } + return attempt.getResourceUsageReport(); } /** @@ -1066,23 +1139,7 @@ public FSSchedulerApp getSchedulerApp(ApplicationAttemptId appAttemptId) { private void updateRootQueueMetrics() { rootMetrics.setAvailableResourcesToQueue( Resources.subtract( - clusterResource, rootMetrics.getAllocatedResources())); - } - - /** - * Check if preemption is enabled and the utilization threshold for - * preemption is met. - * - * @return true if preemption should be attempted, false otherwise. - */ - private boolean shouldAttemptPreemption() { - if (preemptionEnabled) { - return (preemptionUtilizationThreshold < Math.max( - (float) rootMetrics.getAvailableMB() / clusterResource.getMemory(), - (float) rootMetrics.getAvailableVirtualCores() / - clusterResource.getVirtualCores())); - } - return false; + clusterCapacity, rootMetrics.getAllocatedResources())); } @Override @@ -1190,8 +1247,6 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext) nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs(); rackLocalityDelayMs = this.conf.getLocalityDelayRackMs(); preemptionEnabled = this.conf.getPreemptionEnabled(); - preemptionUtilizationThreshold = - this.conf.getPreemptionUtilizationThreshold(); assignMultiple = this.conf.getAssignMultiple(); maxAssign = this.conf.getMaxAssign(); sizeBasedWeight = this.conf.getSizeBasedWeight(); @@ -1203,7 +1258,7 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext) this.rmContext = rmContext; // This stores per-application scheduling information this.applications = - new ConcurrentHashMap>(); + new ConcurrentHashMap(); this.eventLog = new FairSchedulerEventLog(); eventLog.init(this.conf); @@ -1310,7 +1365,7 @@ public void onReload(AllocationConfiguration queueInfo) { // if it does not already exist, so it can be displayed on the web UI. synchronized (FairScheduler.this) { allocConf = queueInfo; - allocConf.getDefaultSchedulingPolicy().initialize(clusterResource); + allocConf.getDefaultSchedulingPolicy().initialize(clusterCapacity); queueMgr.updateAllocationConfiguration(allocConf); } } @@ -1330,7 +1385,7 @@ public void onReload(AllocationConfiguration queueInfo) { @Override public synchronized String moveApplication(ApplicationId appId, String queueName) throws YarnException { - SchedulerApplication app = applications.get(appId); + SchedulerApplication app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); } @@ -1394,8 +1449,8 @@ private void verifyMoveDoesNotViolateConstraints(FSSchedulerApp app, * Helper for moveApplication, which has appropriate synchronization, so all * operations will be atomic. */ - private void executeMove(SchedulerApplication app, - FSSchedulerApp attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) { + private void executeMove(SchedulerApplication app, FSSchedulerApp attempt, + FSLeafQueue oldQueue, FSLeafQueue newQueue) { boolean wasRunnable = oldQueue.removeApp(attempt); // if app was not runnable before, it may be runnable now boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 0fd242d..e271b05 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -101,10 +101,6 @@ /** Whether preemption is enabled. */ protected static final String PREEMPTION = CONF_PREFIX + "preemption"; protected static final boolean DEFAULT_PREEMPTION = false; - - protected static final String PREEMPTION_THRESHOLD = - CONF_PREFIX + "preemption.cluster-utilization-threshold"; - protected static final float DEFAULT_PREEMPTION_THRESHOLD = 0.8f; protected static final String PREEMPTION_INTERVAL = CONF_PREFIX + "preemptionInterval"; protected static final int DEFAULT_PREEMPTION_INTERVAL = 5000; @@ -189,10 +185,6 @@ public boolean getPreemptionEnabled() { return getBoolean(PREEMPTION, DEFAULT_PREEMPTION); } - public float getPreemptionUtilizationThreshold() { - return getFloat(PREEMPTION_THRESHOLD, DEFAULT_PREEMPTION_THRESHOLD); - } - public boolean getAssignMultiple() { return getBoolean(ASSIGN_MULTIPLE, DEFAULT_ASSIGN_MULTIPLE); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 4f8735b..44918f3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -369,7 +369,7 @@ public void updateAllocationConfiguration(AllocationConfiguration queueConf) { // Set scheduling policies try { SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName()); - policy.initialize(scheduler.getClusterResource()); + policy.initialize(scheduler.getClusterCapacity()); queue.setPolicy(policy); } catch (AllocationConfigurationException ex) { LOG.warn("Cannot apply configured scheduling policy to queue " diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java index 30ea213..3ab6036 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java @@ -25,8 +25,6 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Groups; import org.apache.hadoop.util.ReflectionUtils; @@ -34,8 +32,6 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; -@Private -@Unstable public class QueuePlacementPolicy { private static final Map> ruleClasses; static { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java index 056df57..530efac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java @@ -23,8 +23,6 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.Groups; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.w3c.dom.Element; @@ -34,8 +32,6 @@ import com.google.common.annotations.VisibleForTesting; -@Private -@Unstable public abstract class QueuePlacementRule { protected boolean create; @@ -276,48 +272,21 @@ public boolean isTerminal() { } /** - * Places apps in the specified default queue. If no default queue is - * specified the app is placed in root.default queue. + * Places all apps in the default queue */ public static class Default extends QueuePlacementRule { - @VisibleForTesting - String defaultQueueName; - - @Override - public QueuePlacementRule initialize(boolean create, - Map args) { - if (defaultQueueName == null) { - defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; - } - return super.initialize(create, args); - } - - @Override - public void initializeFromXml(Element el) - throws AllocationConfigurationException { - defaultQueueName = el.getAttribute("queue"); - if (defaultQueueName != null && !defaultQueueName.isEmpty()) { - if (!defaultQueueName.startsWith("root.")) { - defaultQueueName = "root." + defaultQueueName; - } - } else { - defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; - } - super.initializeFromXml(el); - } - @Override protected String getQueueForApp(String requestedQueue, String user, Groups groups, Map> configuredQueues) { - return defaultQueueName; + return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; } - + @Override public boolean isTerminal() { return true; } } - + /** * Rejects all apps */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index bc3441b..21fcdec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListMap; import org.apache.commons.logging.Log; @@ -37,6 +38,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -74,9 +76,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.ContainersAndNMTokensAllocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; @@ -100,8 +104,7 @@ @LimitedPrivate("yarn") @Evolving @SuppressWarnings("unchecked") -public class FifoScheduler extends - AbstractYarnScheduler implements +public class FifoScheduler extends AbstractYarnScheduler implements Configurable { private static final Log LOG = LogFactory.getLog(FifoScheduler.class); @@ -111,7 +114,11 @@ Configuration conf; + protected Map nodes = new ConcurrentHashMap(); + private boolean initialized; + private Resource minimumAllocation; + private Resource maximumAllocation; private boolean usePortForNodeName; private ActiveUsersManager activeUsersManager; @@ -211,9 +218,19 @@ public synchronized Configuration getConf() { } @Override + public Resource getMinimumResourceCapability() { + return minimumAllocation; + } + + @Override public int getNumClusterNodes() { return nodes.size(); } + + @Override + public Resource getMaximumResourceCapability() { + return maximumAllocation; + } @Override public synchronized void @@ -225,7 +242,7 @@ public int getNumClusterNodes() { this.rmContext = rmContext; //Use ConcurrentSkipListMap because applications need to be ordered this.applications = - new ConcurrentSkipListMap>(); + new ConcurrentSkipListMap(); this.minimumAllocation = Resources.createResource(conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, @@ -315,6 +332,30 @@ public Allocation allocate( } } + @VisibleForTesting + FiCaSchedulerApp getApplicationAttempt(ApplicationAttemptId applicationAttemptId) { + SchedulerApplication app = + applications.get(applicationAttemptId.getApplicationId()); + if (app != null) { + return (FiCaSchedulerApp) app.getCurrentAppAttempt(); + } + return null; + } + + @Override + public SchedulerAppReport getSchedulerAppInfo( + ApplicationAttemptId applicationAttemptId) { + FiCaSchedulerApp app = getApplicationAttempt(applicationAttemptId); + return app == null ? null : new SchedulerAppReport(app); + } + + @Override + public ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId applicationAttemptId) { + FiCaSchedulerApp app = getApplicationAttempt(applicationAttemptId); + return app == null ? null : app.getResourceUsageReport(); + } + private FiCaSchedulerNode getNode(NodeId nodeId) { return nodes.get(nodeId); } @@ -322,8 +363,8 @@ private FiCaSchedulerNode getNode(NodeId nodeId) { @VisibleForTesting public synchronized void addApplication(ApplicationId applicationId, String queue, String user) { - SchedulerApplication application = - new SchedulerApplication(DEFAULT_QUEUE, user); + SchedulerApplication application = + new SchedulerApplication(DEFAULT_QUEUE, user); applications.put(applicationId, application); metrics.submitApp(user); LOG.info("Accepted application " + applicationId + " from user: " + user @@ -336,7 +377,7 @@ public synchronized void addApplication(ApplicationId applicationId, public synchronized void addApplicationAttempt(ApplicationAttemptId appAttemptId, boolean transferStateFromPreviousAttempt) { - SchedulerApplication application = + SchedulerApplication application = applications.get(appAttemptId.getApplicationId()); String user = application.getUser(); // TODO: Fix store @@ -360,8 +401,7 @@ public synchronized void addApplication(ApplicationId applicationId, private synchronized void doneApplication(ApplicationId applicationId, RMAppState finalState) { - SchedulerApplication application = - applications.get(applicationId); + SchedulerApplication application = applications.get(applicationId); if (application == null){ LOG.warn("Couldn't find application " + applicationId); return; @@ -379,7 +419,7 @@ private synchronized void doneApplicationAttempt( RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) throws IOException { FiCaSchedulerApp attempt = getApplicationAttempt(applicationAttemptId); - SchedulerApplication application = + SchedulerApplication application = applications.get(applicationAttemptId.getApplicationId()); if (application == null || attempt == null) { throw new IOException("Unknown application " + applicationAttemptId + @@ -416,13 +456,13 @@ private void assignContainers(FiCaSchedulerNode node) { " #applications=" + applications.size()); // Try to assign containers to applications in fifo order - for (Map.Entry> e : applications + for (Map.Entry e : applications .entrySet()) { - FiCaSchedulerApp application = e.getValue().getCurrentAppAttempt(); + FiCaSchedulerApp application = + (FiCaSchedulerApp) e.getValue().getCurrentAppAttempt(); if (application == null) { continue; } - LOG.debug("pre-assignContainers"); application.showRequests(); synchronized (application) { @@ -459,7 +499,7 @@ private void assignContainers(FiCaSchedulerNode node) { // Update the applications' headroom to correctly take into // account the containers assigned in this update. - for (SchedulerApplication application : applications.values()) { + for (SchedulerApplication application : applications.values()) { FiCaSchedulerApp attempt = (FiCaSchedulerApp) application.getCurrentAppAttempt(); if (attempt == null) { @@ -824,6 +864,7 @@ private synchronized void containerCompleted(RMContainer rmContainer, } + private Resource clusterResource = recordFactory.newRecordInstance(Resource.class); private Resource usedResource = recordFactory.newRecordInstance(Resource.class); private synchronized void removeNode(RMNode nodeInfo) { @@ -870,11 +911,28 @@ public void recover(RMState state) { } @Override + public synchronized SchedulerNodeReport getNodeReport(NodeId nodeId) { + FiCaSchedulerNode node = getNode(nodeId); + return node == null ? null : new SchedulerNodeReport(node); + } + + @Override public RMContainer getRMContainer(ContainerId containerId) { FiCaSchedulerApp attempt = getCurrentAttemptForContainer(containerId); return (attempt == null) ? null : attempt.getRMContainer(containerId); } + private FiCaSchedulerApp getCurrentAttemptForContainer( + ContainerId containerId) { + SchedulerApplication app = + applications.get(containerId.getApplicationAttemptId() + .getApplicationId()); + if (app != null) { + return (FiCaSchedulerApp) app.getCurrentAppAttempt(); + } + return null; + } + @Override public QueueMetrics getRootQueueMetrics() { return DEFAULT_QUEUE.getMetrics(); @@ -885,14 +943,13 @@ public synchronized boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, String queueName) { return DEFAULT_QUEUE.hasAccess(acl, callerUGI); } - + @Override - public synchronized List - getAppsInQueue(String queueName) { + public synchronized List getAppsInQueue(String queueName) { if (queueName.equals(DEFAULT_QUEUE.getQueueName())) { - List attempts = - new ArrayList(applications.size()); - for (SchedulerApplication app : applications.values()) { + List attempts = new ArrayList( + applications.size()); + for (SchedulerApplication app : applications.values()) { attempts.add(app.getCurrentAppAttempt().getApplicationAttemptId()); } return attempts; @@ -900,4 +957,5 @@ public synchronized boolean checkAccess(UserGroupInformation callerUGI, return null; } } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index 2c1bc47..1c5a79b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -70,7 +70,7 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { queueName = queue.getName(); schedulingPolicy = queue.getPolicy().getName(); - clusterResources = new ResourceInfo(scheduler.getClusterResource()); + clusterResources = new ResourceInfo(scheduler.getClusterCapacity()); usedResources = new ResourceInfo(queue.getResourceUsage()); fractionMemUsed = (float)usedResources.getMemory() / @@ -81,7 +81,7 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { maxResources = new ResourceInfo(queue.getMaxShare()); maxResources = new ResourceInfo( Resources.componentwiseMin(queue.getMaxShare(), - scheduler.getClusterResource())); + scheduler.getClusterCapacity())); fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory(); fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 6a05123..49eff8b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -1638,16 +1638,10 @@ public void testQueueMetricsOnRMRestart() throws Exception { // create app that gets launched and does allocate before RM restart RMApp app1 = rm1.submitApp(200); - // Need to wait first for AppAttempt to be started (RMAppState.ACCEPTED) - // and then for it to reach RMAppAttemptState.SCHEDULED - // inorder to ensure appsPending metric is incremented - rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); - RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); - ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId(); - rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED); assertQueueMetrics(qm1, 1, 1, 0, 0); - nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId(); rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED); MockAM am1 = rm1.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); @@ -1666,12 +1660,11 @@ public void testQueueMetricsOnRMRestart() throws Exception { // PHASE 2: create new RM and start from old state // create new RM to represent restart and recover state MockRM rm2 = new MockRM(conf, memStore); + rm2.start(); + nm1.setResourceTrackerService(rm2.getResourceTrackerService()); QueueMetrics qm2 = rm2.getResourceScheduler().getRootQueueMetrics(); resetQueueMetrics(qm2); assertQueueMetrics(qm2, 0, 0, 0, 0); - - rm2.start(); - nm1.setResourceTrackerService(rm2.getResourceTrackerService()); // recover app RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId()); am1.setAMRMProtocol(rm2.getApplicationMasterService()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index 6a449f5..99b2f2e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -456,7 +456,7 @@ ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData) { Resource clusterResources = Resource.newInstance(leafAbsCapacities(qData[0], qData[7]), 0); - when(mCS.getClusterResource()).thenReturn(clusterResources); + when(mCS.getClusterResources()).thenReturn(clusterResources); return policy; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java index 7f4d66f..8dc3628 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java @@ -41,6 +41,7 @@ import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -203,7 +204,7 @@ public void testZKClientDisconnectAndReconnect() LOG.error(error, e); fail(error); } - Assert.assertEquals("newBytes", new String(ret)); + assertEquals("newBytes", new String(ret)); } @Test(timeout = 20000) @@ -232,7 +233,7 @@ public void testZKSessionTimeout() throws Exception { try { byte[] ret = store.getDataWithRetries(path, false); - Assert.assertEquals("bytes", new String(ret)); + assertEquals("bytes", new String(ret)); } catch (Exception e) { String error = "New session creation failed"; LOG.error(error, e); @@ -281,4 +282,24 @@ public void testZKAuths() throws Exception { zkClientTester.getRMStateStore(conf); } + + @Test + public void testZKRetryInterval() throws Exception { + TestZKClient zkClientTester = new TestZKClient(); + YarnConfiguration conf = new YarnConfiguration(); + + ZKRMStateStore store = + (ZKRMStateStore) zkClientTester.getRMStateStore(conf); + assertEquals(YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS, + store.zkRetryInterval); + store.stop(); + + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + store = + (ZKRMStateStore) zkClientTester.getRMStateStore(conf); + assertEquals(YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS / + YarnConfiguration.DEFAULT_ZK_RM_NUM_RETRIES, + store.zkRetryInterval); + store.stop(); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 460f35e..8fcbf54 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -384,18 +384,15 @@ public void testCreatePreemptedContainerStatus() { Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus()); } - public static SchedulerApplication - verifyAppAddedAndRemovedFromScheduler( - Map> applications, - EventHandler handler, String queueName) - throws Exception { + public static SchedulerApplication verifyAppAddedAndRemovedFromScheduler( + final Map applications, + EventHandler handler, String queueName) throws Exception { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appId, queueName, "user"); handler.handle(appAddedEvent); - SchedulerApplication app = - applications.get(appId); + SchedulerApplication app = applications.get(appId); // verify application is added. Assert.assertNotNull(app); Assert.assertEquals("user", app.getUser()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index a9a9975..2b548ef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -81,7 +81,7 @@ public void setUp() throws IOException { thenReturn(Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()). thenReturn(Resources.createResource(16*GB, 32)); - when(csContext.getClusterResource()). + when(csContext.getClusterResources()). thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32)); when(csContext.getApplicationComparator()). thenReturn(CapacityScheduler.applicationComparator); @@ -165,7 +165,7 @@ public void testLimitsComputation() throws Exception { // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16); - when(csContext.getClusterResource()).thenReturn(clusterResource); + when(csContext.getClusterResources()).thenReturn(clusterResource); Map queues = new HashMap(); CSQueue root = @@ -478,7 +478,7 @@ public void testHeadroom() throws Exception { // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB); - when(csContext.getClusterResource()).thenReturn(clusterResource); + when(csContext.getClusterResources()).thenReturn(clusterResource); Map queues = new HashMap(); CapacityScheduler.parseQueue(csContext, csConf, null, "root", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 1412039..d0ba334 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -29,6 +29,8 @@ import java.util.Comparator; import java.util.List; +import org.junit.Assert; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -54,11 +56,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -73,7 +72,6 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -467,14 +465,14 @@ public void testReconnectedNode() throws Exception { cs.handle(new NodeAddedSchedulerEvent(n1)); cs.handle(new NodeAddedSchedulerEvent(n2)); - Assert.assertEquals(6 * GB, cs.getClusterResource().getMemory()); + Assert.assertEquals(6 * GB, cs.getClusterResources().getMemory()); // reconnect n1 with downgraded memory n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1); cs.handle(new NodeRemovedSchedulerEvent(n1)); cs.handle(new NodeAddedSchedulerEvent(n1)); - Assert.assertEquals(4 * GB, cs.getClusterResource().getMemory()); + Assert.assertEquals(4 * GB, cs.getClusterResources().getMemory()); } @Test @@ -629,17 +627,17 @@ public void testGetAppsInQueue() throws Exception { @Test public void testAddAndRemoveAppFromCapacityScheduler() throws Exception { + + AsyncDispatcher rmDispatcher = new AsyncDispatcher(); + CapacityScheduler cs = new CapacityScheduler(); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - MockRM rm = new MockRM(conf); - @SuppressWarnings("unchecked") - AbstractYarnScheduler cs = - (AbstractYarnScheduler) rm - .getResourceScheduler(); + cs.reinitialize(conf, new RMContextImpl(rmDispatcher, null, null, null, + null, null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null)); - SchedulerApplication app = + SchedulerApplication app = TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( cs.getSchedulerApplications(), cs, "a1"); Assert.assertEquals("a1", app.getQueue().getQueueName()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index fd14ef6..d509771 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -89,7 +89,7 @@ public void setUp() throws Exception { Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(16*GB, 32)); - when(csContext.getClusterResource()). + when(csContext.getClusterResources()). thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32)); when(csContext.getApplicationComparator()). thenReturn(CapacityScheduler.applicationComparator); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 2a26d30..bdf89bb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -122,7 +122,7 @@ public void setUp() throws Exception { thenReturn(Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()). thenReturn(Resources.createResource(16*GB, 32)); - when(csContext.getClusterResource()). + when(csContext.getClusterResources()). thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32)); when(csContext.getApplicationComparator()). thenReturn(CapacityScheduler.applicationComparator); @@ -1651,7 +1651,7 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { newQueues, queues, TestUtils.spyHook); queues = newQueues; - root.reinitialize(newRoot, cs.getClusterResource()); + root.reinitialize(newRoot, cs.getClusterResources()); // after reinitialization assertEquals(3, e.activeApplications.size()); @@ -1676,7 +1676,7 @@ public void testNodeLocalityAfterQueueRefresh() throws Exception { newQueues, queues, TestUtils.spyHook); queues = newQueues; - root.reinitialize(newRoot, cs.getClusterResource()); + root.reinitialize(newRoot, cs.getClusterResources()); // after reinitialization assertEquals(60, e.getNodeLocalityDelay()); @@ -2070,7 +2070,7 @@ private CapacitySchedulerContext mockCSContext( when(csContext.getConfiguration()).thenReturn(csConf); when(csContext.getConf()).thenReturn(new YarnConfiguration()); when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); - when(csContext.getClusterResource()).thenReturn(clusterResource); + when(csContext.getClusterResources()).thenReturn(clusterResource); when(csContext.getMinimumResourceCapability()).thenReturn( Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()).thenReturn( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index fa9edb1..652fcf1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -86,7 +86,7 @@ public void setUp() throws Exception { Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(16*GB, 32)); - when(csContext.getClusterResource()). + when(csContext.getClusterResources()). thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32)); when(csContext.getApplicationComparator()). thenReturn(CapacityScheduler.applicationComparator); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java deleted file mode 100644 index 5f92676..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.Clock; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -public class FairSchedulerTestBase { - protected static class MockClock implements Clock { - private long time = 0; - @Override - public long getTime() { - return time; - } - - public void tick(int seconds) { - time = time + seconds * 1000; - } - } - - protected final static String TEST_DIR = - new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); - - private static RecordFactory - recordFactory = RecordFactoryProvider.getRecordFactory(null); - - protected int APP_ID = 1; // Incrementing counter for scheduling apps - protected int ATTEMPT_ID = 1; // Incrementing counter for scheduling attempts - - protected Configuration conf; - protected FairScheduler scheduler; - protected ResourceManager resourceManager; - - // Helper methods - protected Configuration createConfiguration() { - Configuration conf = new YarnConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, - ResourceScheduler.class); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); - conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, - 1024); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); - conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false); - conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f); - return conf; - } - - protected ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { - ApplicationId appIdImpl = ApplicationId.newInstance(0, appId); - return ApplicationAttemptId.newInstance(appIdImpl, attemptId); - } - - protected ResourceRequest createResourceRequest( - int memory, String host, int priority, int numContainers, - boolean relaxLocality) { - return createResourceRequest(memory, 1, host, priority, numContainers, - relaxLocality); - } - - protected ResourceRequest createResourceRequest( - int memory, int vcores, String host, int priority, int numContainers, - boolean relaxLocality) { - ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); - request.setCapability(BuilderUtils.newResource(memory, vcores)); - request.setResourceName(host); - request.setNumContainers(numContainers); - Priority prio = recordFactory.newRecordInstance(Priority.class); - prio.setPriority(priority); - request.setPriority(prio); - request.setRelaxLocality(relaxLocality); - return request; - } - - /** - * Creates a single container priority-1 request and submits to - * scheduler. - */ - protected ApplicationAttemptId createSchedulingRequest( - int memory, String queueId, String userId) { - return createSchedulingRequest(memory, queueId, userId, 1); - } - - protected ApplicationAttemptId createSchedulingRequest( - int memory, int vcores, String queueId, String userId) { - return createSchedulingRequest(memory, vcores, queueId, userId, 1); - } - - protected ApplicationAttemptId createSchedulingRequest( - int memory, String queueId, String userId, int numContainers) { - return createSchedulingRequest(memory, queueId, userId, numContainers, 1); - } - - protected ApplicationAttemptId createSchedulingRequest( - int memory, int vcores, String queueId, String userId, int numContainers) { - return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1); - } - - protected ApplicationAttemptId createSchedulingRequest( - int memory, String queueId, String userId, int numContainers, int priority) { - return createSchedulingRequest(memory, 1, queueId, userId, numContainers, - priority); - } - - protected ApplicationAttemptId createSchedulingRequest( - int memory, int vcores, String queueId, String userId, int numContainers, - int priority) { - ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(id.getApplicationId(), queueId, userId); - // This conditional is for testAclSubmitApplication where app is rejected - // and no app is added. - if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) { - scheduler.addApplicationAttempt(id, false); - } - List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, - priority, numContainers, true); - ask.add(request); - scheduler.allocate(id, ask, new ArrayList(), null, null); - return id; - } - - protected void createSchedulingRequestExistingApplication( - int memory, int priority, ApplicationAttemptId attId) { - ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, - priority, 1, true); - createSchedulingRequestExistingApplication(request, attId); - } - - protected void createSchedulingRequestExistingApplication( - int memory, int vcores, int priority, ApplicationAttemptId attId) { - ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, - priority, 1, true); - createSchedulingRequestExistingApplication(request, attId); - } - - protected void createSchedulingRequestExistingApplication( - ResourceRequest request, ApplicationAttemptId attId) { - List ask = new ArrayList(); - ask.add(request); - scheduler.allocate(attId, ask, new ArrayList(), null, null); - } -} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 276020f..2524763 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -43,6 +43,7 @@ import javax.xml.parsers.ParserConfigurationException; +import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -55,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; @@ -62,6 +64,8 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -76,11 +80,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -88,28 +89,58 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.xml.sax.SAXException; import com.google.common.collect.Sets; -@SuppressWarnings("unchecked") -public class TestFairScheduler extends FairSchedulerTestBase { - private final static String ALLOC_FILE = - new File(TEST_DIR, "test-queues").getAbsolutePath(); +public class TestFairScheduler { + + static class MockClock implements Clock { + private long time = 0; + @Override + public long getTime() { + return time; + } + + public void tick(int seconds) { + time = time + seconds * 1000; + } + + } + + final static String TEST_DIR = new File(System.getProperty("test.build.data", + "/tmp")).getAbsolutePath(); + final static String ALLOC_FILE = new File(TEST_DIR, + "test-queues").getAbsolutePath(); + + private FairScheduler scheduler; + private ResourceManager resourceManager; + private Configuration conf; + private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + + private int APP_ID = 1; // Incrementing counter for schedling apps + private int ATTEMPT_ID = 1; // Incrementing counter for scheduling attempts + + // HELPER METHODS @Before public void setUp() throws IOException { scheduler = new FairScheduler(); conf = createConfiguration(); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); + conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); + // All tests assume only one assignment per node update + conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new ResourceManager(); resourceManager.init(conf); @@ -163,6 +194,107 @@ public void testConfValidation() throws Exception { } } + private Configuration createConfiguration() { + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, + ResourceScheduler.class); + return conf; + } + + private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { + ApplicationId appIdImpl = ApplicationId.newInstance(0, appId); + ApplicationAttemptId attId = + ApplicationAttemptId.newInstance(appIdImpl, attemptId); + return attId; + } + + private ResourceRequest createResourceRequest(int memory, String host, + int priority, int numContainers, boolean relaxLocality) { + return createResourceRequest(memory, 1, host, priority, numContainers, + relaxLocality); + } + + private ResourceRequest createResourceRequest(int memory, int vcores, String host, + int priority, int numContainers, boolean relaxLocality) { + ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); + request.setCapability(BuilderUtils.newResource(memory, vcores)); + request.setResourceName(host); + request.setNumContainers(numContainers); + Priority prio = recordFactory.newRecordInstance(Priority.class); + prio.setPriority(priority); + request.setPriority(prio); + request.setRelaxLocality(relaxLocality); + return request; + } + + /** + * Creates a single container priority-1 request and submits to + * scheduler. + */ + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId) { + return createSchedulingRequest(memory, queueId, userId, 1); + } + + private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, + String queueId, String userId) { + return createSchedulingRequest(memory, vcores, queueId, userId, 1); + } + + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers) { + return createSchedulingRequest(memory, queueId, userId, numContainers, 1); + } + + private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, + String queueId, String userId, int numContainers) { + return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1); + } + + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers, int priority) { + return createSchedulingRequest(memory, 1, queueId, userId, numContainers, + priority); + } + + private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, + String queueId, String userId, int numContainers, int priority) { + ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); + scheduler.addApplication(id.getApplicationId(), queueId, userId); + // This conditional is for testAclSubmitApplication where app is rejected + // and no app is added. + if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) { + scheduler.addApplicationAttempt(id, false); + } + List ask = new ArrayList(); + ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, + priority, numContainers, true); + ask.add(request); + scheduler.allocate(id, ask, new ArrayList(), null, null); + return id; + } + + private void createSchedulingRequestExistingApplication(int memory, int priority, + ApplicationAttemptId attId) { + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, 1, true); + createSchedulingRequestExistingApplication(request, attId); + } + + private void createSchedulingRequestExistingApplication(int memory, int vcores, + int priority, ApplicationAttemptId attId) { + ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, + priority, 1, true); + createSchedulingRequestExistingApplication(request, attId); + } + + private void createSchedulingRequestExistingApplication(ResourceRequest request, + ApplicationAttemptId attId) { + List ask = new ArrayList(); + ask.add(request); + scheduler.allocate(attId, ask, new ArrayList(), null, null); + } + // TESTS @Test(timeout=2000) @@ -245,19 +377,19 @@ public void testAggregateCapacityTracking() throws Exception { .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); - assertEquals(1024, scheduler.getClusterResource().getMemory()); + assertEquals(1024, scheduler.getClusterCapacity().getMemory()); // Add another node RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2"); NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); scheduler.handle(nodeEvent2); - assertEquals(1536, scheduler.getClusterResource().getMemory()); + assertEquals(1536, scheduler.getClusterCapacity().getMemory()); // Remove the first node NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1); scheduler.handle(nodeEvent3); - assertEquals(512, scheduler.getClusterResource().getMemory()); + assertEquals(512, scheduler.getClusterCapacity().getMemory()); } @Test @@ -1319,7 +1451,7 @@ public void testPreemptionDecision() throws Exception { assertEquals( 1536, scheduler.resToPreempt(schedD, clock.getTime()).getMemory()); } - + @Test (timeout = 5000) public void testMultipleContainersWaitingForReservation() throws IOException { scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1991,7 +2123,7 @@ public void testBasicDRFAssignment() throws Exception { FSSchedulerApp app2 = scheduler.getSchedulerApp(appAttId2); DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy(); - drfPolicy.initialize(scheduler.getClusterResource()); + drfPolicy.initialize(scheduler.getClusterCapacity()); scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy); scheduler.update(); @@ -2035,7 +2167,7 @@ public void testBasicDRFWithQueues() throws Exception { FSSchedulerApp app3 = scheduler.getSchedulerApp(appAttId3); DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy(); - drfPolicy.initialize(scheduler.getClusterResource()); + drfPolicy.initialize(scheduler.getClusterCapacity()); scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy); scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy); scheduler.update(); @@ -2078,7 +2210,7 @@ public void testDRFHierarchicalQueues() throws Exception { FSSchedulerApp app4 = scheduler.getSchedulerApp(appAttId4); DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy(); - drfPolicy.initialize(scheduler.getClusterResource()); + drfPolicy.initialize(scheduler.getClusterCapacity()); scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy); scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy); scheduler.getQueueManager().getQueue("queue1.subqueue1").setPolicy(drfPolicy); @@ -2334,8 +2466,8 @@ public void testContinuousScheduling() throws Exception { fs.handle(nodeEvent2); // available resource - Assert.assertEquals(fs.getClusterResource().getMemory(), 16 * 1024); - Assert.assertEquals(fs.getClusterResource().getVirtualCores(), 16); + Assert.assertEquals(fs.getClusterCapacity().getMemory(), 16 * 1024); + Assert.assertEquals(fs.getClusterCapacity().getVirtualCores(), 16); // send application request ApplicationAttemptId appAttemptId = @@ -2424,35 +2556,6 @@ public void testDontAllowUndeclaredPools() throws Exception{ assertEquals(2, defaultQueue.getRunnableAppSchedulables().size()); } - @Test - public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() - throws IOException { - // This test verifies if default rule in queue placement policy - // initializes properly when policy is not configured and - // undeclared pools is not allowed. - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); - - // Create an alloc file with no queue placement policy - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.close(); - - scheduler.reinitialize(conf, resourceManager.getRMContext()); - - List rules = scheduler.allocConf.placementPolicy - .getRules(); - - for (QueuePlacementRule rule : rules) { - if (rule instanceof Default) { - Default defaultRule = (Default) rule; - assertNotNull(defaultRule.defaultQueueName); - } - } - } - @SuppressWarnings("resource") @Test public void testBlacklistNodes() throws Exception { @@ -2544,9 +2647,8 @@ public void testGetAppsInQueue() throws Exception { @Test public void testAddAndRemoveAppFromFairScheduler() throws Exception { - AbstractYarnScheduler scheduler = - (AbstractYarnScheduler) resourceManager - .getResourceScheduler(); + FairScheduler scheduler = + (FairScheduler) resourceManager.getResourceScheduler(); TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( scheduler.getSchedulerApplications(), scheduler, "default"); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java deleted file mode 100644 index b3ab299..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; -import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; -import org.apache.hadoop.yarn.util.resource.Resources; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Collection; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -public class TestFairSchedulerPreemption extends FairSchedulerTestBase { - private final static String ALLOC_FILE = new File(TEST_DIR, - TestFairSchedulerPreemption.class.getName() + ".xml").getAbsolutePath(); - - private MockClock clock; - - private static class StubbedFairScheduler extends FairScheduler { - public int lastPreemptMemory = -1; - - @Override - protected void preemptResources( - Collection scheds, Resource toPreempt) { - lastPreemptMemory = toPreempt.getMemory(); - } - - public void resetLastPreemptResources() { - lastPreemptMemory = -1; - } - } - - @Override - protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, StubbedFairScheduler.class, - ResourceScheduler.class); - conf.setBoolean(FairSchedulerConfiguration.PREEMPTION, true); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - return conf; - } - - @Before - public void setup() throws IOException { - conf = createConfiguration(); - clock = new MockClock(); - } - - @After - public void teardown() { - if (resourceManager != null) { - resourceManager.stop(); - resourceManager = null; - } - conf = null; - } - - private void startResourceManager(float utilizationThreshold) { - conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, - utilizationThreshold); - resourceManager = new MockRM(conf); - resourceManager.start(); - - assertTrue( - resourceManager.getResourceScheduler() instanceof StubbedFairScheduler); - scheduler = (FairScheduler)resourceManager.getResourceScheduler(); - - scheduler.setClock(clock); - scheduler.UPDATE_INTERVAL = 60 * 1000; - } - - private void registerNodeAndSubmitApp( - int memory, int vcores, int appContainers, int appMemory) { - RMNode node1 = MockNodes.newNodeInfo( - 1, Resources.createResource(memory, vcores), 1, "node1"); - NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); - scheduler.handle(nodeEvent1); - - assertEquals("Incorrect amount of resources in the cluster", - memory, scheduler.rootMetrics.getAvailableMB()); - assertEquals("Incorrect amount of resources in the cluster", - vcores, scheduler.rootMetrics.getAvailableVirtualCores()); - - createSchedulingRequest(appMemory, "queueA", "user1", appContainers); - scheduler.update(); - // Sufficient node check-ins to fully schedule containers - for (int i = 0; i < 3; i++) { - NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); - scheduler.handle(nodeUpdate1); - } - assertEquals("app1's request is not met", - memory - appContainers * appMemory, - scheduler.rootMetrics.getAvailableMB()); - } - - @Test - public void testPreemptionWithFreeResources() throws Exception { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0mb,0vcores"); - out.println(""); - out.println(""); - out.println("1"); - out.println("1024mb,0vcores"); - out.println(""); - out.println(""); - out.println("1"); - out.println("1024mb,0vcores"); - out.println(""); - out.print("5"); - out.print("10"); - out.println(""); - out.close(); - - startResourceManager(0f); - // Create node with 4GB memory and 4 vcores - registerNodeAndSubmitApp(4 * 1024, 4, 2, 1024); - - // Verify submitting another request doesn't trigger preemption - createSchedulingRequest(1024, "queueB", "user1", 1, 1); - scheduler.update(); - clock.tick(6); - - ((StubbedFairScheduler) scheduler).resetLastPreemptResources(); - scheduler.preemptTasksIfNecessary(); - assertEquals("preemptResources() should have been called", 1024, - ((StubbedFairScheduler) scheduler).lastPreemptMemory); - - resourceManager.stop(); - - startResourceManager(0.8f); - // Create node with 4GB memory and 4 vcores - registerNodeAndSubmitApp(4 * 1024, 4, 3, 1024); - - // Verify submitting another request doesn't trigger preemption - createSchedulingRequest(1024, "queueB", "user1", 1, 1); - scheduler.update(); - clock.tick(6); - - ((StubbedFairScheduler) scheduler).resetLastPreemptResources(); - scheduler.preemptTasksIfNecessary(); - assertEquals("preemptResources() should not have been called", -1, - ((StubbedFairScheduler) scheduler).lastPreemptMemory); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java index e20b0c3..640d771 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -134,22 +134,6 @@ public void testTerminals() throws Exception { } @Test - public void testDefaultRuleWithQueueAttribute() throws Exception { - // This test covers the use case where we would like default rule - // to point to a different queue by default rather than root.default - configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue"); - StringBuffer sb = new StringBuffer(); - sb.append(""); - sb.append(" "); - sb.append(" "); - sb.append(""); - - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.someDefaultQueue", - policy.assignAppToQueue("root.default", "user1")); - } - - @Test public void testNestedUserQueueParsingErrors() { // No nested rule specified in hierarchical user queue StringBuffer sb = new StringBuffer(); @@ -327,25 +311,6 @@ public void testNestedUserQueueSpecificRule() throws Exception { policy.assignAppToQueue("root.parent2", "user2")); } - @Test - public void testNestedUserQueueDefaultRule() throws Exception { - // This test covers the use case where we would like user queues to be - // created under a default parent queue - configuredQueues.get(FSQueueType.PARENT).add("root.parentq"); - StringBuffer sb = new StringBuffer(); - sb.append(""); - sb.append(" "); - sb.append(" "); - sb.append(" "); - sb.append(" "); - sb.append(" "); - sb.append(""); - - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.parentq.user1", - policy.assignAppToQueue("root.default", "user1")); - } - private QueuePlacementPolicy parse(String str) throws Exception { // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index f5bfc37..f6dfc3f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; + import static org.mockito.Mockito.mock; import java.io.IOException; @@ -29,6 +30,8 @@ import java.util.List; import java.util.Map; +import org.junit.Assert; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -57,13 +60,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -75,7 +78,6 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -592,12 +594,9 @@ public void testGetAppsInQueue() throws Exception { public void testAddAndRemoveAppFromFiFoScheduler() throws Exception { Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, - ResourceScheduler.class); + ResourceScheduler.class); MockRM rm = new MockRM(conf); - @SuppressWarnings("unchecked") - AbstractYarnScheduler fs = - (AbstractYarnScheduler) rm - .getResourceScheduler(); + FifoScheduler fs = (FifoScheduler)rm.getResourceScheduler(); TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( fs.getSchedulerApplications(), fs, "queue"); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 93d0a12..2baddf7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; @@ -720,7 +719,6 @@ protected synchronized void serviceStop() throws Exception { if (appHistoryServer != null) { appHistoryServer.stop(); } - AHSWebApp.resetInstance(); super.serviceStop(); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 48011cb..02890a1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -156,12 +156,6 @@ Properties that can be placed in yarn-site.xml * Whether to use preemption. Note that preemption is experimental in the current version. Defaults to false. - * <<>> - - * The utilization threshold after which preemption kicks in. The - utilization is computed as the maximum ratio of usage to capacity among - all resources. Defaults to 0.8f. - * <<>> * Whether to assign shares to individual apps based on their size, rather than @@ -316,8 +310,7 @@ Allocation file format to ‘parent’ or by configuring at least one leaf under that queue which makes it a parent. See example allocation for a sample use case. - * default: the app is placed into the queue specified in the ‘queue’ attribute of the - default rule. If ‘queue’ attribute is not specified, the app is placed into ‘root.default’ queue. + * default: the app is placed into the queue named "default". * reject: the app is rejected. @@ -355,7 +348,7 @@ Allocation file format - + ---