diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index c1bc17d..2821b47 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -50,6 +50,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -123,6 +124,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.sharedcache.SharedCacheClient; import org.apache.hadoop.yarn.state.InvalidStateTransitonException; import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition; @@ -1112,6 +1114,32 @@ protected JobStateInternal checkReadyForCommit() { return getInternalState(); } + private void cleanupSharedCacheResources() { + String[] checksums = conf.getStrings(MRJobConfig.SHARED_CACHE_CHECKSUMS); + if (checksums != null && checksums.length != 0) { + SharedCacheClient scClient = null; + try { + scClient = new SharedCacheClient(); + scClient.init(conf); + scClient.start(); + for (String checksum : checksums) { + scClient.release(this.applicationAttemptId.getApplicationId(), + checksum); + metrics.releasedSharedCacheResources(); + } + } finally { + if (scClient != null) { + try { + scClient.close(); + } catch (IOException e) { + LOG.warn("IOException thrown during shared cache client shutdown.", + e); + } + } + } + } + } + JobStateInternal finished(JobStateInternal finalState) { if (getInternalState() == JobStateInternal.RUNNING) { metrics.endRunningJob(this); @@ -1130,6 +1158,8 @@ JobStateInternal finished(JobStateInternal finalState) { break; case SUCCEEDED: metrics.completedJob(this); + // Clean up shared cache resources only for SUCCEEDED scenario + cleanupSharedCacheResources(); break; default: throw new IllegalArgumentException("Illegal job state: " + finalState); @@ -1385,6 +1415,28 @@ public static String escapeString(String data) { new char[] {'"', '=', '.'}); } + // The goal is to make sure only the NM that hosts MRAppMaster will + // upload resources to shared cache. + // Clean up the shared cache policies for all resources so that + // later when TaskAttemptImpl creates ContainerLaunchContext, + // LocalResource.setShouldBeUploadedToSharedCache will be set up to false. + // In that way, the NMs that host the task containers won't try to + // upload the resources to shared cache. + private static void cleanupSharedCacheUploadPolicies(Configuration conf) { + boolean[] archivesPolicies = Job.getArchivesSharedCacheUploadPolicies(conf); + if (archivesPolicies != null) { + boolean[] falseArchivesPolicies = new boolean[archivesPolicies.length]; + Job.setArchivesSharedCacheUploadPolicies(conf, + falseArchivesPolicies); + } + boolean[] filesPolicies = Job.getFilesSharedCacheUploadPolicies(conf); + if (filesPolicies != null) { + boolean[] falseFilesPolicies = new boolean[filesPolicies.length]; + Job.setFilesSharedCacheUploadPolicies(conf, + falseFilesPolicies); + } + } + public static class InitTransition implements MultipleArcTransition { @@ -1463,6 +1515,8 @@ public JobStateInternal transition(JobImpl job, JobEvent event) { job.allowedReduceFailuresPercent = job.conf.getInt(MRJobConfig.REDUCE_FAILURES_MAXPERCENT, 0); + cleanupSharedCacheUploadPolicies(job.conf); + // create the Tasks but don't start them yet createMapTasks(job, inputLength, taskSplitMetaInfo); createReduceTasks(job); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index f33c58e..5c71911 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -595,6 +595,10 @@ private int getCpuRequired(Configuration conf, TaskType taskType) { /** * Create a {@link LocalResource} record with all the given parameters. + * The NM that hosts AM container will upload resources to shared cache. + * Thus there is no need to ask task container's NM to upload the + * resources to shared cache. Set the shared cache upload policy to + * false. */ private static LocalResource createLocalResource(FileSystem fc, Path file, LocalResourceType type, LocalResourceVisibility visibility) @@ -606,7 +610,7 @@ private static LocalResource createLocalResource(FileSystem fc, Path file, long resourceModificationTime = fstat.getModificationTime(); return LocalResource.newInstance(resourceURL, type, visibility, - resourceSize, resourceModificationTime); + resourceSize, resourceModificationTime, false); } /** @@ -659,8 +663,13 @@ private static ContainerLaunchContext createCommonContainerLaunchContext( if (jobJar != null) { Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS .getUri(), remoteFS.getWorkingDirectory()); + boolean jobJarVisibility = + conf.getBoolean(MRJobConfig.JOBJAR_VISIBILITY, + MRJobConfig.JOBJAR_VISIBILITY_DEFAULT); LocalResource rc = createLocalResource(remoteFS, remoteJobJar, - LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION); + LocalResourceType.PATTERN, + jobJarVisibility ? LocalResourceVisibility.PUBLIC + : LocalResourceVisibility.APPLICATION); String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); rc.setPattern(pattern); @@ -683,6 +692,9 @@ private static ContainerLaunchContext createCommonContainerLaunchContext( new Path(path, oldJobId.toString()); Path remoteJobConfPath = new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE); + // There is no point to ask task container's NM to upload the resource + // to shared cache. createLocalResource will set the shared cache upload + // policy to false localResources.put( MRJobConfig.JOB_CONF_FILE, createLocalResource(remoteFS, remoteJobConfPath, diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/metrics/MRAppMetrics.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/metrics/MRAppMetrics.java index f55457c..3e79d32 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/metrics/MRAppMetrics.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/metrics/MRAppMetrics.java @@ -49,6 +49,8 @@ @Metric MutableCounterInt reducesKilled; @Metric MutableGaugeInt reducesRunning; @Metric MutableGaugeInt reducesWaiting; + @Metric + MutableCounterInt sharedCacheResourcesReleased; public static MRAppMetrics create() { return create(DefaultMetricsSystem.instance()); @@ -180,4 +182,12 @@ public void endWaitingTask(Task task) { break; } } + + public void releasedSharedCacheResources() { + sharedCacheResourcesReleased.incr(); + } + + public int getReleasedSharedCacheResources() { + return sharedCacheResourcesReleased.value(); + } } \ No newline at end of file diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 3bd8414..9820327 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.InvalidJobConfException; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -457,7 +458,8 @@ public static void setupDistributedCache( DistributedCache.getCacheArchives(conf), DistributedCache.getArchiveTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), - DistributedCache.getArchiveVisibilities(conf)); + DistributedCache.getArchiveVisibilities(conf), + Job.getArchivesSharedCacheUploadPolicies(conf)); // Cache files parseDistributedCacheArtifacts(conf, @@ -466,7 +468,8 @@ public static void setupDistributedCache( DistributedCache.getCacheFiles(conf), DistributedCache.getFileTimestamps(conf), getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES), - DistributedCache.getFileVisibilities(conf)); + DistributedCache.getFileVisibilities(conf), + Job.getFilesSharedCacheUploadPolicies(conf)); } private static String getResourceDescription(LocalResourceType type) { @@ -483,18 +486,23 @@ private static void parseDistributedCacheArtifacts( Configuration conf, Map localResources, LocalResourceType type, - URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[]) + URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[], + boolean sharedCacheUploadPolicies[]) throws IOException { if (uris != null) { // Sanity check - if ((uris.length != timestamps.length) || (uris.length != sizes.length) || - (uris.length != visibilities.length)) { - throw new IllegalArgumentException("Invalid specification for " + - "distributed-cache artifacts of type " + type + " :" + - " #uris=" + uris.length + - " #timestamps=" + timestamps.length + - " #visibilities=" + visibilities.length + // Check if sharedCacheUploadPolicies is null for backward compatibility, + // given older version of MR client won't set the value. + if ((uris.length != timestamps.length) || (uris.length != sizes.length) + || (uris.length != visibilities.length) + || (sharedCacheUploadPolicies != null + && uris.length != sharedCacheUploadPolicies.length)) { + throw new IllegalArgumentException("Invalid specification for " + + "distributed-cache artifacts of type " + type + " :" + " #uris=" + + uris.length + " #timestamps=" + timestamps.length + + " #visibilities=" + visibilities.length + + " #sharedcacheuploadpolicies=" + sharedCacheUploadPolicies.length ); } @@ -520,9 +528,10 @@ private static void parseDistributedCacheArtifacts( " conflicts with " + getResourceDescription(type) + u); } localResources.put(linkName, LocalResource.newInstance(ConverterUtils - .getYarnUrlFromURI(p.toUri()), type, visibilities[i] - ? LocalResourceVisibility.PUBLIC : LocalResourceVisibility.PRIVATE, - sizes[i], timestamps[i])); + .getYarnUrlFromURI(p.toUri()), type, + visibilities[i] ? LocalResourceVisibility.PUBLIC + : LocalResourceVisibility.PRIVATE, sizes[i], timestamps[i], + sharedCacheUploadPolicies[i])); } } } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java index ec80e65..453e1e5 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java @@ -167,6 +167,7 @@ public FSDataInputStream answer(InvocationOnMock args) throws Throwable { conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "101"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "201"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "false"); + conf.set(MRJobConfig.CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES, "true"); conf.set(MRConfig.LOCAL_DIR, localDir.getAbsolutePath()); LocalDistributedCacheManager manager = new LocalDistributedCacheManager(); try { @@ -275,6 +276,7 @@ public FSDataInputStream answer(InvocationOnMock args) throws Throwable { conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "101,101"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "201,201"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "false,false"); + conf.set(MRJobConfig.CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES, "true,true"); conf.set(MRConfig.LOCAL_DIR, localDir.getAbsolutePath()); LocalDistributedCacheManager manager = new LocalDistributedCacheManager(); try { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 02a59e7..db4b61d 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -380,10 +380,12 @@ public void testSetupDistributedCacheConflicts() throws Exception { conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true"); + conf.set(MRJobConfig.CACHE_ARCHIVES_SHARED_CACHE_UPLOAD_POLICIES, "true"); DistributedCache.addCacheFile(file, conf); conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true"); + conf.set(MRJobConfig.CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES, "true"); Map localResources = new HashMap(); MRApps.setupDistributedCache(conf, localResources); @@ -412,6 +414,7 @@ public void testSetupDistributedCacheConflictsFiles() throws Exception { conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true"); + conf.set(MRJobConfig.CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES, "true,true"); Map localResources = new HashMap(); MRApps.setupDistributedCache(conf, localResources); @@ -439,10 +442,12 @@ public void testSetupDistributedCache() throws Exception { conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10"); conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true"); + conf.set(MRJobConfig.CACHE_ARCHIVES_SHARED_CACHE_UPLOAD_POLICIES, "true"); DistributedCache.addCacheFile(file, conf); conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11"); conf.set(MRJobConfig.CACHE_FILES_SIZES, "11"); conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true"); + conf.set(MRJobConfig.CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES, "true"); Map localResources = new HashMap(); MRApps.setupDistributedCache(conf, localResources); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/FileUploader.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/FileUploader.java new file mode 100644 index 0000000..52d5a51 --- /dev/null +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/FileUploader.java @@ -0,0 +1,521 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.sharedcache.SharedCacheClient; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +class FileUploader { + protected static final Log LOG = LogFactory.getLog(FileUploader.class); + private FileSystem jtFs; + private SharedCacheConfig scConfig = new SharedCacheConfig(); + private SharedCacheClient scClient = null; + private ApplicationId appId = null; + + FileUploader(FileSystem submitFs) { + this.jtFs = submitFs; + } + + private void initSharedCache(JobID jobid, Configuration conf) { + this.scConfig.init(conf); + if (this.scConfig.isSharedCacheEnabled()) { + this.scClient = createSharedCacheClient(conf); + appId = jobid.getAppId(); + } + } + + private void stopSharedCache() { + if (scClient != null) { + scClient.stop(); + } + } + + // make it protected so that test code can overload the method + // to provide test or mock SharedCacheClient + protected SharedCacheClient createSharedCacheClient(Configuration conf) { + SharedCacheClient scClient = new SharedCacheClient(); + scClient.init(conf); + scClient.start(); + return scClient; + } + + private boolean isSharedCacheFilesEnabled() { + return (scConfig.isSharedCacheFilesEnabled() && scClient.isScmAvailable()); + } + + private boolean isSharedCacheLibjarsEnabled() { + return (scConfig.isSharedCacheLibjarsEnabled() && scClient.isScmAvailable()); + } + + private boolean isSharedCacheArchivesEnabled() { + return (scConfig.isSharedCacheArchivesEnabled() && scClient.isScmAvailable()); + } + + private boolean isSharedCacheJobjarEnabled() { + return (scConfig.isSharedCacheJobjarEnabled() && scClient.isScmAvailable()); + } + + private Path useSharedCache(Path sourceFile, Configuration conf, + boolean createSymLinkIfNecessary) throws IOException { + // If for whatever reason, we can't even calculate checksum for + // local resource, something is really wrong with the file system; + // even non-SCM approach won't work. Let us just throw the exception. + String checksum = scClient.getFileChecksum(sourceFile); + Path path = scClient.use(this.appId, checksum); + if (path != null) { + addChecksum(checksum, conf); + // Symlink is used to address the following scenario. + // There are two jars A.jar and B.jar. They are available in SCM, + // but are stored under the same name, e.g., checksum1/job.jar", + // checksum2/job.jar. Without explicit symlink, "job.jar" will be used + // as the symlink name and thus cause one symlink overrides the other. + // With symlink, the resource URL will be checksum1/job.jar#A.jar and + // checksum2/job.jar#B.jar. YARN will use the explicit symlinks during + // localization, e.g., + // $(container_id_dir)/A.jar -> yarn/local/filecache/10/job.jar + // $(container_id_dir)/B.jar -> yarn/local/filecache/11/job.jar + String symlink = (sourceFile.toUri().getFragment() == null) ? + sourceFile.getName() : sourceFile.toUri().getFragment(); + if (createSymLinkIfNecessary && !path.getName().equals(symlink)) { + try { + path = new Path(getPathURI(path, symlink)); + } catch(URISyntaxException ue) { + //should not throw a uri exception + throw new IOException("Failed to create uri for " + path, ue); + } + } + } + return path; + } + + /** + * Add a checksum to the conf. MR master will release the resource + * later after the job succeeds. + * @param checksum The checksum + * @param conf Configuration to add the cache to + */ + private static void addChecksum(String checksum, Configuration conf) { + String checksums = conf.get(MRJobConfig.SHARED_CACHE_CHECKSUMS); + conf.set(MRJobConfig.SHARED_CACHE_CHECKSUMS, checksums == null ? + checksum : checksums + "," + checksum); + } + + // Merge the files passed from CLI with those via DistributedCache APIs + // addCacheFileShared, addFileToClassPathShared, addCacheArchiveToShared + private String getFiles(Configuration conf, String cliConfigName, + String sharedCacheConfigName) { + String files = conf.get(cliConfigName); + String dcFiles = conf.get(sharedCacheConfigName); + if (files != null && dcFiles != null) { + files = files + "," + dcFiles; + } else if (files == null) { + files = dcFiles; + } + return files; + } + + /** + * Upload and configure files, libjars, jobjars, and archives pertaining to + * the passed job. This client will use the shared cache for libjars and + * jobjars if it is enabled. + * When shared cache is enabled, it will try to use the shared cache and fall back + * to the default behavior when shared cache isn't available. + * 1.For the resources that have been successfully shared, + * we will continue to use them in a shared fashion. + * 2.For the resources that weren't in the cache and need to be uploaded by NM, + * we won't ask NM to upload them. * + * @param job the job containing the files to be uploaded + * @param submitJobDir the submission directory of the job + * @throws IOException + */ + public void uploadFiles(Job job, Path submitJobDir) + throws IOException { + Configuration conf = job.getConfiguration(); + short replication = + (short) conf.getInt(Job.SUBMIT_REPLICATION, + MRJobConfig.SUBMIT_FILE_REPLICATION_DEFAULT); + + if (!(conf.getBoolean(Job.USED_GENERIC_PARSER, false))) { + LOG.warn("Hadoop command-line option parsing not performed. " + + "Implement the Tool interface and execute your application " + + "with ToolRunner to remedy this."); + } + + initSharedCache(job.getJobID(), conf); + + // get all the command line arguments passed in by the user conf + // as well as DistributedCache APIs + String files = getFiles( + conf, "tmpfiles", MRJobConfig.FILES_FOR_SHARED_CACHE); + String libjars = getFiles( + conf, "tmpjars", MRJobConfig.FILES_FOR_CLASSPATH_AND_SHARED_CACHE); + String archives = getFiles(conf, "tmparchives", + MRJobConfig.ARCHIVES_FOR_SHARED_CACHE); + String jobJar = job.getJar(); + + // + // Figure out what fs the JobTracker is using. Copy the + // job to it, under a temporary name. This allows DFS to work, + // and under the local fs also provides UNIX-like object loading + // semantics. (that is, if the job file is deleted right after + // submission, we can still run the submission to completion) + // + + // Create a number of filenames in the JobTracker's fs namespace + LOG.debug("default FileSystem: " + jtFs.getUri()); + if (jtFs.exists(submitJobDir)) { + throw new IOException("Not submitting job. Job directory " + submitJobDir + +" already exists!! This is unexpected.Please check what's there in" + + " that directory"); + } + + submitJobDir = jtFs.makeQualified(submitJobDir); + submitJobDir = new Path(submitJobDir.toUri().getPath()); + FsPermission mapredSysPerms = new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION); + FileSystem.mkdirs(jtFs, submitJobDir, mapredSysPerms); + Path filesDir = JobSubmissionFiles.getJobDistCacheFiles(submitJobDir); + Path archivesDir = JobSubmissionFiles.getJobDistCacheArchives(submitJobDir); + // add all the command line files/ jars and archive + // first copy them to jobtrackers filesystem + + // We need to account for the distributed cache files applications specify via + // Job or Distributed Cache APIs such as addCacheFile. These files are already in HDFS. + // We don't ask YARN to upload those files to shared cache. + int numOfFilesSCUploadPolicies = + job.getCacheFiles() != null ? job.getCacheFiles().length : 0; + int indexOfFilesSCUploadPolicies = numOfFilesSCUploadPolicies; + int numOfArchivesSCUploadPolicies = + job.getCacheArchives() != null ? job.getCacheArchives().length : 0; + int indexOfArchivesSCUploadPolicies = numOfArchivesSCUploadPolicies; + + String[] fileArr = null; + if (files != null) { + FileSystem.mkdirs(jtFs, filesDir, mapredSysPerms); + fileArr = files.split(","); + numOfFilesSCUploadPolicies += fileArr.length; + } + + String[] libjarsArr = null; + if (libjars != null) { + libjarsArr = libjars.split(","); + numOfFilesSCUploadPolicies += libjarsArr.length; + } + + String[] archivesArr = null; + if (archives != null) { + FileSystem.mkdirs(jtFs, archivesDir, mapredSysPerms); + archivesArr = archives.split(","); + numOfArchivesSCUploadPolicies += archivesArr.length; + } + + boolean[] filesSCUploadPolicies = new boolean[numOfFilesSCUploadPolicies]; + boolean[] archivesSCUploadPolicies = new boolean[numOfArchivesSCUploadPolicies]; + + if (files != null) { + for (String tmpFile: fileArr) { + URI tmpURI = null; + try { + tmpURI = new URI(tmpFile); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + Path tmp = new Path(tmpURI); + Path newPath = null; + if (isSharedCacheFilesEnabled()) { + newPath = useSharedCache(tmp, conf, true); + } + + // need to inform NM to upload the file to shared cache. + if (newPath == null && isSharedCacheFilesEnabled()) { + filesSCUploadPolicies[indexOfFilesSCUploadPolicies] = true; + } + indexOfFilesSCUploadPolicies++; + + if (newPath == null) { + newPath = copyRemoteFiles(filesDir, tmp, conf, replication); + } + try { + URI pathURI = getPathURI(newPath, tmpURI.getFragment()); + job.addCacheFile(pathURI); + } catch(URISyntaxException ue) { + //should not throw a uri exception + throw new IOException("Failed to create uri for " + tmpFile, ue); + } + } + } + + if (libjars != null) { + for (String tmpjars: libjarsArr) { + Path tmp = new Path(tmpjars); + Path newPath = null; + if (isSharedCacheLibjarsEnabled()) { + newPath = useSharedCache(tmp, conf, true); + } + + // need to inform NM to upload the file to shared cache. + if (newPath == null && isSharedCacheLibjarsEnabled()) { + filesSCUploadPolicies[indexOfFilesSCUploadPolicies] = true; + } + indexOfFilesSCUploadPolicies++; + + if(newPath == null) { + Path libjarsDir = + JobSubmissionFiles.getJobDistCacheLibjars(submitJobDir); + FileSystem.mkdirs(jtFs, libjarsDir, mapredSysPerms); + newPath = copyRemoteFiles(libjarsDir, tmp, conf, replication); + } + + // In order to support symlink for libjars, we pass the full URL + // to DistributedCache.addFileToClassPath. + // DistributedCache.addFileToClassPath will set the classpath using + // the Path part without fragment, e.g. file.toUri().getPath(). + // DistributedCache.addFileToClassPath will call addCacheFile with + // the full URL with fragment. + job.addFileToClassPath(newPath); + } + } + + if (archives != null) { + for (String tmpArchives: archivesArr) { + URI tmpURI; + try { + tmpURI = new URI(tmpArchives); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + Path tmp = new Path(tmpURI); + Path newPath = null; + if (isSharedCacheArchivesEnabled()) { + newPath = useSharedCache(tmp, conf, true); + } + + // need to inform NM to upload the file to shared cache. + if (newPath == null && isSharedCacheArchivesEnabled()) { + archivesSCUploadPolicies[indexOfArchivesSCUploadPolicies] = true; + } + indexOfArchivesSCUploadPolicies++; + + if (newPath == null) { + newPath = copyRemoteFiles(archivesDir, tmp, conf, replication); + } + try { + URI pathURI = getPathURI(newPath, tmpURI.getFragment()); + job.addCacheArchive(pathURI); + } catch(URISyntaxException ue) { + //should not throw an uri excpetion + throw new IOException("Failed to create uri for " + tmpArchives, ue); + } + } + } + + if (jobJar != null) { // copy jar to JobTracker's fs + // use jar name if job is not named. + if ("".equals(job.getJobName())){ + job.setJobName(new Path(jobJar).getName()); + } + Path jobJarPath = new Path(jobJar); + URI jobJarURI = jobJarPath.toUri(); + // If the job jar is already in fs, we don't need to copy it from local fs + if (jobJarURI.getScheme() == null || jobJarURI.getAuthority() == null + || !(jobJarURI.getScheme().equals(jtFs.getUri().getScheme()) + && jobJarURI.getAuthority().equals( + jtFs.getUri().getAuthority()))) { + Path newJarPath = null; + if (isSharedCacheJobjarEnabled()) { + // jobJarPath has the format of "/..." without "file:///" scheme when + // users submit local job jar from command line. + // We fix up the scheme so that jobJarPath.getFileSystem(conf) can + // return the correct file system. + // Otherwise, scClient won't be able to locate the file when + // it tries to compute the checksum. + // This won't break anything given copyJar function below has + // the assumption the source is a local file. + // Currently jar file has to be either local or on the same cluster + // as MR cluster. + if (jobJarPath.toUri().getScheme() == null && + jobJarPath.toUri().getAuthority() == null) { + jobJarPath = FileSystem.getLocal(conf).makeQualified(jobJarPath); + } + newJarPath = useSharedCache(jobJarPath, conf, false); + } + + // Inform NM to upload job jar to shared cache. + if (newJarPath == null && isSharedCacheJobjarEnabled()) { + conf.setBoolean(MRJobConfig.JOBJAR_SHARED_CACHE_UPLOAD_POLICY, true); + } else { + conf.setBoolean(MRJobConfig.JOBJAR_SHARED_CACHE_UPLOAD_POLICY, false); + } + + if(newJarPath == null) { + copyJar(jobJarPath, JobSubmissionFiles.getJobJar(submitJobDir), + replication); + newJarPath = JobSubmissionFiles.getJobJar(submitJobDir); + } else { + + // The application wants to use a job jar on the local file system + // while the job jar is already in shared cache. + // Set the visibility flag properly so that job jar will be localized + // to public cache by yarn localizer + // We don't set it for other scenarios given default value of + // JOBJAR_VISIBILITY is false + + conf.setBoolean(MRJobConfig.JOBJAR_VISIBILITY, true); + } + // set the job jar size if it is being copied + job.getConfiguration().setLong(MRJobConfig.JAR + ".size", + getJobJarSize(jobJarPath)); + job.setJar(newJarPath.toString()); + } + } else { + LOG.warn("No job jar file set. User classes may not be found. "+ + "See Job or Job#setJar(String)."); + } + + // if scm fails in the middle, we will set shared cache upload policies for all resources + // to be false. The resources that are shared successfully via SharedCacheClient.use will + // continued to be shared. + if (scClient != null && !scClient.isScmAvailable()) { + conf.setBoolean(MRJobConfig.JOBJAR_SHARED_CACHE_UPLOAD_POLICY, false); + Job.setFilesSharedCacheUploadPolicies(conf, + new boolean[filesSCUploadPolicies.length]); + Job.setArchivesSharedCacheUploadPolicies(conf, + new boolean[archivesSCUploadPolicies.length]); + } else + { + Job.setFilesSharedCacheUploadPolicies(conf, + filesSCUploadPolicies); + Job.setArchivesSharedCacheUploadPolicies(conf, + archivesSCUploadPolicies); + } + + // set the timestamps of the archives and files + // set the public/private visibility of the archives and files + ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(conf); + // get DelegationToken for each cached file + ClientDistributedCacheManager.getDelegationTokens(conf, job + .getCredentials()); + + stopSharedCache(); + + } + + // copies a file to the jobtracker filesystem and returns the path where it + // was copied to + private Path copyRemoteFiles(Path parentDir, Path originalPath, + Configuration conf, short replication) throws IOException { + // check if we do not need to copy the files + // is jt using the same file system. + // just checking for uri strings... doing no dns lookups + // to see if the filesystems are the same. This is not optimal. + // but avoids name resolution. + + FileSystem remoteFs = null; + remoteFs = originalPath.getFileSystem(conf); + if (compareFs(remoteFs, jtFs)) { + return originalPath; + } + // this might have name collisions. copy will throw an exception + // parse the original path to create new path + Path newPath = new Path(parentDir, originalPath.getName()); + FileUtil.copy(remoteFs, originalPath, jtFs, newPath, false, conf); + jtFs.setReplication(newPath, replication); + return newPath; + } + + private void copyJar(Path originalJarPath, Path submitJarFile, + short replication) throws IOException { + jtFs.copyFromLocalFile(originalJarPath, submitJarFile); + jtFs.setReplication(submitJarFile, replication); + jtFs.setPermission(submitJarFile, new FsPermission( + JobSubmissionFiles.JOB_FILE_PERMISSION)); + } + + private long getJobJarSize(Path jobJarPath) throws IOException { + FileSystem fs = FileSystem.getLocal(jtFs.getConf()); + FileStatus status = fs.getFileStatus(jobJarPath); + return status.getLen(); + } + + private URI getPathURI(Path destPath, String fragment) + throws URISyntaxException { + URI pathURI = destPath.toUri(); + if (pathURI.getFragment() == null) { + if (fragment == null) { + pathURI = new URI(pathURI.toString() + "#" + destPath.getName()); + } else { + pathURI = new URI(pathURI.toString() + "#" + fragment); + } + } + return pathURI; + } + + /* + * see if two file systems are the same or not. + */ + private boolean compareFs(FileSystem srcFs, FileSystem destFs) { + URI srcUri = srcFs.getUri(); + URI dstUri = destFs.getUri(); + if (srcUri.getScheme() == null) { + return false; + } + if (!srcUri.getScheme().equals(dstUri.getScheme())) { + return false; + } + String srcHost = srcUri.getHost(); + String dstHost = dstUri.getHost(); + if ((srcHost != null) && (dstHost != null)) { + try { + srcHost = InetAddress.getByName(srcHost).getCanonicalHostName(); + dstHost = InetAddress.getByName(dstHost).getCanonicalHostName(); + } catch (UnknownHostException ue) { + return false; + } + if (!srcHost.equals(dstHost)) { + return false; + } + } else if (srcHost == null && dstHost != null) { + return false; + } else if (srcHost != null && dstHost == null) { + return false; + } + // check for ports + if (srcUri.getPort() != dstUri.getPort()) { + return false; + } + return true; + } +} diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java index 3f8d139..4340089 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java @@ -1253,6 +1253,181 @@ private void setUseNewAPI() throws IOException { } } + /** + * Add a file to job config for shared cache processing. If shared cache is + * enabled, it will return true, otherwise, return false. We don't check with + * SCM here given application might not be able to provide the job id; + * ClientSCMProtocol.use requires the application id. Job Submitter will read + * the files from job config and take care of things. Intended to be used by + * user code. + * + * @param resource The resource that Job Submitter will process later using + * shared cache. + * @param conf Configuration to add the resource to + * @return whether the resource has been added to the configuration + */ + public static boolean addCacheFileShared(URI resource, Configuration conf) { + SharedCacheConfig scConfig = new SharedCacheConfig(); + scConfig.init(conf); + if (scConfig.isSharedCacheFilesEnabled()) { + String files = conf.get(MRJobConfig.FILES_FOR_SHARED_CACHE); + conf.set( + MRJobConfig.FILES_FOR_SHARED_CACHE, + files == null ? resource.toString() : files + "," + + resource.toString()); + return true; + } else { + return false; + } + } + + /** + * Add a file to job config for shared cache processing. If shared cache is + * enabled, it will return true, otherwise, return false. We don't check with + * SCM here given application might not be able to provide the job id; + * ClientSCMProtocol.use requires the application id. Job Submitter will read + * the files from job config and take care of things. Job Submitter will also + * add the file to classpath. Intended to be used by user code. + * + * @param resource The resource that Job Submitter will process later using + * shared cache. + * @param conf Configuration to add the resource to + * @return whether the resource has been added to the configuration + */ + public static boolean addFileToClassPathShared(URI resource, + Configuration conf) { + SharedCacheConfig scConfig = new SharedCacheConfig(); + scConfig.init(conf); + if (scConfig.isSharedCacheLibjarsEnabled()) { + String files = conf.get(MRJobConfig.FILES_FOR_CLASSPATH_AND_SHARED_CACHE); + conf.set( + MRJobConfig.FILES_FOR_CLASSPATH_AND_SHARED_CACHE, + files == null ? resource.toString() : files + "," + + resource.toString()); + return true; + } else { + return false; + } + } + + /** + * Add an archive to job config for shared cache processing. If shared cache + * is enabled, it will return true, otherwise, return false. We don't check + * with SCM here given application might not be able to provide the job id; + * ClientSCMProtocol.use requires the application id. Job Submitter will read + * the files from job config and take care of things. Intended to be used by + * user code. + * + * @param resource The resource that Job Submitter will process later using + * shared cache. + * @param conf Configuration to add the resource to + * @return whether the resource has been added to the configuration + */ + public static boolean addCacheArchiveToShared(URI resource, Configuration conf) { + SharedCacheConfig scConfig = new SharedCacheConfig(); + scConfig.init(conf); + if (scConfig.isSharedCacheArchivesEnabled()) { + String files = conf.get(MRJobConfig.ARCHIVES_FOR_SHARED_CACHE); + conf.set( + MRJobConfig.ARCHIVES_FOR_SHARED_CACHE, + files == null ? resource.toString() : files + "," + + resource.toString()); + return true; + } else { + return false; + } + } + + /** + * It isn't supported. + * + * @param resource The resource that Job Submitter will process later using + * shared cache. + * @param conf Configuration to add the resource to + * @return whether the resource has been added to the configuration + */ + public static boolean addArchiveToClassPathShared(URI resource, + Configuration conf) { + return false; + } + + /** + * This is to set the shared cache upload policies for files + * + * @param conf Configuration which stores the shared cache upload policies + * @param booleans comma separated list of booleans (true - upload) The order + * should be the same as the order in which the files are added. + */ + public static void setFilesSharedCacheUploadPolicies(Configuration conf, + boolean[] booleans) { + String policies = booleansToString(booleans); + if (policies != null) { + conf.set(MRJobConfig.CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES, policies); + } + } + + /** + * This is to set the shared cache upload policies for archives + * + * @param conf Configuration which stores the shared cache upload policies + * @param booleans comma separated list of booleans (true - upload) The order + * should be the same as the order in which the archives are added. + */ + public static void setArchivesSharedCacheUploadPolicies(Configuration conf, + boolean[] booleans) { + String policies = booleansToString(booleans); + if (policies != null) { + conf.set(MRJobConfig.CACHE_ARCHIVES_SHARED_CACHE_UPLOAD_POLICIES, + policies); + } + } + + /** + * Get the booleans on whether the files should be uploaded to shared cache + * + * @param conf The configuration which stored the shared cache upload policies + * @return a string array of booleans + */ + public static boolean[] getFilesSharedCacheUploadPolicies(Configuration conf) { + return parseBooleans(conf + .getStrings(MRJobConfig.CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES)); + } + + /** + * Get the booleans on whether the archives should be uploaded. + * + * @param conf The configuration which stored the shared cache upload policies + * @return a string array of booleans + */ + public static boolean[] getArchivesSharedCacheUploadPolicies( + Configuration conf) { + return parseBooleans(conf + .getStrings(MRJobConfig.CACHE_ARCHIVES_SHARED_CACHE_UPLOAD_POLICIES)); + } + + static private String booleansToString(boolean[] policies) { + if (policies == null || policies.length == 0) { + return null; + } + StringBuilder ret = new StringBuilder(String.valueOf(policies[0])); + for (int i = 1; i < policies.length; i++) { + ret.append(","); + ret.append(String.valueOf(policies[i])); + } + return ret.toString(); + } + + private static boolean[] parseBooleans(String[] strs) { + if (null == strs) { + return null; + } + boolean[] result = new boolean[strs.length]; + for (int i = 0; i < strs.length; ++i) { + result[i] = Boolean.parseBoolean(strs[i]); + } + return result; + } + private synchronized void connect() throws IOException, InterruptedException, ClassNotFoundException { if (cluster == null) { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java index 57494e5..0998a6c 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Text; +import org.apache.hadoop.yarn.api.records.ApplicationId; /** * JobID represents the immutable and unique identifier for @@ -157,4 +158,13 @@ public static JobID forName(String str) throws IllegalArgumentException { + " is not properly formed"); } + public ApplicationId getAppId() { + return ApplicationId.newInstance( + toClusterTimeStamp(this.getJtIdentifier()), this.getId()); + } + + private static long toClusterTimeStamp(String identifier) { + return Long.parseLong(identifier); + } + } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index 0734e7f..6c7f40a 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -22,7 +22,6 @@ import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; -import java.net.UnknownHostException; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; @@ -41,7 +40,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.Text; @@ -49,7 +47,6 @@ import org.apache.hadoop.mapred.QueueACL; import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; -import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.protocol.ClientProtocol; import org.apache.hadoop.mapreduce.security.TokenCache; @@ -82,231 +79,8 @@ this.submitClient = submitClient; this.jtFs = submitFs; } - /* - * see if two file systems are the same or not. - */ - private boolean compareFs(FileSystem srcFs, FileSystem destFs) { - URI srcUri = srcFs.getUri(); - URI dstUri = destFs.getUri(); - if (srcUri.getScheme() == null) { - return false; - } - if (!srcUri.getScheme().equals(dstUri.getScheme())) { - return false; - } - String srcHost = srcUri.getHost(); - String dstHost = dstUri.getHost(); - if ((srcHost != null) && (dstHost != null)) { - try { - srcHost = InetAddress.getByName(srcHost).getCanonicalHostName(); - dstHost = InetAddress.getByName(dstHost).getCanonicalHostName(); - } catch(UnknownHostException ue) { - return false; - } - if (!srcHost.equals(dstHost)) { - return false; - } - } else if (srcHost == null && dstHost != null) { - return false; - } else if (srcHost != null && dstHost == null) { - return false; - } - //check for ports - if (srcUri.getPort() != dstUri.getPort()) { - return false; - } - return true; - } - - // copies a file to the jobtracker filesystem and returns the path where it - // was copied to - private Path copyRemoteFiles(Path parentDir, - Path originalPath, Configuration conf, short replication) - throws IOException { - //check if we do not need to copy the files - // is jt using the same file system. - // just checking for uri strings... doing no dns lookups - // to see if the filesystems are the same. This is not optimal. - // but avoids name resolution. - - FileSystem remoteFs = null; - remoteFs = originalPath.getFileSystem(conf); - if (compareFs(remoteFs, jtFs)) { - return originalPath; - } - // this might have name collisions. copy will throw an exception - //parse the original path to create new path - Path newPath = new Path(parentDir, originalPath.getName()); - FileUtil.copy(remoteFs, originalPath, jtFs, newPath, false, conf); - jtFs.setReplication(newPath, replication); - return newPath; - } - - // configures -files, -libjars and -archives. - private void copyAndConfigureFiles(Job job, Path submitJobDir, - short replication) throws IOException { - Configuration conf = job.getConfiguration(); - if (!(conf.getBoolean(Job.USED_GENERIC_PARSER, false))) { - LOG.warn("Hadoop command-line option parsing not performed. " + - "Implement the Tool interface and execute your application " + - "with ToolRunner to remedy this."); - } - - // get all the command line arguments passed in by the user conf - String files = conf.get("tmpfiles"); - String libjars = conf.get("tmpjars"); - String archives = conf.get("tmparchives"); - String jobJar = job.getJar(); - - // - // Figure out what fs the JobTracker is using. Copy the - // job to it, under a temporary name. This allows DFS to work, - // and under the local fs also provides UNIX-like object loading - // semantics. (that is, if the job file is deleted right after - // submission, we can still run the submission to completion) - // - - // Create a number of filenames in the JobTracker's fs namespace - LOG.debug("default FileSystem: " + jtFs.getUri()); - if (jtFs.exists(submitJobDir)) { - throw new IOException("Not submitting job. Job directory " + submitJobDir - +" already exists!! This is unexpected.Please check what's there in" + - " that directory"); - } - submitJobDir = jtFs.makeQualified(submitJobDir); - submitJobDir = new Path(submitJobDir.toUri().getPath()); - FsPermission mapredSysPerms = new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION); - FileSystem.mkdirs(jtFs, submitJobDir, mapredSysPerms); - Path filesDir = JobSubmissionFiles.getJobDistCacheFiles(submitJobDir); - Path archivesDir = JobSubmissionFiles.getJobDistCacheArchives(submitJobDir); - Path libjarsDir = JobSubmissionFiles.getJobDistCacheLibjars(submitJobDir); - // add all the command line files/ jars and archive - // first copy them to jobtrackers filesystem - - if (files != null) { - FileSystem.mkdirs(jtFs, filesDir, mapredSysPerms); - String[] fileArr = files.split(","); - for (String tmpFile: fileArr) { - URI tmpURI = null; - try { - tmpURI = new URI(tmpFile); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - Path tmp = new Path(tmpURI); - Path newPath = copyRemoteFiles(filesDir, tmp, conf, replication); - try { - URI pathURI = getPathURI(newPath, tmpURI.getFragment()); - DistributedCache.addCacheFile(pathURI, conf); - } catch(URISyntaxException ue) { - //should not throw a uri exception - throw new IOException("Failed to create uri for " + tmpFile, ue); - } - } - } - - if (libjars != null) { - FileSystem.mkdirs(jtFs, libjarsDir, mapredSysPerms); - String[] libjarsArr = libjars.split(","); - for (String tmpjars: libjarsArr) { - Path tmp = new Path(tmpjars); - Path newPath = copyRemoteFiles(libjarsDir, tmp, conf, replication); - DistributedCache.addFileToClassPath( - new Path(newPath.toUri().getPath()), conf); - } - } - - if (archives != null) { - FileSystem.mkdirs(jtFs, archivesDir, mapredSysPerms); - String[] archivesArr = archives.split(","); - for (String tmpArchives: archivesArr) { - URI tmpURI; - try { - tmpURI = new URI(tmpArchives); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - Path tmp = new Path(tmpURI); - Path newPath = copyRemoteFiles(archivesDir, tmp, conf, - replication); - try { - URI pathURI = getPathURI(newPath, tmpURI.getFragment()); - DistributedCache.addCacheArchive(pathURI, conf); - } catch(URISyntaxException ue) { - //should not throw an uri excpetion - throw new IOException("Failed to create uri for " + tmpArchives, ue); - } - } - } - - if (jobJar != null) { // copy jar to JobTracker's fs - // use jar name if job is not named. - if ("".equals(job.getJobName())){ - job.setJobName(new Path(jobJar).getName()); - } - Path jobJarPath = new Path(jobJar); - URI jobJarURI = jobJarPath.toUri(); - // If the job jar is already in fs, we don't need to copy it from local fs - if (jobJarURI.getScheme() == null || jobJarURI.getAuthority() == null - || !(jobJarURI.getScheme().equals(jtFs.getUri().getScheme()) - && jobJarURI.getAuthority().equals( - jtFs.getUri().getAuthority()))) { - copyJar(jobJarPath, JobSubmissionFiles.getJobJar(submitJobDir), - replication); - job.setJar(JobSubmissionFiles.getJobJar(submitJobDir).toString()); - } - } else { - LOG.warn("No job jar file set. User classes may not be found. "+ - "See Job or Job#setJar(String)."); - } - - // set the timestamps of the archives and files - // set the public/private visibility of the archives and files - ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(conf); - // get DelegationToken for each cached file - ClientDistributedCacheManager.getDelegationTokens(conf, job - .getCredentials()); - } - - private URI getPathURI(Path destPath, String fragment) - throws URISyntaxException { - URI pathURI = destPath.toUri(); - if (pathURI.getFragment() == null) { - if (fragment == null) { - pathURI = new URI(pathURI.toString() + "#" + destPath.getName()); - } else { - pathURI = new URI(pathURI.toString() + "#" + fragment); - } - } - return pathURI; - } - - private void copyJar(Path originalJarPath, Path submitJarFile, - short replication) throws IOException { - jtFs.copyFromLocalFile(originalJarPath, submitJarFile); - jtFs.setReplication(submitJarFile, replication); - jtFs.setPermission(submitJarFile, new FsPermission(JobSubmissionFiles.JOB_FILE_PERMISSION)); - } /** - * configure the jobconf of the user with the command line options of - * -libjars, -files, -archives. - * @param job - * @throws IOException - */ - private void copyAndConfigureFiles(Job job, Path jobSubmitDir) - throws IOException { - Configuration conf = job.getConfiguration(); - short replication = (short)conf.getInt(Job.SUBMIT_REPLICATION, 10); - copyAndConfigureFiles(job, jobSubmitDir, replication); - - // Set the working directory - if (job.getWorkingDirectory() == null) { - job.setWorkingDirectory(jtFs.getWorkingDirectory()); - } - - } - /** * Internal method for submitting jobs to the system. * *

The job submission process involves: @@ -451,6 +225,27 @@ JobStatus submitJobInternal(Job job, Cluster cluster) } } + /** + * configure the jobconf of the user with the command line options of + * -libjars, -files, -archives. + * + * @param job + * @param jobSubmitDir + * @throws IOException + */ + private void copyAndConfigureFiles(Job job, Path jobSubmitDir) + throws IOException { + + FileUploader fUploader = new FileUploader(jtFs); + fUploader.uploadFiles(job, jobSubmitDir); + + // Set the working directory + if (job.getWorkingDirectory() == null) { + job.setWorkingDirectory(jtFs.getWorkingDirectory()); + } + + } + private void checkSpecs(Job job) throws ClassNotFoundException, InterruptedException, IOException { JobConf jConf = (JobConf)job.getConfiguration(); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 4c48cf5..6505ea3 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -128,6 +128,46 @@ public static final String CACHE_ARCHIVES_VISIBILITIES = "mapreduce.job.cache.archives.visibilities"; + public static final String JOBJAR_VISIBILITY = + "mapreduce.job.jobjar.visibility"; + public static final boolean JOBJAR_VISIBILITY_DEFAULT = false; + + public static final String JOBJAR_SHARED_CACHE_UPLOAD_POLICY = + "mapreduce.job.jobjar.sharedcache.uploadpolicy"; + public static final boolean JOBJAR_SHARED_CACHE_UPLOAD_POLICY_DEFAULT = false; + + public static final String CACHE_FILES_SHARED_CACHE_UPLOAD_POLICIES = + "mapreduce.job.cache.files.sharedcache.uploadpolicies"; + + public static final String CACHE_ARCHIVES_SHARED_CACHE_UPLOAD_POLICIES = + "mapreduce.job.cache.archives.sharedcache.uploadpolicies"; + + public static final String FILES_FOR_SHARED_CACHE = + "mapreduce.job.cache.sharedcache.files"; + + public static final String FILES_FOR_CLASSPATH_AND_SHARED_CACHE = + "mapreduce.job.cache.sharedcache.files.addtoclasspath"; + + public static final String ARCHIVES_FOR_SHARED_CACHE = + "mapreduce.job.cache.sharedcache.archives"; + + // Job submitter will store checksums of those resources found in shared cache + // MRAppMaster will release them upon job completion. + public static final String SHARED_CACHE_CHECKSUMS = + "mapreduce.job.sharedcache.checksums"; + + public static final int SUBMIT_FILE_REPLICATION_DEFAULT = 10; + + // A comma delimited list of resource categories to submit to the shared + // cache. + // The valid categories are: jobjar, libjars, files, archives. + // If "disabled" is specified then the job submission code will not use + // the shared cache. + public static final String SHARED_CACHE_MODE = + "mapreduce.job.sharedcache.mode"; + + public static final String SHARED_CACHE_MODE_DEFAULT = "disabled"; + /** * @deprecated Symlinks are always on and cannot be disabled. */ diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/SharedCacheConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/SharedCacheConfig.java new file mode 100644 index 0000000..8dd3dba --- /dev/null +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/SharedCacheConfig.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce; + +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +public class SharedCacheConfig { + protected static final Log LOG = LogFactory.getLog(SharedCacheConfig.class); + + private boolean sharedCacheFilesEnabled = false; + private boolean sharedCacheLibjarsEnabled = false; + private boolean sharedCacheArchivesEnabled = false; + private boolean sharedCacheJobjarEnabled = false; + + public void init(Configuration conf) { + if (!MRConfig.YARN_FRAMEWORK_NAME.equals(conf.get(MRConfig.FRAMEWORK_NAME))) { + // Shared cache is only valid if the job runs on yarn + return; + } + + if(!conf.getBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, + YarnConfiguration.DEFAULT_SHARED_CACHE_ENABLED)) { + return; + } + + + Collection configs = StringUtils.getTrimmedStringCollection( + conf.get(MRJobConfig.SHARED_CACHE_MODE, + MRJobConfig.SHARED_CACHE_MODE_DEFAULT)); + if (configs.contains("files")) { + this.sharedCacheFilesEnabled = true; + } + if (configs.contains("libjars")) { + this.sharedCacheLibjarsEnabled = true; + } + if (configs.contains("archives")) { + this.sharedCacheArchivesEnabled = true; + } + if (configs.contains("jobjar")) { + this.sharedCacheJobjarEnabled = true; + } + if (configs.contains("enabled")) { + this.sharedCacheFilesEnabled = true; + this.sharedCacheLibjarsEnabled = true; + this.sharedCacheArchivesEnabled = true; + this.sharedCacheJobjarEnabled = true; + } + if (configs.contains("disabled")) { + this.sharedCacheFilesEnabled = false; + this.sharedCacheLibjarsEnabled = false; + this.sharedCacheArchivesEnabled = false; + this.sharedCacheJobjarEnabled = false; + } + } + + public boolean isSharedCacheFilesEnabled() { return sharedCacheFilesEnabled; } + public boolean isSharedCacheLibjarsEnabled() { + return sharedCacheLibjarsEnabled; + } + public boolean isSharedCacheArchivesEnabled() { + return sharedCacheArchivesEnabled; + } + public boolean isSharedCacheJobjarEnabled() { + return sharedCacheJobjarEnabled; + } + public boolean isSharedCacheEnabled() { + return (sharedCacheFilesEnabled || sharedCacheLibjarsEnabled || + sharedCacheArchivesEnabled || sharedCacheJobjarEnabled); + } +} diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java index eaa5af8..34c7bf8 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java @@ -29,6 +29,7 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.SharedCacheConfig; import java.net.URI; @@ -132,7 +133,7 @@ @Deprecated @InterfaceAudience.Private public class DistributedCache { - + /** * Set the configuration with the given set of archives. Intended * to be used by user code. @@ -314,8 +315,9 @@ public static void addFileToClassPath(Path file, Configuration conf) (Path file, Configuration conf, FileSystem fs) throws IOException { String classpath = conf.get(MRJobConfig.CLASSPATH_FILES); - conf.set(MRJobConfig.CLASSPATH_FILES, classpath == null ? file.toString() - : classpath + "," + file.toString()); + String fileForClassPath = getPathStringWithoutFragment(file); + conf.set(MRJobConfig.CLASSPATH_FILES, classpath == null ? + fileForClassPath : classpath + "," + fileForClassPath); URI uri = fs.makeQualified(file).toUri(); addCacheFile(uri, conf); } @@ -357,11 +359,12 @@ public static void addArchiveToClassPath(Path archive, Configuration conf) /** * Add an archive path to the current set of classpath entries. It adds the - * archive to cache as well. Intended to be used by user code. - * + * archive to cache as well. Intended to be used by user code. + * * @param archive Path of the archive to be added * @param conf Configuration that contains the classpath setting - * @param fs FileSystem with respect to which {@code archive} should be interpreted. + * @param fs FileSystem with respect to which {@code archive} should be + * interpreted. */ public static void addArchiveToClassPath (Path archive, Configuration conf, FileSystem fs) @@ -498,4 +501,12 @@ public static boolean checkURIs(URI[] uriFiles, URI[] uriArchives) { return true; } + private static String getPathStringWithoutFragment(Path filePath) { + String fileName = filePath.toString(); + int index = fileName.indexOf("#"); + if (index != -1) { + fileName = fileName.substring(0, index); + } + return fileName; + } } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 703a103..c935584 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -543,6 +543,17 @@ + mapreduce.job.sharedcache.mode + disabled + + A comma delimited list of resource categories to submit to the shared cache. + The valid categories are: jobjar, libjars, files, archives. + If "disabled" is specified then the job submission code will not use + the shared cache. + + + + mapreduce.input.fileinputformat.split.minsize 0 The minimum size chunk that map input should be split diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 5120c85..372084e 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -306,8 +306,9 @@ public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) } } - private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type) - throws IOException { + private LocalResource createApplicationResource(FileContext fs, Path p, + LocalResourceType type, boolean shouldBeUploadedToSharedCache, + boolean visibility) throws IOException { LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs @@ -315,7 +316,9 @@ private LocalResource createApplicationResource(FileContext fs, Path p, LocalRes rsrc.setSize(rsrcStat.getLen()); rsrc.setTimestamp(rsrcStat.getModificationTime()); rsrc.setType(type); - rsrc.setVisibility(LocalResourceVisibility.APPLICATION); + rsrc.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); + rsrc.setVisibility(visibility ? LocalResourceVisibility.PUBLIC + : LocalResourceVisibility.APPLICATION); return rsrc; } @@ -352,13 +355,17 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( + yarnUrlForJobSubmitDir); localResources.put(MRJobConfig.JOB_CONF_FILE, - createApplicationResource(defaultFileContext, - jobConfPath, LocalResourceType.FILE)); + createApplicationResource(defaultFileContext, jobConfPath, + LocalResourceType.FILE, false, false)); if (jobConf.get(MRJobConfig.JAR) != null) { Path jobJarPath = new Path(jobConf.get(MRJobConfig.JAR)); - LocalResource rc = createApplicationResource(defaultFileContext, - jobJarPath, - LocalResourceType.PATTERN); + LocalResource rc = + createApplicationResource(defaultFileContext, jobJarPath, + LocalResourceType.PATTERN, jobConf.getBoolean( + MRJobConfig.JOBJAR_SHARED_CACHE_UPLOAD_POLICY, + MRJobConfig.JOBJAR_SHARED_CACHE_UPLOAD_POLICY_DEFAULT), + jobConf.getBoolean(MRJobConfig.JOBJAR_VISIBILITY, + MRJobConfig.JOBJAR_VISIBILITY_DEFAULT)); String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); rc.setPattern(pattern); @@ -376,8 +383,8 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( MRJobConfig.JOB_SPLIT_METAINFO }) { localResources.put( MRJobConfig.JOB_SUBMIT_DIR + "/" + s, - createApplicationResource(defaultFileContext, - new Path(jobSubmitDir, s), LocalResourceType.FILE)); + createApplicationResource(defaultFileContext, new Path(jobSubmitDir, + s), LocalResourceType.FILE, false, false)); } // Setup security tokens diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestFileUploader.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestFileUploader.java new file mode 100644 index 0000000..34bf219 --- /dev/null +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestFileUploader.java @@ -0,0 +1,382 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.mapreduce; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.v2.util.MRApps; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.sharedcache.SharedCacheClient; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +public class TestFileUploader { + protected static final Log LOG = LogFactory.getLog(TestFileUploader.class); + private static MiniDFSCluster dfs; + private static FileSystem localFs; + private static FileSystem remoteFs; + private static Configuration conf = new Configuration(); + private static Path TEST_ROOT_DIR; + private static Path remoteStagingDir = + new Path(MRJobConfig.DEFAULT_MR_AM_STAGING_DIR); + protected String input = "roses.are.red\nviolets.are.blue\nbunnies.are.pink\n"; + + @Before + public void cleanup() throws Exception { + remoteFs.delete(remoteStagingDir, true); + } + + @BeforeClass + public static void setup() throws IOException { + // create configuration, dfs, file system + localFs = FileSystem.getLocal(conf); + TEST_ROOT_DIR = + new Path("target", TestFileUploader.class.getName() + "-tmpDir") + .makeQualified(localFs.getUri(), localFs.getWorkingDirectory()); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); + remoteFs = dfs.getFileSystem(); + } + + @AfterClass + public static void tearDown() { + try { + if (localFs != null) { + localFs.close(); + } + if (remoteFs != null) { + remoteFs.close(); + } + if (dfs != null) { + dfs.shutdown(); + } + } catch (IOException ioe) { + LOG.info("IO exception in closing file system)" ); + ioe.printStackTrace(); + } + } + + private class MyFileUploader extends FileUploader { + private SharedCacheClient mockscClient = mock(SharedCacheClient.class); + + // Used for checksum calculation + private SharedCacheClient scClient = new SharedCacheClient(); + + MyFileUploader(FileSystem submitFs, Configuration conf) + throws IOException { + super(submitFs); + scClient.init(conf); + when(mockscClient.isScmAvailable()).thenReturn(true); + when(mockscClient.getFileChecksum(any(Path.class))).thenAnswer( + new Answer() { + @Override + public String answer(InvocationOnMock invocation) throws Throwable { + Path file = (Path)invocation.getArguments()[0]; + return scClient.getFileChecksum(file); + } + }); + } + + public void makeFileInSharedCache(Path localFile, Path remoteFile) + throws IOException { + when(mockscClient.use(any(ApplicationId.class), + eq(scClient.getFileChecksum(localFile)))).thenReturn(remoteFile); + } + + @Override + protected SharedCacheClient createSharedCacheClient(Configuration conf) { + return mockscClient; + } + } + + @Test + public void testSharedCacheDisabled() throws Exception { + JobConf jobConf = createJobConf(); + Job job = new Job(jobConf); + job.setJobID(new JobID("1234", 1)); + + // shared cache is disabled by default + uploadFilesToRemoteFS(job, jobConf, 0, 0, 0, 0, false); + + } + + @Test + public void testSharedCacheEnabled() throws Exception { + JobConf jobConf = createJobConf(); + jobConf.set(MRJobConfig.SHARED_CACHE_MODE, "enabled"); + Job job = new Job(jobConf); + job.setJobID(new JobID("1234", 1)); + + // shared cache is enabled for every file type + // the # of times SharedCacheClient.use is called should == + // total # of files/libjars/archive/jobjar + uploadFilesToRemoteFS(job, jobConf, 8, 2, 3, 2, false); + } + + @Test + public void testSharedCacheEnabledWithJobJarInSharedCache() + throws Exception { + JobConf jobConf = createJobConf(); + jobConf.set(MRJobConfig.SHARED_CACHE_MODE, "enabled"); + Job job = new Job(jobConf); + job.setJobID(new JobID("1234", 1)); + + // shared cache is enabled for every file type + // the # of times SharedCacheClient.use is called should == + // total # of files/libjars/archive/jobjar + uploadFilesToRemoteFS(job, jobConf, 8, 3, 3, 2, true); + } + + @Test + public void testSharedCacheArchivesAndLibjarsEnabled() throws Exception { + JobConf jobConf = createJobConf(); + jobConf.set(MRJobConfig.SHARED_CACHE_MODE, "archives,libjars"); + Job job = new Job(jobConf); + job.setJobID(new JobID("1234", 1)); + + // shared cache is enabled for archives and libjars type + // the # of times SharedCacheClient.use is called should == + // total # of libjars and archives + uploadFilesToRemoteFS(job, jobConf, 5, 2, 1, 2, false); + } + + private JobConf createJobConf() { + JobConf jobConf = new JobConf(); + jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + jobConf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); + + jobConf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, remoteFs.getUri().toString()); + return jobConf; + } + + private Path copyToRemote(Path jar) throws IOException { + Path remoteFile = new Path("/tmp", jar.getName()); + remoteFs.copyFromLocalFile(jar, remoteFile); + return remoteFile; + } + + private void makeJarAvailableInSharedCache( + Path jar, MyFileUploader fileUploader) + throws IOException { + // make jar cache available + copyToRemote(jar); + Path remoteFile = copyToRemote(jar); + fileUploader.makeFileInSharedCache(jar, remoteFile); + } + + private void uploadFilesToRemoteFS(Job job, JobConf jobConf, + int useCallCountExpected, int cachedFileCountExpected, + int numOfFilesShouldBeUploadedToSharedCacheExpected, + int numOfArchivesShouldBeUploadedToSharedCacheExpected, + boolean jobJarInSharedCacheBeforeUpload) throws Exception { + MyFileUploader fileUploader = new MyFileUploader(remoteFs, jobConf); + SharedCacheConfig sharedCacheConfig = new SharedCacheConfig(); + sharedCacheConfig.init(jobConf); + + Path firstFile = createTempFile("first-input-file", "x"); + Path secondFile = createTempFile("second-input-file", "xx"); + + // Add files to job conf via distributed cache API as well as command line + boolean fileAdded = Job.addCacheFileShared(firstFile.toUri(), jobConf); + assertEquals(sharedCacheConfig.isSharedCacheFilesEnabled(), fileAdded); + if (!fileAdded) { + Path remoteFile = copyToRemote(firstFile); + job.addCacheFile(remoteFile.toUri()); + } + jobConf.set("tmpfiles", secondFile.toString()); + + // Create jars with a single file inside them. + Path firstJar = makeJar(new Path(TEST_ROOT_DIR, "distributed.first.jar"), 1); + Path secondJar = makeJar(new Path(TEST_ROOT_DIR, "distributed.second.jar"), 2); + + // Verify duplicated contents can be handled properly. + Path thirdJar = new Path(TEST_ROOT_DIR, "distributed.third.jar"); + localFs.copyFromLocalFile(secondJar, thirdJar); + + // make secondJar cache available + makeJarAvailableInSharedCache(secondJar, fileUploader); + + // Add libjars to job conf via distributed cache API as well as command line + boolean libjarAdded = + Job.addFileToClassPathShared(firstJar.toUri(), jobConf); + assertEquals(sharedCacheConfig.isSharedCacheLibjarsEnabled(), libjarAdded); + if (!libjarAdded) { + Path remoteJar = copyToRemote(firstJar); + job.addFileToClassPath(remoteJar); + } + + jobConf.set("tmpjars", secondJar.toString() + "," + thirdJar.toString()); + + Path firstArchive = makeArchive("first-archive.zip", "first-file"); + Path secondArchive = makeArchive("second-archive.zip", "second-file"); + + // Add archives to job conf via distributed cache API as well as command line + boolean archiveAdded = + Job.addCacheArchiveToShared(firstArchive.toUri(), jobConf); + assertEquals(sharedCacheConfig.isSharedCacheArchivesEnabled(), archiveAdded); + if (!archiveAdded) { + Path remoteArchive = copyToRemote(firstArchive); + job.addCacheArchive(remoteArchive.toUri()); + } + + jobConf.set("tmparchives", secondArchive.toString()); + + // Add job jar to job conf + Path jobJar = makeJar(new Path(TEST_ROOT_DIR, "test-job.jar"), 4); + if (jobJarInSharedCacheBeforeUpload) { + makeJarAvailableInSharedCache(jobJar, fileUploader); + } + jobConf.setJar(jobJar.toString()); + + fileUploader.uploadFiles(job, remoteStagingDir); + + verify(fileUploader.mockscClient, times(useCallCountExpected)).use( + any(ApplicationId.class), anyString()); + + int numOfFilesShouldBeUploadedToSharedCache = 0; + boolean[] filesSharedCacheUploadPolicies = + Job.getFilesSharedCacheUploadPolicies(jobConf); + int i = 0; + for (i=0; i localResources = new HashMap(); + MRApps.setupDistributedCache(jobConf, localResources); + for (Entry localResource : localResources.entrySet() ) { + if (localResource.getKey().compareTo("distributed.first.jar") == 0 && + numOfFilesShouldBeUploadedToSharedCacheExpected != 0) { + assertTrue(localResource.getValue().getShouldBeUploadedToSharedCache()); + } + } + assertEquals(7, localResources.size()); + } + + + private Path createTempFile(String filename, String contents) + throws IOException { + Path path = new Path(TEST_ROOT_DIR, filename); + FSDataOutputStream os = localFs.create(path); + os.writeBytes(contents); + os.close(); + localFs.setPermission(path, new FsPermission("700")); + return path; + } + + private Path makeJar(Path p, int index) throws FileNotFoundException, + IOException { + FileOutputStream fos = + new FileOutputStream(new File(p.toUri().getPath())); + JarOutputStream jos = new JarOutputStream(fos); + ZipEntry ze = new ZipEntry("distributed.jar.inside" + index); + jos.putNextEntry(ze); + jos.write(("inside the jar!" + index).getBytes()); + jos.closeEntry(); + jos.close(); + localFs.setPermission(p, new FsPermission("700")); + return p; + } + + private Path makeArchive(String archiveFile, String filename) throws Exception{ + Path archive = new Path(TEST_ROOT_DIR, archiveFile); + Path file = new Path(TEST_ROOT_DIR, filename); + DataOutputStream out = localFs.create(archive); + ZipOutputStream zos = new ZipOutputStream(out); + ZipEntry ze = new ZipEntry(file.toString()); + zos.putNextEntry(ze); + zos.write(input.getBytes("UTF-8")); + zos.closeEntry(); + zos.close(); + return archive; + } +} diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index 3215399..b39cac3 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -913,6 +913,62 @@ private void createAndAddJarToJar(JarOutputStream jos, File jarFile) jarFile.delete(); } + @Test + public void testSharedCache() throws Exception { + Path localJobJarPath = makeJobJarWithLib(TEST_ROOT_DIR.toUri().toString()); + + if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { + LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + + " not found. Not running test."); + return; + } + + Job job = Job.getInstance(mrCluster.getConfig()); + + Configuration jobConf = job.getConfiguration(); + jobConf.set(MRJobConfig.SHARED_CACHE_MODE, "enabled"); + + Path inputFile = createTempFile("input-file", "x"); + + // Create jars with a single file inside them. + Path second = makeJar(new Path(TEST_ROOT_DIR, "distributed.second.jar"), 2); + Path third = makeJar(new Path(TEST_ROOT_DIR, "distributed.third.jar"), 3); + Path fourth = makeJar(new Path(TEST_ROOT_DIR, "distributed.fourth.jar"), 4); + + // Add libjars to job conf + jobConf.set("tmpjars", second.toString() + "," + third.toString() + "," + + fourth.toString()); + + // Because the job jar is a "dummy" jar, we need to include the jar with + // DistributedCacheChecker or it won't be able to find it + Path distributedCacheCheckerJar = + new Path(JarFinder.getJar(SharedCacheChecker.class)); + job.addFileToClassPath(distributedCacheCheckerJar.makeQualified( + localFs.getUri(), distributedCacheCheckerJar.getParent())); + + job.setMapperClass(SharedCacheChecker.class); + job.setOutputFormatClass(NullOutputFormat.class); + + FileInputFormat.setInputPaths(job, inputFile); + + job.setMaxMapAttempts(1); // speed up failures + + job.submit(); + String trackingUrl = job.getTrackingURL(); + String jobId = job.getJobID().toString(); + Assert.assertTrue(job.waitForCompletion(true)); + Assert.assertTrue("Tracking URL was " + trackingUrl + + " but didn't Match Job ID " + jobId, + trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/")); + } + + public static class SharedCacheChecker extends + Mapper { + @Override + public void setup(Context context) throws IOException { + } + } + public static class ConfVerificationMapper extends SleepMapper { @Override protected void setup(Context context) diff --git hadoop-yarn-project/hadoop-yarn/bin/yarn hadoop-yarn-project/hadoop-yarn/bin/yarn index dfef811..2c90b14 100644 --- hadoop-yarn-project/hadoop-yarn/bin/yarn +++ hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -33,6 +33,8 @@ function hadoop_usage echo " resourcemanager run the ResourceManager" echo " resourcemanager -format-state-store deletes the RMStateStore" echo " rmadmin admin tools" + echo " sharedcachemanager run the SharedCacheManager daemon" + echo " scmadmin SharedCacheManager admin tools" echo " timelineserver run the timeline server" echo " version print the version" echo " or" @@ -139,6 +141,15 @@ case "${COMMAND}" in JAVA_HEAP_MAX="-Xmx${YARN_TIMELINESERVER_HEAPSIZE}m" fi ;; + sharedcachemanager) + daemon="true" + CLASS='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager' + YARN_OPTS="$YARN_OPTS $YARN_SHAREDCACHEMANAGER_OPTS" + ;; + scmadmin) + CLASS='org.apache.hadoop.yarn.client.SCMAdmin' + YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" + ;; version) CLASS=org.apache.hadoop.util.VersionInfo YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index affbe03..600bb54 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -96,6 +96,8 @@ server/resourcemanager_administration_protocol.proto application_history_client.proto server/application_history_server.proto + client_SCM_protocol.proto + SCM_Admin_protocol.proto ${project.build.directory}/generated-sources/java diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java new file mode 100644 index 0000000..74efbe9 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + *

+ * The protocol between clients and the SharedCacheManager to claim + * and release resources in the shared cache. + *

+ */ +@Public +@Stable +public interface ClientSCMProtocol { + /** + *

+ * The interface used by clients to claim a resource with the + * SharedCacheManager. The client uses a checksum to identify the + * resource and an {@link ApplicationId} to identify which application will be + * using the resource. + *

+ * + *

+ * The SharedCacheManager responds with whether or not the + * resource exists in the cache. If the resource exists, a Path + * to the resource in the shared cache is returned. If the resource does not + * exist, the response is empty. + *

+ * + * @param request request to claim a resource in the shared cache + * @return response indicating if the resource is already in the cache + * @throws YarnException + * @throws IOException + */ + public UseSharedCacheResourceResponse use( + UseSharedCacheResourceRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by clients to release a resource with the + * SharedCacheManager. This method is called once an application + * is no longer using a claimed resource in the shared cache. The client uses + * a checksum to identify the resource and an {@link ApplicationId} to + * identify which application is releasing the resource. + *

+ * + *

+ * Note: This method is an optimization and the client is not required to call + * it for correctness. + *

+ * + *

+ * Currently the SharedCacheManager sends an empty response. + *

+ * + * @param request request to release a resource in the shared cache + * @return (empty) response on releasing the resource + * @throws YarnException + * @throws IOException + */ + public ReleaseSharedCacheResourceResponse release( + ReleaseSharedCacheResourceRequest request) throws YarnException, IOException; + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocolPB.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocolPB.java new file mode 100644 index 0000000..b0a9fb5 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocolPB.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api; + +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.proto.ClientSCMProtocol.ClientSCMProtocolService; + +@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.api.ClientSCMProtocolPB", + protocolVersion = 1) +public interface ClientSCMProtocolPB extends + ClientSCMProtocolService.BlockingInterface { + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocol.java new file mode 100644 index 0000000..1bc618d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocol.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + *

+ * The protocol between administrators and the SharedCacheManager + *

+ */ +@Public +@Stable +public interface SCMAdminProtocol { + /** + *

+ * The method used by administrators to ask SCM to run cleaner task right away + *

+ * + * @param request request SharedCacheManager to run a cleaner task + * @return SharedCacheManager returns an empty response + * on success and throws an exception on rejecting the request + * @throws YarnException + * @throws IOException + */ + public RunSharedCacheCleanerTaskResponse runCleanerTask( + RunSharedCacheCleanerTaskRequest request) throws YarnException, IOException; + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocolPB.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocolPB.java new file mode 100644 index 0000000..1883c16c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocolPB.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api; + +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.proto.SCMAdminProtocol.SCMAdminProtocolService; + +@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.api.SCMAdminProtocolPB", + protocolVersion = 1) +public interface SCMAdminProtocolPB extends + SCMAdminProtocolService.BlockingInterface { +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceRequest.java new file mode 100644 index 0000000..993451b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceRequest.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + *

The request from clients to release a resource in the shared cache.

+ */ +@Public +@Stable +public abstract class ReleaseSharedCacheResourceRequest { + + /** + * Get the ApplicationId of the resource to be released. + * + * @return ApplicationId + */ + @Public + @Stable + public abstract ApplicationId getAppId(); + + /** + * Set the ApplicationId of the resource to be released. + * + * @param id ApplicationId + */ + @Public + @Stable + public abstract void setAppId(ApplicationId id); + + /** + * Get the key of the resource to be released. + * + * @return key + */ + @Public + @Stable + public abstract String getResourceKey(); + + /** + * Set the key of the resource to be released. + * + * @param key unique identifier for the resource + */ + @Public + @Stable + public abstract void setResourceKey(String key); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceResponse.java new file mode 100644 index 0000000..c36f53d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceResponse.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The response to clients from the SharedCacheManager when + * releasing a resource in the shared cache. + *

+ * + *

+ * Currently, this is empty. + *

+ */ +@Public +@Stable +public abstract class ReleaseSharedCacheResourceResponse { +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskRequest.java new file mode 100644 index 0000000..7407b28 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskRequest.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The request from admin to ask the SharedCacheManager to run + * cleaner service right away. + *

+ * + *

+ * Currently, this is empty. + *

+ */ +@Public +@Stable +public abstract class RunSharedCacheCleanerTaskRequest { +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskResponse.java new file mode 100644 index 0000000..e68baa7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskResponse.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The response to admin from the SharedCacheManager when + * is asked to run the cleaner service. + *

+ * + *

+ * Currently, this is empty. + *

+ */ +@Public +@Stable +public abstract class RunSharedCacheCleanerTaskResponse { + + /** + * Get whether or not the shared cache manager has accepted the request. + * Shared cache manager will reject the request if there is an ongoing task + * + * @return boolean True if the resource has been accepted, false otherwise. + */ + @Public + @Stable + public abstract boolean getAccepted(); + + /** + * Set whether or not the shared cache manager has accepted the request + * Shared cache manager will reject the request if there is an ongoing task + * + * @param b True if the resource has been accepted, false otherwise. + */ + @Public + @Stable + public abstract void setAccepted(boolean b); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceRequest.java new file mode 100644 index 0000000..a0869fe --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceRequest.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + *

+ * The request from clients to the SharedCacheManager that claims a + * resource in the shared cache. + *

+ */ +@Public +@Stable +public abstract class UseSharedCacheResourceRequest { + + /** + * Get the ApplicationId of the resource to be used. + * + * @return ApplicationId + */ + @Public + @Stable + public abstract ApplicationId getAppId(); + + /** + * Set the ApplicationId of the resource to be used. + * + * @param id ApplicationId + */ + @Public + @Stable + public abstract void setAppId(ApplicationId id); + + /** + * Get the key of the resource to be used. + * + * @return key + */ + @Public + @Stable + public abstract String getResourceKey(); + + /** + * Set the key of the resource to be used. + * + * @param key unique identifier for the resource + */ + @Public + @Stable + public abstract void setResourceKey(String key); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceResponse.java new file mode 100644 index 0000000..35d1a70 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceResponse.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The response from the SharedCacheManager to the client that indicates whether + * a requested resource exists in the cache. + *

+ */ +@Public +@Stable +public abstract class UseSharedCacheResourceResponse { + + /** + * Get the Path corresponding to the requested resource in the + * shared cache. + * + * @return String A Path if the resource exists in the shared + * cache, null otherwise + */ + @Public + @Stable + public abstract String getPath(); + + /** + * Set the Path corresponding to a resource in the shared cache. + * + * @param p A Path corresponding to a resource in the shared + * cache + */ + @Public + @Stable + public abstract void setPath(String p); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java index f14a136..7ece41b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java @@ -48,6 +48,14 @@ public static LocalResource newInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, String pattern) { + return newInstance(url, type, visibility, size, timestamp, pattern, false); + } + + @Public + @Stable + public static LocalResource newInstance(URL url, LocalResourceType type, + LocalResourceVisibility visibility, long size, long timestamp, + String pattern, boolean shouldBeUploadedToSharedCache) { LocalResource resource = Records.newRecord(LocalResource.class); resource.setResource(url); resource.setType(type); @@ -55,6 +63,7 @@ public static LocalResource newInstance(URL url, LocalResourceType type, resource.setSize(size); resource.setTimestamp(timestamp); resource.setPattern(pattern); + resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return resource; } @@ -65,6 +74,15 @@ public static LocalResource newInstance(URL url, LocalResourceType type, return newInstance(url, type, visibility, size, timestamp, null); } + @Public + @Stable + public static LocalResource newInstance(URL url, LocalResourceType type, + LocalResourceVisibility visibility, long size, long timestamp, + boolean shouldBeUploadedToSharedCache) { + return newInstance(url, type, visibility, size, timestamp, null, + shouldBeUploadedToSharedCache); + } + /** * Get the location of the resource to be localized. * @return location of the resource to be localized @@ -170,4 +188,23 @@ public static LocalResource newInstance(URL url, LocalResourceType type, @Public @Stable public abstract void setPattern(String pattern); + + /** + * NM uses it to decide whether if it is necessary to upload the resource + * shared cache + */ + @Public + @Stable + public abstract boolean getShouldBeUploadedToSharedCache(); + + /** + * Inform NM whether upload to SCM is needed. + * + * @param shouldBeUploadedToSharedCache shouldBeUploadedToSharedCache + * of this request + */ + @Public + @Stable + public abstract void setShouldBeUploadedToSharedCache( + boolean shouldBeUploadedToSharedCache); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 034ec4f..7e9dd3b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1239,6 +1239,126 @@ public static final String TIMELINE_SERVICE_KEYTAB = TIMELINE_SERVICE_PREFIX + "keytab"; + // /////////////////////////////// + // Shared Cache Configs + // /////////////////////////////// + public static final String SHARED_CACHE_PREFIX = "yarn.sharedcache."; + + // common configs + /** whether the shared cache is enabled/disabled */ + public static final String SHARED_CACHE_ENABLED = SHARED_CACHE_PREFIX + + "enabled"; + public static final boolean DEFAULT_SHARED_CACHE_ENABLED = false; + + /** The config key for the shared cache root directory. */ + public static final String SHARED_CACHE_ROOT = SHARED_CACHE_PREFIX + + "root"; + public static final String DEFAULT_SHARED_CACHE_ROOT = "/sharedcache"; + + /** The config key for the level of nested directories before getting to the + * checksum directory. */ + public static final String SHARED_CACHE_NESTED_LEVEL = SHARED_CACHE_PREFIX + + "nested.level"; + public static final int DEFAULT_SHARED_CACHE_NESTED_LEVEL = 3; + + // Shared Cache Manager Configs + public static final String SCM_PREFIX = SHARED_CACHE_PREFIX + "manager."; + + public static final String SCM_STORE_IMPL = SCM_PREFIX + "store.impl"; + public static final String DEFAULT_SCM_STORE_IMPL = + "org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore"; + + // Cleaner configs + private static final String SCM_CLEANER_PREFIX = SHARED_CACHE_PREFIX + + "cleaner."; + + /** The config key for the shared cache staleness criteria. */ + public static final String SCM_CLEANER_STALENESS_MINUTES = SCM_CLEANER_PREFIX + + "staleness.minutes"; + public static final int DEFAULT_SCM_CLEANER_STALENESS_MINUTES = 7 * 24 * 60; + + /** + * The config key for how often the cleaner service for the shared cache runs. + */ + public static final String SCM_CLEANER_CLEANING_PERIOD_MINUTES = + SCM_CLEANER_PREFIX + "period.minutes"; + public static final int DEFAULT_SCM_CLEANER_CLEANING_PERIOD_MINUTES = 24 * 60; + + /** The config key for the initial delay in starting the cleaner service. */ + public static final String SCM_CLEANER_CLEANING_INITIAL_DELAY_MINUTES = + SCM_CLEANER_PREFIX + "initial.delay.minutes"; + public static final int DEFAULT_SCM_CLEANER_CLEANING_INITIAL_DELAY_MINUTES = + 10; + + /** The config key for sleep time between cleaning each directory. */ + public static final String SCM_CLEANER_CLEANING_SLEEP_BETWEEN_CLEAN_MS = + SCM_CLEANER_PREFIX + "cleaner.sleep.between.clean.ms"; + public static final long DEFAULT_SCM_CLEANER_CLEANING_SLEEP_BETWEEN_CLEAN_MS = + 0L; + + /** The address of the node manager interface in the SCM. */ + public static final String NM_SCM_ADDRESS = SCM_PREFIX + + "nodemanager.address"; + public static final int DEFAULT_NM_SCM_PORT = 8046; + public static final String DEFAULT_NM_SCM_ADDRESS = "0.0.0.0:" + + DEFAULT_NM_SCM_PORT; + + /** + * The number of SCM threads used to handle notify requests from the node + * manager. + */ + public static final String SCM_NM_THREAD_COUNT = SCM_PREFIX + + "nodemanager.thread-count"; + public static final int DEFAULT_SCM_NM_THREAD_COUNT = 50; + + /** The address of the client interface in the SCM. */ + public static final String SCM_ADDRESS = SCM_PREFIX + "client.address"; + public static final int DEFAULT_SCM_PORT = 8045; + public static final String DEFAULT_SCM_ADDRESS = "0.0.0.0:" + + DEFAULT_SCM_PORT; + + /** The number of threads used to handle shared cache manager requests. */ + public static final String SCM_CLIENT_THREAD_COUNT = SCM_PREFIX + + "client.thread-count"; + public static final int DEFAULT_SCM_CLIENT_THREAD_COUNT = 50; + + /** The address of the SCM admin interface. */ + public static final String SCM_ADMIN_ADDRESS = SCM_PREFIX + "admin.address"; + public static final int DEFAULT_SCM_ADMIN_PORT = 8047; + public static final String DEFAULT_SCM_ADMIN_ADDRESS = "0.0.0.0:" + + DEFAULT_SCM_ADMIN_PORT; + + /** Number of threads used to handle SCM admin interface. */ + public static final String SCM_ADMIN_CLIENT_THREAD_COUNT = SCM_PREFIX + + "admin.client.thread-count"; + public static final int DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT = 1; + + /** The address of the SCM web application. */ + public static final String SCM_WEBAPP_ADDRESS = SCM_PREFIX + "webapp.address"; + + public static final int DEFAULT_SCM_WEBAPP_PORT = 8788; + public static final String DEFAULT_SCM_WEBAPP_ADDRESS = "0.0.0.0:" + + DEFAULT_SCM_WEBAPP_PORT; + + /** the checksum algorithm implementation **/ + public static final String SHARED_CACHE_CHECKSUM_ALGO_IMPL = + SHARED_CACHE_PREFIX + "checksum.algo.impl"; + public static final String DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL = + "org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl"; + + // node manager (uploader) configs + /** + * The replication factor for the node manager uploader for the shared cache. + */ + public static final String SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR = + SHARED_CACHE_PREFIX + "nm.uploader.replication.factor"; + public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR = + 10; + + public static final String SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = + SHARED_CACHE_PREFIX + "nm.uploader.thread-count"; + public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 20; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/SCM_Admin_protocol.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/SCM_Admin_protocol.proto new file mode 100644 index 0000000..4e46c57 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/SCM_Admin_protocol.proto @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "SCMAdminProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_service_protos.proto"; + +service SCMAdminProtocolService { + rpc runCleanerTask (RunSharedCacheCleanerTaskRequestProto) returns (RunSharedCacheCleanerTaskResponseProto); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_SCM_protocol.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_SCM_protocol.proto new file mode 100644 index 0000000..fbc3c42 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_SCM_protocol.proto @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "ClientSCMProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_service_protos.proto"; + +service ClientSCMProtocolService { + rpc use (UseSharedCacheResourceRequestProto) returns (UseSharedCacheResourceResponseProto); + rpc release (ReleaseSharedCacheResourceRequestProto) returns (ReleaseSharedCacheResourceResponseProto); +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 3f1fa6c..33e017c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -159,6 +159,7 @@ message LocalResourceProto { optional LocalResourceTypeProto type = 4; optional LocalResourceVisibilityProto visibility = 5; optional string pattern = 6; + optional bool should_be_uploaded_to_shared_cache = 7; } message ApplicationResourceUsageReportProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index df8784b..56d66a5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -286,3 +286,35 @@ message GetContainersRequestProto { message GetContainersResponseProto { repeated ContainerReportProto containers = 1; } + +////////////////////////////////////////////////////// +/////// client_SCM_Protocol ////////////////////////// +////////////////////////////////////////////////////// + +message UseSharedCacheResourceRequestProto { + optional ApplicationIdProto applicationId = 1; + optional string resourceKey = 2; +} + +message UseSharedCacheResourceResponseProto { + optional string path = 1; +} + +message ReleaseSharedCacheResourceRequestProto { + optional ApplicationIdProto applicationId = 1; + optional string resourceKey = 2; +} + +message ReleaseSharedCacheResourceResponseProto { +} + +////////////////////////////////////////////////////// +/////// SCM_Admin_Protocol ////////////////////////// +////////////////////////////////////////////////////// + +message RunSharedCacheCleanerTaskRequestProto { +} + +message RunSharedCacheCleanerTaskResponseProto { + optional bool accepted = 1; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java new file mode 100644 index 0000000..a743e11 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/SCMAdmin.java @@ -0,0 +1,207 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.security.PrivilegedAction; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.SCMAdminProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.ipc.YarnRPC; + +public class SCMAdmin extends Configured implements Tool { + + private final RecordFactory recordFactory = + RecordFactoryProvider.getRecordFactory(null); + + public SCMAdmin() { + super(); + } + + public SCMAdmin(Configuration conf) { + super(conf); + } + + private static void printHelp(String cmd) { + String summary = "scmadmin is the command to execute shared cache manager" + + "administrative commands.\n" + + "The full syntax is: \n\n" + + "hadoop scmadmin" + + " [-runCleanerTask]" + + " [-help [cmd]]\n"; + + String runCleanerTask = + "-runCleanerTask: Run cleaner task right away.\n"; + + String help = "-help [cmd]: \tDisplays help for the given command or all commands if none\n" + + "\t\tis specified.\n"; + + if ("runCleanerTask".equals(cmd)) { + System.out.println(runCleanerTask); + } else if ("help".equals(cmd)) { + System.out.println(help); + } else { + System.out.println(summary); + System.out.println(runCleanerTask); + System.out.println(help); + System.out.println(); + ToolRunner.printGenericCommandUsage(System.out); + } + } + + /** + * Displays format of commands. + * @param cmd The command that is being executed. + */ + private static void printUsage(String cmd) { + if ("-runCleanerTask".equals(cmd)) { + System.err.println("Usage: yarn scmadmin" + " [-runCleanerTask]"); + } else { + System.err.println("Usage: yarn scmadmin"); + System.err.println(" [-runCleanerTask]"); + System.err.println(" [-help [cmd]]"); + System.err.println(); + ToolRunner.printGenericCommandUsage(System.err); + } + } + + private static UserGroupInformation getUGI(Configuration conf + ) throws IOException { + return UserGroupInformation.getCurrentUser(); + } + + private SCMAdminProtocol createSCMAdminProtocol() throws IOException { + // Get the current configuration + final YarnConfiguration conf = new YarnConfiguration(getConf()); + + // Create the admin client + final InetSocketAddress addr = conf.getSocketAddr( + YarnConfiguration.SCM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADMIN_PORT); + final YarnRPC rpc = YarnRPC.create(conf); + + SCMAdminProtocol scmAdminProtocol = + getUGI(conf).doAs(new PrivilegedAction() { + @Override + public SCMAdminProtocol run() { + return (SCMAdminProtocol) rpc.getProxy(SCMAdminProtocol.class, + addr, conf); + } + }); + + return scmAdminProtocol; + } + + private int runCleanerTask() throws YarnException, IOException { + // run cleaner task right away + SCMAdminProtocol scmAdminProtocol = createSCMAdminProtocol(); + RunSharedCacheCleanerTaskRequest request = + recordFactory.newRecordInstance(RunSharedCacheCleanerTaskRequest.class); + RunSharedCacheCleanerTaskResponse response = + scmAdminProtocol.runCleanerTask(request); + if (response.getAccepted()) { + System.out.println("request accepted by shared cache manager"); + return 0; + } else { + System.out.println("request rejected by shared cache manager"); + return 1; + } + } + + @Override + public int run(String[] args) throws Exception { + if (args.length < 1) { + printUsage(""); + return -1; + } + + int exitCode = -1; + int i = 0; + String cmd = args[i++]; + // + // verify that we have enough command line parameters + // + if ("-runCleanerTask".equals(cmd)) { + if (args.length != 1) { + printUsage(cmd); + return exitCode; + } + } + + exitCode = 0; + try { + if ("-runCleanerTask".equals(cmd)) { + exitCode = runCleanerTask(); + } else if ("-help".equals(cmd)) { + if (i < args.length) { + printUsage(args[i]); + } else { + printHelp(""); + } + } else { + exitCode = -1; + System.err.println(cmd.substring(1) + ": Unknown command"); + printUsage(""); + printUsage(""); + } + + } catch (IllegalArgumentException arge) { + exitCode = -1; + System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage()); + printUsage(cmd); + } catch (RemoteException e) { + // + // This is a error returned by hadoop server. Print + // out the first line of the error mesage, ignore the stack trace. + exitCode = -1; + try { + String[] content; + content = e.getLocalizedMessage().split("\n"); + System.err.println(cmd.substring(1) + ": " + + content[0]); + } catch (Exception ex) { + System.err.println(cmd.substring(1) + ": " + + ex.getLocalizedMessage()); + } + } catch (Exception e) { + exitCode = -1; + System.err.println(cmd.substring(1) + ": " + + e.getLocalizedMessage()); + } + return exitCode; + } + + public static void main(String[] args) throws Exception { + int result = ToolRunner.run(new SCMAdmin(), args); + System.exit(result); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientSCMProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientSCMProtocolPBClientImpl.java new file mode 100644 index 0000000..58af3c4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientSCMProtocolPBClientImpl.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.impl.pb.client; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.ClientSCMProtocol; +import org.apache.hadoop.yarn.api.ClientSCMProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReleaseSharedCacheResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReleaseSharedCacheResourceResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceRequestProto; + +import com.google.protobuf.ServiceException; + +public class ClientSCMProtocolPBClientImpl implements ClientSCMProtocol, + Closeable { + + private ClientSCMProtocolPB proxy; + + public ClientSCMProtocolPBClientImpl(long clientVersion, + InetSocketAddress addr, Configuration conf) throws IOException { + RPC.setProtocolEngine(conf, ClientSCMProtocolPB.class, + ProtobufRpcEngine.class); + proxy = RPC.getProxy(ClientSCMProtocolPB.class, clientVersion, addr, conf); + } + + @Override + public void close() { + if (this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } + + @Override + public UseSharedCacheResourceResponse use( + UseSharedCacheResourceRequest request) throws YarnException, IOException { + UseSharedCacheResourceRequestProto requestProto = + ((UseSharedCacheResourceRequestPBImpl) request).getProto(); + try { + return new UseSharedCacheResourceResponsePBImpl(proxy.use(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public ReleaseSharedCacheResourceResponse release( + ReleaseSharedCacheResourceRequest request) throws YarnException, + IOException { + ReleaseSharedCacheResourceRequestProto requestProto = + ((ReleaseSharedCacheResourceRequestPBImpl) request).getProto(); + try { + return new ReleaseSharedCacheResourceResponsePBImpl(proxy.release(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/SCMAdminProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/SCMAdminProtocolPBClientImpl.java new file mode 100644 index 0000000..aabc2a3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/SCMAdminProtocolPBClientImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.impl.pb.client; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.ClientSCMProtocolPB; +import org.apache.hadoop.yarn.api.SCMAdminProtocol; +import org.apache.hadoop.yarn.api.SCMAdminProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; + +import com.google.protobuf.ServiceException; + +public class SCMAdminProtocolPBClientImpl implements SCMAdminProtocol, + Closeable { + + private SCMAdminProtocolPB proxy; + + public SCMAdminProtocolPBClientImpl(long clientVersion, + InetSocketAddress addr, Configuration conf) throws IOException { + RPC.setProtocolEngine(conf, SCMAdminProtocolPB.class, + ProtobufRpcEngine.class); + proxy = RPC.getProxy(SCMAdminProtocolPB.class, clientVersion, addr, conf); + } + + @Override + public void close() { + if (this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } + + @Override + public RunSharedCacheCleanerTaskResponse runCleanerTask( + RunSharedCacheCleanerTaskRequest request) throws YarnException, + IOException { + YarnServiceProtos.RunSharedCacheCleanerTaskRequestProto requestProto = + ((RunSharedCacheCleanerTaskRequestPBImpl) request).getProto(); + try { + return new RunSharedCacheCleanerTaskResponsePBImpl(proxy.runCleanerTask(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientSCMProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientSCMProtocolPBServiceImpl.java new file mode 100644 index 0000000..8c4e549 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientSCMProtocolPBServiceImpl.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.impl.pb.service; + +import java.io.IOException; + +import org.apache.hadoop.yarn.api.ClientSCMProtocol; +import org.apache.hadoop.yarn.api.ClientSCMProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReleaseSharedCacheResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReleaseSharedCacheResourceResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceResponseProto; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + +public class ClientSCMProtocolPBServiceImpl implements ClientSCMProtocolPB { + + private ClientSCMProtocol real; + + public ClientSCMProtocolPBServiceImpl(ClientSCMProtocol impl) { + this.real = impl; + } + + @Override + public UseSharedCacheResourceResponseProto use(RpcController controller, + UseSharedCacheResourceRequestProto proto) throws ServiceException { + UseSharedCacheResourceRequestPBImpl request = + new UseSharedCacheResourceRequestPBImpl(proto); + try { + UseSharedCacheResourceResponse response = real.use(request); + return ((UseSharedCacheResourceResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public ReleaseSharedCacheResourceResponseProto release( + RpcController controller, ReleaseSharedCacheResourceRequestProto proto) + throws ServiceException { + ReleaseSharedCacheResourceRequestPBImpl request = + new ReleaseSharedCacheResourceRequestPBImpl(proto); + try { + ReleaseSharedCacheResourceResponse response = real.release(request); + return ((ReleaseSharedCacheResourceResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/SCMAdminProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/SCMAdminProtocolPBServiceImpl.java new file mode 100644 index 0000000..937e363 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/SCMAdminProtocolPBServiceImpl.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.impl.pb.service; + +import java.io.IOException; + +import org.apache.hadoop.yarn.api.SCMAdminProtocol; +import org.apache.hadoop.yarn.api.SCMAdminProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskResponseProto; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + +public class SCMAdminProtocolPBServiceImpl implements SCMAdminProtocolPB { + + private SCMAdminProtocol real; + + public SCMAdminProtocolPBServiceImpl(SCMAdminProtocol impl) { + this.real = impl; + } + + @Override + public RunSharedCacheCleanerTaskResponseProto runCleanerTask(RpcController controller, + RunSharedCacheCleanerTaskRequestProto proto) throws ServiceException { + RunSharedCacheCleanerTaskRequestPBImpl request = + new RunSharedCacheCleanerTaskRequestPBImpl(proto); + try { + RunSharedCacheCleanerTaskResponse response = real.runCleanerTask(request); + return ((RunSharedCacheCleanerTaskResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceRequestPBImpl.java new file mode 100644 index 0000000..6db61a1 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceRequestPBImpl.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceRequestProtoOrBuilder; + +public class ReleaseSharedCacheResourceRequestPBImpl extends + ReleaseSharedCacheResourceRequest { + ReleaseSharedCacheResourceRequestProto proto = + ReleaseSharedCacheResourceRequestProto.getDefaultInstance(); + ReleaseSharedCacheResourceRequestProto.Builder builder = null; + boolean viaProto = false; + + private ApplicationId applicationId = null; + + public ReleaseSharedCacheResourceRequestPBImpl() { + builder = ReleaseSharedCacheResourceRequestProto.newBuilder(); + } + + public ReleaseSharedCacheResourceRequestPBImpl( + ReleaseSharedCacheResourceRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public ReleaseSharedCacheResourceRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public ApplicationId getAppId() { + ReleaseSharedCacheResourceRequestProtoOrBuilder p = + viaProto ? proto : builder; + if (this.applicationId != null) { + return this.applicationId; + } + if (!p.hasApplicationId()) { + return null; + } + this.applicationId = convertFromProtoFormat(p.getApplicationId()); + return this.applicationId; + } + + @Override + public void setAppId(ApplicationId id) { + maybeInitBuilder(); + if (id == null) + builder.clearApplicationId(); + this.applicationId = id; + } + + @Override + public String getResourceKey() { + ReleaseSharedCacheResourceRequestProtoOrBuilder p = + viaProto ? proto : builder; + return (p.hasResourceKey()) ? p.getResourceKey() : null; + } + + @Override + public void setResourceKey(String key) { + maybeInitBuilder(); + if (key == null) { + builder.clearResourceKey(); + return; + } + builder.setResourceKey(key); + } + + private void mergeLocalToBuilder() { + if (applicationId != null) { + builder.setApplicationId(convertToProtoFormat(this.applicationId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ReleaseSharedCacheResourceRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { + return new ApplicationIdPBImpl(p); + } + + private ApplicationIdProto convertToProtoFormat(ApplicationId t) { + return ((ApplicationIdPBImpl) t).getProto(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceResponsePBImpl.java new file mode 100644 index 0000000..559f2c8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceResponsePBImpl.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceResponseProto; + +public class ReleaseSharedCacheResourceResponsePBImpl extends + ReleaseSharedCacheResourceResponse { + ReleaseSharedCacheResourceResponseProto proto = + ReleaseSharedCacheResourceResponseProto.getDefaultInstance(); + ReleaseSharedCacheResourceResponseProto.Builder builder = null; + boolean viaProto = false; + + public ReleaseSharedCacheResourceResponsePBImpl() { + builder = ReleaseSharedCacheResourceResponseProto.newBuilder(); + } + + public ReleaseSharedCacheResourceResponsePBImpl( + ReleaseSharedCacheResourceResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public ReleaseSharedCacheResourceResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ReleaseSharedCacheResourceResponseProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskRequestPBImpl.java new file mode 100644 index 0000000..d432332 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskRequestPBImpl.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskRequestProto; + +public class RunSharedCacheCleanerTaskRequestPBImpl extends + RunSharedCacheCleanerTaskRequest { + RunSharedCacheCleanerTaskRequestProto proto = + RunSharedCacheCleanerTaskRequestProto.getDefaultInstance(); + RunSharedCacheCleanerTaskRequestProto.Builder builder = null; + boolean viaProto = false; + + public RunSharedCacheCleanerTaskRequestPBImpl() { + builder = RunSharedCacheCleanerTaskRequestProto.newBuilder(); + } + + public RunSharedCacheCleanerTaskRequestPBImpl( + RunSharedCacheCleanerTaskRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public RunSharedCacheCleanerTaskRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RunSharedCacheCleanerTaskRequestProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskResponsePBImpl.java new file mode 100644 index 0000000..c7da352 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskResponsePBImpl.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskResponseProtoOrBuilder; + +public class RunSharedCacheCleanerTaskResponsePBImpl extends + RunSharedCacheCleanerTaskResponse { + RunSharedCacheCleanerTaskResponseProto proto = + RunSharedCacheCleanerTaskResponseProto.getDefaultInstance(); + RunSharedCacheCleanerTaskResponseProto.Builder builder = null; + boolean viaProto = false; + + public RunSharedCacheCleanerTaskResponsePBImpl() { + builder = RunSharedCacheCleanerTaskResponseProto.newBuilder(); + } + + public RunSharedCacheCleanerTaskResponsePBImpl( + RunSharedCacheCleanerTaskResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public boolean getAccepted() { + RunSharedCacheCleanerTaskResponseProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasAccepted()) ? p.getAccepted() : false; + } + + @Override + public void setAccepted(boolean b) { + maybeInitBuilder(); + builder.setAccepted(b); + } + + public RunSharedCacheCleanerTaskResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RunSharedCacheCleanerTaskResponseProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceRequestPBImpl.java new file mode 100644 index 0000000..8499b9f --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceRequestPBImpl.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceRequestProtoOrBuilder; + +public class UseSharedCacheResourceRequestPBImpl extends + UseSharedCacheResourceRequest { + UseSharedCacheResourceRequestProto proto = UseSharedCacheResourceRequestProto + .getDefaultInstance(); + UseSharedCacheResourceRequestProto.Builder builder = null; + boolean viaProto = false; + + private ApplicationId applicationId = null; + + public UseSharedCacheResourceRequestPBImpl() { + builder = UseSharedCacheResourceRequestProto.newBuilder(); + } + + public UseSharedCacheResourceRequestPBImpl( + UseSharedCacheResourceRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public UseSharedCacheResourceRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public ApplicationId getAppId() { + UseSharedCacheResourceRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.applicationId != null) { + return this.applicationId; + } + if (!p.hasApplicationId()) { + return null; + } + this.applicationId = convertFromProtoFormat(p.getApplicationId()); + return this.applicationId; + } + + @Override + public void setAppId(ApplicationId id) { + maybeInitBuilder(); + if (id == null) + builder.clearApplicationId(); + this.applicationId = id; + } + + @Override + public String getResourceKey() { + UseSharedCacheResourceRequestProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasResourceKey()) ? p.getResourceKey() : null; + } + + @Override + public void setResourceKey(String key) { + maybeInitBuilder(); + if (key == null) { + builder.clearResourceKey(); + return; + } + builder.setResourceKey(key); + } + + private void mergeLocalToBuilder() { + if (applicationId != null) { + builder.setApplicationId(convertToProtoFormat(this.applicationId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = UseSharedCacheResourceRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { + return new ApplicationIdPBImpl(p); + } + + private ApplicationIdProto convertToProtoFormat(ApplicationId t) { + return ((ApplicationIdPBImpl) t).getProto(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceResponsePBImpl.java new file mode 100644 index 0000000..5a9c14b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceResponsePBImpl.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceResponseProtoOrBuilder; + +public class UseSharedCacheResourceResponsePBImpl extends + UseSharedCacheResourceResponse { + UseSharedCacheResourceResponseProto proto = + UseSharedCacheResourceResponseProto + .getDefaultInstance(); + UseSharedCacheResourceResponseProto.Builder builder = null; + boolean viaProto = false; + + public UseSharedCacheResourceResponsePBImpl() { + builder = UseSharedCacheResourceResponseProto.newBuilder(); + } + + public UseSharedCacheResourceResponsePBImpl( + UseSharedCacheResourceResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public UseSharedCacheResourceResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public String getPath() { + UseSharedCacheResourceResponseProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasPath()) ? p.getPath() : null; + } + + @Override + public void setPath(String path) { + maybeInitBuilder(); + if (path == null) { + builder.clearPath(); + return; + } + builder.setPath(path); + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = UseSharedCacheResourceResponseProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java index 16bd597..560b081 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java @@ -192,6 +192,26 @@ public synchronized void setPattern(String pattern) { builder.setPattern(pattern); } + @Override + public synchronized boolean getShouldBeUploadedToSharedCache() { + LocalResourceProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasShouldBeUploadedToSharedCache()) { + return false; + } + return p.getShouldBeUploadedToSharedCache(); + } + + @Override + public synchronized void setShouldBeUploadedToSharedCache( + boolean shouldBeUploadedToSharedCache) { + maybeInitBuilder(); + if (!shouldBeUploadedToSharedCache) { + builder.clearShouldBeUploadedToSharedCache(); + return; + } + builder.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); + } + private LocalResourceTypeProto convertToProtoFormat(LocalResourceType e) { return ProtoUtils.convertToProtoFormat(e); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/Checksum.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/Checksum.java new file mode 100644 index 0000000..2807d95 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/Checksum.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.sharedcache; + +import java.io.IOException; +import java.io.InputStream; + +/** + * An interface to calculate a checksum. The checksum implementation should be + * thread safe. + */ +public interface Checksum { + + /** + * Calculate the checksum of the passed input stream. + * + * @param in InputStream to be checksumed + * @return the message digest of the input stream + * @throws IOException + */ + public String computeChecksum(InputStream in) throws IOException; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumFactory.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumFactory.java new file mode 100644 index 0000000..d904911 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumFactory.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.sharedcache; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +/** + * A factory class for creating checksum objects based on a configurable + * algorithm implementation + */ +public class ChecksumFactory { + private static final ConcurrentMap instances = + new ConcurrentHashMap(); + + /** + * Get a new Checksum object based on the configurable algorithm + * implementation (see yarn.sharedcache.checksum.algo.impl) + * + * @return Checksum object + */ + public static Checksum getChecksum(Configuration conf) { + String className = + conf.get(YarnConfiguration.SHARED_CACHE_CHECKSUM_ALGO_IMPL, + YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL); + Checksum checksum = instances.get(className); + if (checksum == null) { + try { + Class clazz = Class.forName(className); + checksum = (Checksum) clazz.newInstance(); + Checksum old = instances.putIfAbsent(className, checksum); + if (old != null) { + checksum = old; + } + } catch (Exception e) { + throw new YarnRuntimeException(e); + } + } + + return checksum; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumSHA256Impl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumSHA256Impl.java new file mode 100644 index 0000000..f25b717 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumSHA256Impl.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.sharedcache; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.codec.digest.DigestUtils; + +/** + * The SHA-256 implementation of the shared cache checksum interface. + */ +public class ChecksumSHA256Impl implements Checksum { + public String computeChecksum(InputStream in) throws IOException { + return DigestUtils.sha256Hex(in); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheClient.java new file mode 100644 index 0000000..6e330e8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheClient.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.sharedcache; + + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.ClientSCMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.util.Records; + +/** + * This is the client for YARN's shared cache. + */ +public class SharedCacheClient extends AbstractService { + private static final Log LOG = LogFactory + .getLog(SharedCacheClient.class); + + private ClientSCMProtocol scmClient; + private InetSocketAddress scmAddress; + private Configuration conf; + private Checksum checksumAlg; + + // If scm isn't available, we will mark this instance + // of SharedCacheClient unusable. This is useful when + // the caller of SharedCacheClient needs to call the same + // instance of SharedCacheClient multiple times; it allows + // the caller to quickly fall back to the non-SCM approach. + private volatile boolean scmAvailable = false; + + public SharedCacheClient() { + this(null); + } + + public SharedCacheClient(InetSocketAddress scmAddress) { + super(SharedCacheClient.class.getName()); + this.scmAddress = scmAddress; + } + + private static InetSocketAddress getScmAddress(Configuration conf) { + return conf.getSocketAddr(YarnConfiguration.SCM_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADDRESS, + YarnConfiguration.DEFAULT_SCM_PORT); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + if (this.scmAddress == null) { + this.scmAddress = getScmAddress(conf); + } + this.conf = conf; + this.checksumAlg = ChecksumFactory.getChecksum(conf); + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + YarnRPC rpc = YarnRPC.create(getConfig()); + + this.scmClient = (ClientSCMProtocol) rpc.getProxy( + ClientSCMProtocol.class, this.scmAddress, getConfig()); + if (LOG.isDebugEnabled()) { + LOG.debug("Connecting to Shared Cache Manager at " + this.scmAddress); + } + scmAvailable = true; + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.scmClient != null) { + RPC.stopProxy(this.scmClient); + } + super.serviceStop(); + } + + public boolean isScmAvailable() { + return this.scmAvailable; + } + + public Path use(ApplicationId applicationId, Path sourceFile) + throws IOException { + + // If for whatever reason, we can't even calculate checksum for + // local resource, something is really wrong with the file system; + // even non-SCM approach won't work. Let us just throw the exception. + String checksum = getFileChecksum(sourceFile); + return use(applicationId, checksum); + } + + public Path use(ApplicationId applicationId, String resourceKey) { + + Path resourcePath = null; + UseSharedCacheResourceRequest request = Records.newRecord( + UseSharedCacheResourceRequest.class); + request.setAppId(applicationId); + request.setResourceKey(resourceKey); + try { + UseSharedCacheResourceResponse response = this.scmClient.use(request); + if (response != null && response.getPath() != null) { + resourcePath = new Path(response.getPath()); + } + } catch (Exception e) { + // Just catching IOException isn't enough. + // RPC call can throw ConnectionException. + // We don't handle different exceptions separately at this point. + LOG.warn("SCM might be down. The exception is " + e.getMessage()); + e.printStackTrace(); + scmAvailable = false; + } + return resourcePath; + } + + public void release(ApplicationId applicationId, String resourceKey) { + if (!scmAvailable) { + return; + } + ReleaseSharedCacheResourceRequest request = Records.newRecord( + ReleaseSharedCacheResourceRequest.class); + request.setAppId(applicationId); + request.setResourceKey(resourceKey); + try { + // We do not care about the response because it is empty. + this.scmClient.release(request); + } catch (Exception e) { + // Just catching IOException isn't enough. + // RPC call can throw ConnectionException. + LOG.warn("SCM might be down. The exception is " + e.getMessage()); + e.printStackTrace(); + scmAvailable = false; + } + } + + + /** + * Calculates the SHA-256 checksum for a given file and verifies the file + * length. + * + * @return A hex string containing the SHA-256 digest + * @throws IOException + */ + public String getFileChecksum(Path sourceFile) + throws IOException { + FileSystem fs = sourceFile.getFileSystem(this.conf); + FSDataInputStream in = null; + try { + in = fs.open(sourceFile); + return this.checksumAlg.computeChecksum(in); + } finally { + if (in != null) { + in.close(); + } + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java index 8cc5ed3..a32f08c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java @@ -134,7 +134,7 @@ private void createDir(Path path, FsPermission perm) throws IOException { * otherwise */ @VisibleForTesting - static boolean isPublic(FileSystem fs, Path current, FileStatus sStat, + public static boolean isPublic(FileSystem fs, Path current, FileStatus sStat, LoadingCache> statCache) throws IOException { current = fs.makeQualified(current); //the leaf level file should be readable by others diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9b4a90f..e0d0ada 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1339,6 +1339,121 @@ org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore + + + Whether the shared cache is enabled + yarn.sharedcache.enabled + false + + + + The root directory for the shared cache + yarn.sharedcache.root + /sharedcache + + + + The level of nested directories before getting to the checksum + directories. It must be non-negative. + yarn.sharedcache.nested.level + 3 + + + + The implementation to be used for the SCM store + yarn.sharedcache.manager.store.impl + org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore + + + + The staleness criteria in days beyond which shared cache + entries may be deleted (minutes). It must be positive. It is 7 days by + default. + yarn.sharedcache.cleaner.staleness.minutes + 10080 + + + + How often the cleaner service for the shared cache runs + (minutes). It must be positive. It is 1 day by default. + yarn.sharedcache.cleaner.period.minutes + 1440 + + + + The initial delay in starting the cleaner service (minutes). It + must be non-negative. + yarn.sharedcache.cleaner.initial.delay.minutes + 10 + + + + Sleep time between cleaning each directory (ms). It must be + non-negative. + yarn.sharedcache.cleaner.sleep.between.clean.ms + 10 + + + + The address of the node manager interface in the SCM (shared cache manager) + yarn.sharedcache.manager.nodemanager.address + 0.0.0.0:8046 + + + + The number of threads used to handle shared cache manager requests from the node manager (50 by default) + yarn.sharedcache.manager.nodemanager.thread-count + 50 + + + + The address of the client interface in the SCM (shared cache manager) + yarn.sharedcache.manager.client.address + 0.0.0.0:8045 + + + + The number of threads used to handle shared cache manager requests from clients (50 by default) + yarn.sharedcache.manager.client.thread-count + 50 + + + + The address of the admin interface in the SCM (shared cache manager) + yarn.sharedcache.manager.admin.address + 0.0.0.0:8047 + + + + The number of threads used to handle SCM admin interface (1 by default) + yarn.sharedcache.manager.admin.thread-count + 1 + + + + The address of the web application in the SCM (shared cache manager) + yarn.sharedcache.manager.webapp.address + 0.0.0.0:8788 + + + + The algorithm used to compute checksums of files (SHA-256 by default) + yarn.sharedcache.checksum.algo.impl + org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl + + + + The replication factor for the node manager uploader for the shared cache (10 by default) + yarn.sharedcache.nm.uploader.replication.factor + 10 + + + + The number of threads used to upload files from a node manager instance (20 by default) + yarn.sharedcache.nm.uploader.thread-count + 20 + + The interval that the yarn client library uses to poll the diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index acf330f..ca01d31 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -135,6 +135,7 @@ yarn_server_common_service_protos.proto yarn_server_common_service_protos.proto ResourceTracker.proto + NMCacheUploader_SCM_protocol.proto ${project.build.directory}/generated-sources/java diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocol.java new file mode 100644 index 0000000..89d78c0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocol.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; + +/** + *

+ * The protocol between a NodeManager's + * SharedCacheUploadService and the + * SharedCacheManager. + *

+ */ +@Public +@Stable +public interface NMCacheUploaderSCMProtocol { + /** + *

+ * The method used by the NodeManager's SharedCacheUploadService + * to notify the shared cache manager of a newly cached resource. + *

+ * + *

+ * The SharedCacheManager responds with whether or not the + * NodeManager should delete the uploaded file. + *

+ * + * @param request notify the shared cache manager of a newly uploaded resource + * to the shared cache + * @return response indicating if the newly uploaded resource should be + * deleted + * @throws YarnException + * @throws IOException + */ + public NotifySCMResponse notify(NotifySCMRequest request) + throws YarnException, IOException; + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocolPB.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocolPB.java new file mode 100644 index 0000000..13e055b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocolPB.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api; + +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.proto.NMCacheUploaderSCMProtocol.NMCacheUploaderSCMProtocolService; + +@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocolPB", + protocolVersion = 1) +public interface NMCacheUploaderSCMProtocolPB extends + NMCacheUploaderSCMProtocolService.BlockingInterface { + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/NMCacheUploaderSCMProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/NMCacheUploaderSCMProtocolPBClientImpl.java new file mode 100644 index 0000000..1cebf6d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/NMCacheUploaderSCMProtocolPBClientImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.impl.pb.client; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMRequestProto; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocolPB; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NotifySCMRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NotifySCMResponsePBImpl; + +import com.google.protobuf.ServiceException; + +public class NMCacheUploaderSCMProtocolPBClientImpl implements + NMCacheUploaderSCMProtocol, Closeable { + + private NMCacheUploaderSCMProtocolPB proxy; + + public NMCacheUploaderSCMProtocolPBClientImpl(long clientVersion, + InetSocketAddress addr, Configuration conf) throws IOException { + RPC.setProtocolEngine(conf, NMCacheUploaderSCMProtocolPB.class, + ProtobufRpcEngine.class); + proxy = + RPC.getProxy(NMCacheUploaderSCMProtocolPB.class, clientVersion, addr, + conf); + } + + @Override + public void close() { + if (this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } + + @Override + public NotifySCMResponse notify(NotifySCMRequest request) + throws YarnException, IOException { + NotifySCMRequestProto requestProto = + ((NotifySCMRequestPBImpl) request).getProto(); + try { + return new NotifySCMResponsePBImpl(proxy.notify(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/NMCacheUploaderSCMProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/NMCacheUploaderSCMProtocolPBServiceImpl.java new file mode 100644 index 0000000..7bac95c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/NMCacheUploaderSCMProtocolPBServiceImpl.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.impl.pb.service; + +import java.io.IOException; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMResponseProto; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocolPB; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NotifySCMRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NotifySCMResponsePBImpl; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + +public class NMCacheUploaderSCMProtocolPBServiceImpl implements + NMCacheUploaderSCMProtocolPB { + + private NMCacheUploaderSCMProtocol real; + + public NMCacheUploaderSCMProtocolPBServiceImpl(NMCacheUploaderSCMProtocol impl) { + this.real = impl; + } + + @Override + public NotifySCMResponseProto notify(RpcController controller, + NotifySCMRequestProto proto) throws ServiceException { + NotifySCMRequestPBImpl request = new NotifySCMRequestPBImpl(proto); + try { + NotifySCMResponse response = real.notify(request); + return ((NotifySCMResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMRequest.java new file mode 100644 index 0000000..b53fd10 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMRequest.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The request from clients to the SharedCacheManager that claims a + * resource in the shared cache. + *

+ */ +@Public +@Stable +public abstract class NotifySCMRequest { + + /** + * Get the filename of the resource that was just uploaded to the shared + * cache. + * + * @return the filename + */ + @Public + @Stable + public abstract String getFileName(); + + /** + * Set the filename of the resource that was just uploaded to the shared + * cache. + * + * @param filename the name of the file + */ + @Public + @Stable + public abstract void setFilename(String filename); + + /** + * Get the key of the resource that was just uploaded to the + * shared cache. + * + * @return key + */ + @Public + @Stable + public abstract String getResourceKey(); + + /** + * Set the key of the resource that was just uploaded to the + * shared cache. + * + * @param key unique identifier for the resource + */ + @Public + @Stable + public abstract void setResourceKey(String key); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMResponse.java new file mode 100644 index 0000000..c17b430 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMResponse.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The response from the SharedCacheManager to the NodeManager that indicates + * whether the NodeManager needs to delete the cached resource it was sending + * the notification for. + *

+ */ +@Public +@Stable +public abstract class NotifySCMResponse { + + /** + * Get whether or not the shared cache manager has accepted the notified + * resource (i.e. the uploaded file should remain in the cache). + * + * @return boolean True if the resource has been accepted, false otherwise. + */ + @Public + @Stable + public abstract boolean getAccepted(); + + /** + * Set whether or not the shared cache manager has accepted the notified + * resource (i.e. the uploaded file should remain in the cache). + * + * @param b True if the resource has been accepted, false otherwise. + */ + @Public + @Stable + public abstract void setAccepted(boolean b); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMRequestPBImpl.java new file mode 100644 index 0000000..6bddb96 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMRequestPBImpl.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; + +public class NotifySCMRequestPBImpl extends + NotifySCMRequest { + NotifySCMRequestProto proto = NotifySCMRequestProto + .getDefaultInstance(); + NotifySCMRequestProto.Builder builder = null; + boolean viaProto = false; + + public NotifySCMRequestPBImpl() { + builder = NotifySCMRequestProto.newBuilder(); + } + + public NotifySCMRequestPBImpl(NotifySCMRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public NotifySCMRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public String getResourceKey() { + NotifySCMRequestProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasResourceKey()) ? p.getResourceKey() : null; + } + + @Override + public void setResourceKey(String key) { + maybeInitBuilder(); + if (key == null) { + builder.clearResourceKey(); + return; + } + builder.setResourceKey(key); + } + + @Override + public String getFileName() { + NotifySCMRequestProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasFilename()) ? p.getFilename() : null; + } + + @Override + public void setFilename(String filename) { + maybeInitBuilder(); + if (filename == null) { + builder.clearFilename(); + return; + } + builder.setFilename(filename); + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = NotifySCMRequestProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMResponsePBImpl.java new file mode 100644 index 0000000..cb5f978 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMResponsePBImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; + +public class NotifySCMResponsePBImpl extends + NotifySCMResponse { + NotifySCMResponseProto proto = NotifySCMResponseProto + .getDefaultInstance(); + NotifySCMResponseProto.Builder builder = null; + boolean viaProto = false; + + public NotifySCMResponsePBImpl() { + builder = NotifySCMResponseProto.newBuilder(); + } + + public NotifySCMResponsePBImpl( +NotifySCMResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public NotifySCMResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public boolean getAccepted() { + NotifySCMResponseProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasAccepted()) ? p.getAccepted() : null; + } + + @Override + public void setAccepted(boolean b) { + maybeInitBuilder(); + builder.setAccepted(b); + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = NotifySCMResponseProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/CacheStructureUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/CacheStructureUtil.java new file mode 100644 index 0000000..cd620c3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/CacheStructureUtil.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcache; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +/** + * A utility class that contains helper methods for dealing with the internal + * shared cache structure. + * + */ +public class CacheStructureUtil { + + private static final Log LOG = LogFactory.getLog(CacheStructureUtil.class); + + public static int getCacheDepth(Configuration conf) { + int cacheDepth = + conf.getInt(YarnConfiguration.SHARED_CACHE_NESTED_LEVEL, + YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL); + + if (cacheDepth <= 0) { + LOG.warn("Specified cache depth was less than or equal to zero." + + " Using default value instead. Default: " + + YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL + + ", Specified: " + cacheDepth); + cacheDepth = YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL; + } + + return cacheDepth; + } + + public static String getCacheEntryPath(int cacheDepth, String cacheRoot, + String checksum) { + + if (cacheDepth <= 0) { + throw new IllegalArgumentException( + "The cache depth must be greater than 0. Passed value: " + cacheDepth); + } + if (checksum.length() < cacheDepth) { + throw new IllegalArgumentException("The checksum passed was too short: " + + checksum); + } + + // Build the cache entry path to the specified depth. For example, if the + // depth is 3 and the checksum is 3c4f, the path would be: + // SHARED_CACHE_ROOT/3/c/4/3c4f + StringBuilder sb = new StringBuilder(cacheRoot); + for (int i = 0; i < cacheDepth; i++) { + sb.append(Path.SEPARATOR_CHAR); + sb.append(checksum.charAt(i)); + } + sb.append(Path.SEPARATOR_CHAR).append(checksum); + + return sb.toString(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 64eb428..ab89e4a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -94,7 +94,8 @@ public int compare(ContainerId c1, } public static LocalResource newLocalResource(URL url, LocalResourceType type, - LocalResourceVisibility visibility, long size, long timestamp) { + LocalResourceVisibility visibility, long size, long timestamp, + boolean shouldBeUploadedToSharedCache) { LocalResource resource = recordFactory.newRecordInstance(LocalResource.class); resource.setResource(url); @@ -102,14 +103,16 @@ public static LocalResource newLocalResource(URL url, LocalResourceType type, resource.setVisibility(visibility); resource.setSize(size); resource.setTimestamp(timestamp); + resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return resource; } public static LocalResource newLocalResource(URI uri, LocalResourceType type, LocalResourceVisibility visibility, long size, - long timestamp) { + long timestamp, + boolean shouldBeUploadedToSharedCache) { return newLocalResource(ConverterUtils.getYarnUrlFromURI(uri), type, - visibility, size, timestamp); + visibility, size, timestamp, shouldBeUploadedToSharedCache); } public static ApplicationId newApplicationId(RecordFactory recordFactory, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/NMCacheUploader_SCM_protocol.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/NMCacheUploader_SCM_protocol.proto new file mode 100644 index 0000000..9e4c33d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/NMCacheUploader_SCM_protocol.proto @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "NMCacheUploaderSCMProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_server_common_service_protos.proto"; + +service NMCacheUploaderSCMProtocolService { + rpc notify(NotifySCMRequestProto) returns (NotifySCMResponseProto); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index 29cd64e..f8f063a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -68,4 +68,13 @@ message NMContainerStatusProto { optional string diagnostics = 5 [default = "N/A"]; optional int32 container_exit_status = 6; optional int64 creation_time = 7; -} \ No newline at end of file +} + +message NotifySCMRequestProto { + optional string resource_key = 1; + optional string filename = 2; +} + +message NotifySCMResponseProto { + optional bool accepted = 1; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 12166e0..098953b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -117,6 +117,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.NonAggregatingLogHandler; @@ -224,6 +226,13 @@ public void serviceInit(Configuration conf) throws Exception { addIfService(logHandler); dispatcher.register(LogHandlerEventType.class, logHandler); + // add the shared cache upload service (it will do nothing if the shared + // cache is disabled) + SharedCacheUploadService sharedCacheUploader = + createSharedCacheUploaderService(); + addService(sharedCacheUploader); + dispatcher.register(SharedCacheUploadEventType.class, sharedCacheUploader); + waitForContainersOnShutdownMillis = conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS) + @@ -358,6 +367,10 @@ protected ResourceLocalizationService createResourceLocalizationService( deletionContext, dirsHandler, context.getNMStateStore()); } + protected SharedCacheUploadService createSharedCacheUploaderService() { + return new SharedCacheUploadService(); + } + protected ContainersLauncher createContainersLauncher(Context context, ContainerExecutor exec) { return new ContainersLauncher(context, this.dispatcher, exec, dirsHandler, this); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index fa54ee1..0f49ffb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -59,6 +60,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStartMonitoringEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStopMonitoringEvent; @@ -104,6 +107,10 @@ new ArrayList(); private final List appRsrcs = new ArrayList(); + private final Map resourcesToBeUploaded = + new ConcurrentHashMap(); + private final Map resourcesUploadPolicies = + new ConcurrentHashMap(); // whether container has been recovered after a restart private RecoveredContainerStatus recoveredStatus = @@ -513,7 +520,7 @@ private void sendLaunchEvent() { if (recoveredStatus == RecoveredContainerStatus.LAUNCHED) { // try to recover a container that was previously launched launcherEvent = ContainersLauncherEventType.RECOVER_CONTAINER; - } + } dispatcher.getEventHandler().handle( new ContainersLauncherEvent(this, launcherEvent)); } @@ -637,6 +644,8 @@ public ContainerState transition(ContainerImpl container, container.pendingResources.put(req, links); } links.add(rsrc.getKey()); + storeSharedCacheUploadPolicies(container, req, rsrc.getValue() + .getShouldBeUploadedToSharedCache()); switch (rsrc.getValue().getVisibility()) { case PUBLIC: container.publicRsrcs.add(req); @@ -685,31 +694,76 @@ public ContainerState transition(ContainerImpl container, } } + // Store the resource's shared cache upload policies + // Given LocalResourceRequest can be shared across containers in + // LocalResourcesTrackerImpl, we preserve the upload policies here. + // In addition, it is possible for the application to create several + // "identical" LocalResources as part of + // ContainerLaunchContext.setLocalResources with different symlinks. + // There is a corner case where these "identical" local resources have + // different upload policies. For that scenario, upload policy will be set to + // true as long as there is at least one LocalResource entry with + // upload policy set to true. + private static void storeSharedCacheUploadPolicies(ContainerImpl container, + LocalResourceRequest resourceRequest, Boolean uploadPolicy) { + Boolean storedUploadPolicy = + container.resourcesUploadPolicies.get(resourceRequest); + if (storedUploadPolicy == null || (!storedUploadPolicy && uploadPolicy)) { + container.resourcesUploadPolicies.put(resourceRequest, uploadPolicy); + } + } + /** * Transition when one of the requested resources for this container * has been successfully localized. */ static class LocalizedTransition implements MultipleArcTransition { + @SuppressWarnings("unchecked") @Override public ContainerState transition(ContainerImpl container, ContainerEvent event) { ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event; - List syms = - container.pendingResources.remove(rsrcEvent.getResource()); + LocalResourceRequest resourceRequest = rsrcEvent.getResource(); + Path location = rsrcEvent.getLocation(); + List syms = container.pendingResources.remove(resourceRequest); if (null == syms) { - LOG.warn("Localized unknown resource " + rsrcEvent.getResource() + + LOG.warn("Localized unknown resource " + resourceRequest + + " for container " + container.containerId); assert false; // fail container? return ContainerState.LOCALIZING; } - container.localizedResources.put(rsrcEvent.getLocation(), syms); + container.localizedResources.put(location, syms); + + // check to see if this resource should be uploaded to the shared cache + // as well + if (shouldBeUploadedToSharedCache(container, resourceRequest)) { + container.resourcesToBeUploaded.put(resourceRequest, location); + } if (!container.pendingResources.isEmpty()) { return ContainerState.LOCALIZING; } container.sendLaunchEvent(); + + // If this is a recovered container that has already launched, skip + // uploading resources to the shared cache. We do this to avoid uploading + // the same resources multiple times. The tradeoff is that in the case of + // a recovered container, there is a chance that resources don't get + // uploaded into the shared cache. This is OK because resources are not + // acknowledged by the SCM until they have been uploaded by the node + // manager. + if (container.recoveredStatus != RecoveredContainerStatus.LAUNCHED + && container.recoveredStatus != RecoveredContainerStatus.COMPLETED) { + // kick off uploads to the shared cache + container.dispatcher.getEventHandler().handle( + new SharedCacheUploadEvent(container.resourcesToBeUploaded, container + .getLaunchContext(), container.getUser(), + SharedCacheUploadEventType.UPLOAD)); + } + container.metrics.endInitingContainer(); return ContainerState.LOCALIZED; } @@ -1018,4 +1072,13 @@ public String toString() { private boolean hasDefaultExitCode() { return (this.exitCode == ContainerExitStatus.INVALID); } + + /** + * Returns whether the specific resource should be uploaded to the shared + * cache. + */ + private static boolean shouldBeUploadedToSharedCache(ContainerImpl container, + LocalResourceRequest resource) { + return container.resourcesUploadPolicies.get(resource); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java index 70bead7..65a331a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java @@ -20,6 +20,8 @@ import java.net.URISyntaxException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -30,6 +32,8 @@ public class LocalResourceRequest extends LocalResource implements Comparable { + private static final Log LOG = LogFactory.getLog(LocalResourceRequest.class); + private final Path loc; private final long timestamp; private final LocalResourceType type; @@ -152,6 +156,17 @@ public String getPattern() { } @Override + public boolean getShouldBeUploadedToSharedCache() { + throw new UnsupportedOperationException(); + } + + @Override + public void setShouldBeUploadedToSharedCache( + boolean shouldBeUploadedToSharedCache) { + throw new UnsupportedOperationException(); + } + + @Override public void setResource(URL resource) { throw new UnsupportedOperationException(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEvent.java new file mode 100644 index 0000000..2f27770 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEvent.java @@ -0,0 +1,54 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache; + +import java.util.Map; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.event.AbstractEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest; + +public class SharedCacheUploadEvent extends + AbstractEvent { + private final Map resources; + private final ContainerLaunchContext context; + private final String user; + + public SharedCacheUploadEvent(Map resources, + ContainerLaunchContext context, String user, + SharedCacheUploadEventType eventType) { + super(eventType); + this.resources = resources; + this.context = context; + this.user = user; + } + + public Map getResources() { + return resources; + } + + public ContainerLaunchContext getContainerLaunchContext() { + return context; + } + + public String getUser() { + return user; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEventType.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEventType.java new file mode 100644 index 0000000..19280ec --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEventType.java @@ -0,0 +1,23 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache; + +public enum SharedCacheUploadEventType { + UPLOAD +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java new file mode 100644 index 0000000..d32819c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java @@ -0,0 +1,122 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Service that uploads localized files to the shared cache. The upload is + * considered not critical, and is done on a best-effort basis. Failure to + * upload is not fatal. + */ +public class SharedCacheUploadService extends AbstractService implements + EventHandler { + private static final Log LOG = + LogFactory.getLog(SharedCacheUploadService.class); + + private boolean enabled; + private FileSystem fs; + private FileSystem localFs; + private ExecutorService uploaderPool; + private NMCacheUploaderSCMProtocol scmClient; + + public SharedCacheUploadService() { + super(SharedCacheUploadService.class.getName()); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + enabled = conf.getBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, + YarnConfiguration.DEFAULT_SHARED_CACHE_ENABLED); + if (enabled) { + int threadCount = + conf.getInt(YarnConfiguration.SHARED_CACHE_NM_UPLOADER_THREAD_COUNT, + YarnConfiguration.DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT); + uploaderPool = Executors.newFixedThreadPool(threadCount, + new ThreadFactoryBuilder(). + setNameFormat("Shared cache uploader #%d"). + build()); + scmClient = createSCMClient(conf); + try { + fs = FileSystem.get(conf); + localFs = FileSystem.getLocal(conf); + } catch (IOException e) { + LOG.error("Unexpected exception in getting the filesystem", e); + throw new RuntimeException(e); + } + } + super.serviceInit(conf); + } + + private NMCacheUploaderSCMProtocol createSCMClient(Configuration conf) { + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress scmAddress = + conf.getSocketAddr(YarnConfiguration.NM_SCM_ADDRESS, + YarnConfiguration.DEFAULT_NM_SCM_ADDRESS, + YarnConfiguration.DEFAULT_NM_SCM_PORT); + return (NMCacheUploaderSCMProtocol)rpc.getProxy( + NMCacheUploaderSCMProtocol.class, scmAddress, conf); + } + + @Override + protected void serviceStop() throws Exception { + if (enabled) { + uploaderPool.shutdown(); + RPC.stopProxy(scmClient); + } + super.serviceStop(); + } + + @Override + public void handle(SharedCacheUploadEvent event) { + if (enabled) { + Map resources = event.getResources(); + for (Map.Entry e: resources.entrySet()) { + Uploader uploader = + new Uploader(e.getKey(), e.getValue(), event.getUser(), + getConfig(), scmClient, fs, localFs); + // fire off an upload task + uploaderPool.submit(uploader); + } + } + } + + public boolean isEnabled() { + return enabled; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/Uploader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/Uploader.java new file mode 100644 index 0000000..a3c1578 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/Uploader.java @@ -0,0 +1,290 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.URISyntaxException; +import java.util.Random; +import java.util.concurrent.Callable; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; +import org.apache.hadoop.yarn.server.sharedcache.CacheStructureUtil; +import org.apache.hadoop.yarn.sharedcache.Checksum; +import org.apache.hadoop.yarn.sharedcache.ChecksumFactory; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.FSDownload; + +import com.google.common.annotations.VisibleForTesting; + +/** + * The callable class that handles the actual upload. + */ +class Uploader implements Callable { + // rwxr-xr-x + static final FsPermission DIRECTORY_PERMISSION = + new FsPermission((short)00755); + // r-xr-xr-x + static final FsPermission FILE_PERMISSION = + new FsPermission((short)00555); + + private static final Log LOG = LogFactory.getLog(Uploader.class); + private static final ThreadLocal randomTl = new ThreadLocal() { + @Override + protected Random initialValue() { + return new Random(System.nanoTime()); + } + }; + + private final LocalResource resource; + private final Path localPath; + private final String user; + private final Configuration conf; + private final NMCacheUploaderSCMProtocol scmClient; + private final FileSystem fs; + private final FileSystem localFs; + private final String sharedCacheRootDir; + private final int nestedLevel; + private final Checksum checksum; + private final RecordFactory recordFactory; + + public Uploader(LocalResource resource, Path localPath, String user, + Configuration conf, NMCacheUploaderSCMProtocol scmClient) + throws IOException { + this(resource, localPath, user, conf, scmClient, + FileSystem.get(conf), localPath.getFileSystem(conf)); + } + + /** + * @param resource the local resource that contains the original remote path + * @param localPath the path in the local filesystem where the resource is + * localized + * @param fs the filesystem of the shared cache + * @param localFs the local filesystem + */ + public Uploader(LocalResource resource, Path localPath, String user, + Configuration conf, NMCacheUploaderSCMProtocol scmClient, FileSystem fs, + FileSystem localFs) { + this.resource = resource; + this.localPath = localPath; + this.user = user; + this.conf = conf; + this.scmClient = scmClient; + this.fs = fs; + this.sharedCacheRootDir = + conf.get(YarnConfiguration.SHARED_CACHE_ROOT, + YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT); + this.nestedLevel = CacheStructureUtil.getCacheDepth(conf); + this.checksum = ChecksumFactory.getChecksum(conf); + this.localFs = localFs; + this.recordFactory = RecordFactoryProvider.getRecordFactory(null); + } + + /** + * Uploads the file under the shared cache, and notifies the shared cache + * manager. If it is unable to upload the file because it already exists, it + * returns false. + */ + @Override + public Boolean call() throws Exception { + Path tempPath = null; + try { + if (!verifyAccess()) { + LOG.warn("User " + user + " is not authorized to upload file " + + localPath.getName()); + return false; + } + + // first determine the actual local path that will be used for upload + Path actualPath = getActualPath(); + // compute the checksum + String checksumVal = computeChecksum(actualPath); + // create the directory (if it doesn't exist) + Path directoryPath = + new Path(CacheStructureUtil.getCacheEntryPath(nestedLevel, + sharedCacheRootDir, checksumVal)); + // let's not check if the directory already exists: in the vast majority + // of the cases, the directory does not exist; as long as mkdirs does not + // error out if it exists, we should be fine + fs.mkdirs(directoryPath, DIRECTORY_PERMISSION); + // create the temporary file + tempPath = new Path(directoryPath, getTemporaryFileName(actualPath)); + if (!uploadFile(actualPath, tempPath)) { + LOG.warn("Could not copy the file to the shared cache at " + tempPath); + return false; + } + + // set the permission so that it is readable but not writable + // TODO should I create the file with the right permission so I save the + // permission call? + fs.setPermission(tempPath, FILE_PERMISSION); + // rename it to the final filename + Path finalPath = new Path(directoryPath, actualPath.getName()); + if (!fs.rename(tempPath, finalPath)) { + LOG.warn("The file already exists under " + finalPath + + ". Ignoring this attempt."); + deleteTempFile(tempPath); + return false; + } + + // notify the SCM + if (!notifySharedCacheManager(checksumVal, actualPath.getName())) { + // the shared cache manager rejected the upload (as it is likely + // uploaded under a different name + // clean up this file and exit + fs.delete(finalPath, false); + return false; + } + + // set the replication factor + short replication = + (short)conf.getInt(YarnConfiguration.SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR, + YarnConfiguration.DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR); + fs.setReplication(finalPath, replication); + LOG.info("File " + actualPath.getName() + + " was uploaded to the shared cache at " + finalPath); + return true; + } catch (IOException e) { + LOG.warn("Exception while uploading the file " + localPath.getName(), e); + // in case an exception is thrown, delete the temp file + deleteTempFile(tempPath); + throw e; + } + } + + @VisibleForTesting + Path getActualPath() throws IOException { + Path path = localPath; + FileStatus status = localFs.getFileStatus(path); + if (status != null && status.isDirectory()) { + // for certain types of resources that get unpacked, the original file may + // be found under the directory with the same name (see + // FSDownload.unpack); check if the path is a directory and if so look + // under it + path = new Path(path, path.getName()); + } + return path; + } + + private void deleteTempFile(Path tempPath) { + try { + if (tempPath != null && fs.exists(tempPath)) { + fs.delete(tempPath, false); + } + } catch (IOException ignore) {} + } + + /** + * Checks that the (original) remote file is either owned by the user who + * started the app or public. + */ + @VisibleForTesting + boolean verifyAccess() throws IOException { + // if it is in the public cache, it's trivially OK + if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) { + return true; + } + + final Path remotePath; + try { + remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource()); + } catch (URISyntaxException e) { + throw new IOException("Invalid resource", e); + } + + // get the file status of the HDFS file + FileSystem remoteFs = remotePath.getFileSystem(conf); + FileStatus status = remoteFs.getFileStatus(remotePath); + // check to see if the file has been modified in any way + if (status.getModificationTime() != resource.getTimestamp()) { + LOG.warn("The remote file " + remotePath + + " has changed since it's localized; will not consider it for upload"); + return false; + } + + // check for the user ownership + if (status.getOwner().equals(user)) { + return true; // the user owns the file + } + // check if the file is publicly readable otherwise + return fileIsPublic(remotePath, remoteFs, status); + } + + @VisibleForTesting + boolean fileIsPublic(final Path remotePath, FileSystem remoteFs, + FileStatus status) throws IOException { + return FSDownload.isPublic(remoteFs, remotePath, status, null); + } + + /** + * Uploads the file to the shared cache under a temporary name, and returns + * the result. + */ + @VisibleForTesting + boolean uploadFile(Path sourcePath, Path tempPath) throws IOException { + return FileUtil.copy(localFs, sourcePath, fs, tempPath, false, conf); + } + + @VisibleForTesting + String computeChecksum(Path path) throws IOException { + InputStream is = localFs.open(path); + try { + return checksum.computeChecksum(is); + } finally { + try { is.close(); } catch (IOException ignore) {} + } + } + + private String getTemporaryFileName(Path path) { + return path.getName() + "-" + randomTl.get().nextLong(); + } + + @VisibleForTesting + boolean notifySharedCacheManager(String checksumVal, String fileName) + throws IOException { + try { + NotifySCMRequest request = + recordFactory.newRecordInstance(NotifySCMRequest.class); + request.setResourceKey(checksumVal); + request.setFilename(fileName); + return scmClient.notify(request).getAccepted(); + } catch (YarnException e) { + throw new IOException(e); + } catch (UndeclaredThrowableException e) { + // retrieve the cause of the exception and throw it as an IOException + throw new IOException(e.getCause() == null ? e : e.getCause()); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index a813e98..20e3569 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -642,7 +642,7 @@ public boolean matches(Object o) { URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name); LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis, - r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L); + r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false); return new SimpleEntry(name, rsrc); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index ed59ddd..c7ae2cc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -1645,7 +1645,7 @@ private static LocalResource getMockedResource(Random r, URL url = getPath("/local/PRIVATE/" + name); LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis, - r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L); + r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false); return rsrc; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploadService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploadService.java new file mode 100644 index 0000000..1b2b2f0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploadService.java @@ -0,0 +1,50 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache; + +import static org.junit.Assert.assertSame; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; + +public class TestSharedCacheUploadService { + + @Test + public void testInitDisabled() { + testInit(false); + } + + @Test + public void testInitEnabled() { + testInit(true); + } + + public void testInit(boolean enabled) { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, enabled); + + SharedCacheUploadService service = new SharedCacheUploadService(); + service.init(conf); + assertSame(enabled, service.isEnabled()); + + service.stop(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestUploader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestUploader.java new file mode 100644 index 0000000..e9eea0e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestUploader.java @@ -0,0 +1,244 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; +import org.junit.Test; + +public class TestUploader { + + /** + * If verifyAccess fails, the upload should fail + */ + @Test + public void testFailVerifyAccess() throws Exception { + Uploader spied = createSpiedUploader(); + doReturn(false).when(spied).verifyAccess(); + + assertFalse(spied.call()); + } + + /** + * If rename fails, the upload should fail + */ + @Test + public void testRenameFail() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); + LocalResource resource = mock(LocalResource.class); + Path localPath = mock(Path.class); + when(localPath.getName()).thenReturn("foo.jar"); + String user = "joe"; + NMCacheUploaderSCMProtocol scmClient = + mock(NMCacheUploaderSCMProtocol.class); + NotifySCMResponse response = mock(NotifySCMResponse.class); + when(response.getAccepted()).thenReturn(true); + when(scmClient.notify(isA(NotifySCMRequest.class))).thenReturn(response); + FileSystem fs = mock(FileSystem.class); + // return false when rename is called + when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(false); + FileSystem localFs = FileSystem.getLocal(conf); + Uploader spied = + createSpiedUploader(resource, localPath, user, conf, scmClient, fs, + localFs); + // stub verifyAccess() to return true + doReturn(true).when(spied).verifyAccess(); + // stub getActualPath() + doReturn(localPath).when(spied).getActualPath(); + // stub computeChecksum() + doReturn("abcdef0123456789").when(spied).computeChecksum(isA(Path.class)); + // stub uploadFile() to return true + doReturn(true).when(spied).uploadFile(isA(Path.class), isA(Path.class)); + + assertFalse(spied.call()); + } + + /** + * If verifyAccess, uploadFile, rename, and notification succeed, the upload + * should succeed + */ + @Test + public void testSuccess() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); + LocalResource resource = mock(LocalResource.class); + Path localPath = mock(Path.class); + when(localPath.getName()).thenReturn("foo.jar"); + String user = "joe"; + NMCacheUploaderSCMProtocol scmClient = + mock(NMCacheUploaderSCMProtocol.class); + NotifySCMResponse response = mock(NotifySCMResponse.class); + when(response.getAccepted()).thenReturn(true); + when(scmClient.notify(isA(NotifySCMRequest.class))).thenReturn(response); + FileSystem fs = mock(FileSystem.class); + // return false when rename is called + when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(true); + FileSystem localFs = FileSystem.getLocal(conf); + Uploader spied = + createSpiedUploader(resource, localPath, user, conf, scmClient, fs, + localFs); + // stub verifyAccess() to return true + doReturn(true).when(spied).verifyAccess(); + // stub getActualPath() + doReturn(localPath).when(spied).getActualPath(); + // stub computeChecksum() + doReturn("abcdef0123456789").when(spied).computeChecksum(isA(Path.class)); + // stub uploadFile() to return true + doReturn(true).when(spied).uploadFile(isA(Path.class), isA(Path.class)); + // stub notifySharedCacheManager to return true + doReturn(true).when(spied).notifySharedCacheManager(isA(String.class), + isA(String.class)); + + assertTrue(spied.call()); + } + + /** + * If verifyAccess, uploadFile, and rename succed, but it receives a nay from + * SCM, the file should be deleted + */ + @Test + public void testNotifySCMFail() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); + LocalResource resource = mock(LocalResource.class); + Path localPath = mock(Path.class); + when(localPath.getName()).thenReturn("foo.jar"); + String user = "joe"; + FileSystem fs = mock(FileSystem.class); + // return false when rename is called + when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(true); + FileSystem localFs = FileSystem.getLocal(conf); + Uploader spied = + createSpiedUploader(resource, localPath, user, conf, null, fs, + localFs); + // stub verifyAccess() to return true + doReturn(true).when(spied).verifyAccess(); + // stub getActualPath() + doReturn(localPath).when(spied).getActualPath(); + // stub computeChecksum() + doReturn("abcdef0123456789").when(spied).computeChecksum(isA(Path.class)); + // stub uploadFile() to return true + doReturn(true).when(spied).uploadFile(isA(Path.class), isA(Path.class)); + // stub notifySharedCacheManager to return true + doReturn(false).when(spied).notifySharedCacheManager(isA(String.class), + isA(String.class)); + + assertFalse(spied.call()); + verify(fs).delete(isA(Path.class), anyBoolean()); + } + + /** + * If resource is public, verifyAccess should succeed + */ + @Test + public void testVerifyAccessPublicResource() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); + LocalResource resource = mock(LocalResource.class); + // give public visibility + when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC); + Path localPath = mock(Path.class); + when(localPath.getName()).thenReturn("foo.jar"); + String user = "joe"; + NMCacheUploaderSCMProtocol scmClient = + mock(NMCacheUploaderSCMProtocol.class); + FileSystem fs = mock(FileSystem.class); + FileSystem localFs = FileSystem.getLocal(conf); + Uploader spied = + createSpiedUploader(resource, localPath, user, conf, scmClient, fs, + localFs); + + assertTrue(spied.verifyAccess()); + } + + /** + * If the localPath does not exists, getActualPath should get to one level + * down + */ + @Test + public void testGetActualPath() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); + LocalResource resource = mock(LocalResource.class); + // give public visibility + when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC); + Path localPath = new Path("foo.jar"); + String user = "joe"; + NMCacheUploaderSCMProtocol scmClient = + mock(NMCacheUploaderSCMProtocol.class); + FileSystem fs = mock(FileSystem.class); + FileSystem localFs = mock(FileSystem.class); + // stub it to return a status that indicates a directory + FileStatus status = mock(FileStatus.class); + when(status.isDirectory()).thenReturn(true); + when(localFs.getFileStatus(localPath)).thenReturn(status); + Uploader spied = + createSpiedUploader(resource, localPath, user, conf, scmClient, fs, + localFs); + + Path actualPath = spied.getActualPath(); + assertEquals(actualPath.getName(), localPath.getName()); + assertEquals(actualPath.getParent().getName(), localPath.getName()); + } + + private Uploader createSpiedUploader() throws IOException { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); + LocalResource resource = mock(LocalResource.class); + Path localPath = mock(Path.class); + String user = "foo"; + NMCacheUploaderSCMProtocol scmClient = + mock(NMCacheUploaderSCMProtocol.class); + FileSystem fs = FileSystem.get(conf); + FileSystem localFs = FileSystem.getLocal(conf); + return createSpiedUploader(resource, localPath, user, conf, scmClient, fs, + localFs); + } + + private Uploader createSpiedUploader(LocalResource resource, Path localPath, + String user, Configuration conf, NMCacheUploaderSCMProtocol scmClient, + FileSystem fs, FileSystem localFs) + throws IOException { + Uploader uploader = new Uploader(resource, localPath, user, conf, scmClient, + fs, localFs); + return spy(uploader); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml new file mode 100644 index 0000000..0969274 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml @@ -0,0 +1,96 @@ + + + + 4.0.0 + + hadoop-yarn-server + org.apache.hadoop + 3.0.0-SNAPSHOT + + org.apache.hadoop + hadoop-yarn-server-sharedcachemanager + 3.0.0-SNAPSHOT + hadoop-yarn-server-sharedcachemanager + + + + ${project.parent.parent.basedir} + + + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + junit + junit + test + + + org.mockito + mockito-all + test + + + org.apache.hadoop + hadoop-common + test-jar + test + + + org.apache.hadoop + hadoop-yarn-server-tests + test + test-jar + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + test + test-jar + + + + + + + + + maven-jar-plugin + + + + test-jar + + test-compile + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/AppChecker.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/AppChecker.java new file mode 100644 index 0000000..f0f67ee --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/AppChecker.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + * An interface for checking whether an app is running so that the cleaner + * service may determine if it can safely remove a cached entry. + */ +public interface AppChecker { + /** + * Returns whether the app is in the active state. + * + * @return true if the app is found and is not in one of the completed states; + * false otherwise + * @throws IOException if there is an error in determining the app state + */ + boolean appIsActive(ApplicationId id) throws IOException; + + /** + * Returns the list of all active apps at the given time. + * + * @return the list of active apps, or an empty list if there is none + * @throws IOException if there is an error in obtaining the list + */ + Collection getAllActiveApps() throws IOException; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerService.java new file mode 100644 index 0000000..872fd52 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerService.java @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * The cleaner service that maintains the shared cache area, and cleans up stale + * entries on a regular basis. + */ +public class CleanerService extends AbstractService { + /** + * Priority of the cleaner shutdown hook. + */ + public static final int SHUTDOWN_HOOK_PRIORITY = 30; + /** + * The name of the global cleaner lock that the cleaner creates to indicate + * that a cleaning process is in progress. + */ + public static final String GLOBAL_CLEANER_PID = ".cleaner_pid"; + + private static final Log LOG = LogFactory.getLog(CleanerService.class); + + private Configuration conf; + private final AppChecker appChecker; + private CleanerMetrics metrics; + private ScheduledExecutorService scheduler; + private final SCMStore store; + private final SCMContext context; + private final AtomicBoolean cleanerTaskRunning; + + public CleanerService(AppChecker appChecker, SCMStore store, + SCMContext context) { + super("CleanerService"); + this.appChecker = appChecker; + this.store = store; + this.context = context; + this.cleanerTaskRunning = new AtomicBoolean(); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.conf = conf; + // create a single threaded scheduler service that services the cleaner task + ThreadFactory tf = + new ThreadFactoryBuilder().setNameFormat("Shared cache cleaner").build(); + scheduler = Executors.newSingleThreadScheduledExecutor(tf); + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + if (!writeGlobalCleanerPidFile()) { + throw new YarnException("The global cleaner pid file already exists!"); + } + + long initialDelayInSeconds; + long periodInSeconds; + + this.metrics = CleanerMetrics.initSingleton(conf); + + int initialDelayInMinutes = + conf.getInt(YarnConfiguration.SCM_CLEANER_CLEANING_INITIAL_DELAY_MINUTES, + YarnConfiguration.DEFAULT_SCM_CLEANER_CLEANING_INITIAL_DELAY_MINUTES); + // negative value is invalid; use the default + if (initialDelayInMinutes < 0) { + LOG.warn("Negative initial delay value: " + initialDelayInMinutes + + ". The default will be used instead."); + initialDelayInMinutes = + YarnConfiguration.DEFAULT_SCM_CLEANER_CLEANING_INITIAL_DELAY_MINUTES; + } + initialDelayInSeconds = TimeUnit.MINUTES.toSeconds(initialDelayInMinutes); + int periodInMinutes = + conf.getInt(YarnConfiguration.SCM_CLEANER_CLEANING_PERIOD_MINUTES, + YarnConfiguration.DEFAULT_SCM_CLEANER_CLEANING_PERIOD_MINUTES); + // non-positive value is invalid; use the default + if (periodInMinutes <= 0) { + LOG.warn("Non-positive period value: " + periodInMinutes + + ". The default will be used instead."); + periodInMinutes = YarnConfiguration.DEFAULT_SCM_CLEANER_CLEANING_PERIOD_MINUTES; + } + periodInSeconds = TimeUnit.MINUTES.toSeconds(periodInMinutes); + + Runnable task = CleanerTask.create(conf, appChecker, store, + context, metrics, cleanerTaskRunning, true); + scheduler.scheduleAtFixedRate(task, initialDelayInSeconds, periodInSeconds, + TimeUnit.SECONDS); + LOG.info("Scheduled the shared cache cleaner task to run every " + + periodInSeconds + " seconds."); + + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + LOG.info("Shutting down the background thread."); + scheduler.shutdownNow(); + try { + if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) { + LOG.warn("Gave up waiting for the cleaner task to shutdown."); + } + } catch (InterruptedException e) { + LOG.warn("The cleaner service was interrupted while shutting down the task.", + e); + } + LOG.info("The background thread stopped."); + + removeGlobalCleanerPidFile(); + + super.serviceStop(); + } + + /** + * If no other cleaner task is running, execute an on-demand cleaner task. + * + * @return true if the cleaner task was started, false if there was already a + * cleaner task running. + */ + protected boolean runCleanerTask() { + + if (!this.cleanerTaskRunning.compareAndSet(false, true)) { + LOG.warn("A cleaner task is already running. " + + "A new on-demand cleaner task will not be submitted."); + return false; + } + + Runnable task = + CleanerTask.create(conf, appChecker, store, context, metrics, + cleanerTaskRunning, false); + // this is a non-blocking call (it simply submits the task to the executor + // queue and returns) + this.scheduler.execute(task); + /* + * We return true if the task is accepted for execution by the executor. Two + * notes: 1. There is a small race here between a scheduled task and an + * on-demand task. If the scheduled task happens to start after we check/set + * cleanerTaskRunning, but before we call execute, we will get two tasks + * that run back to back. Luckily, since we have already set + * cleanerTaskRunning, the scheduled task will do nothing and the on-demand + * task will clean. 2. We do not need to set the cleanerTaskRunning boolean + * back to false because this will be done in the task itself. + */ + return true; + } + + /** + * To ensure there are not multiple instances of the SCM running on a given + * cluster, a global pid file is used. This file contains the hostname of the + * machine that owns the pid file. + * + * @return true if the pid file was written, false otherwise + * @throws IOException + */ + private boolean writeGlobalCleanerPidFile() throws YarnException { + String root = + conf.get(YarnConfiguration.SHARED_CACHE_ROOT, + YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT); + Path pidPath = new Path(root, GLOBAL_CLEANER_PID); + try { + FileSystem fs = FileSystem.get(this.conf); + + if (fs.exists(pidPath)) { + return false; + } + + FSDataOutputStream os = fs.create(pidPath, false); + // write the hostname and the process id in the global cleaner pid file + final String ID = ManagementFactory.getRuntimeMXBean().getName(); + os.writeUTF(ID); + os.close(); + // add it to the delete-on-exit to ensure it gets deleted when the JVM + // exits + fs.deleteOnExit(pidPath); + } catch (IOException e) { + throw new YarnException(e); + } + LOG.info("Created the global cleaner pid file at " + pidPath.toString()); + return true; + } + + private void removeGlobalCleanerPidFile() { + try { + FileSystem fs = FileSystem.get(this.conf); + String root = + conf.get(YarnConfiguration.SHARED_CACHE_ROOT, + YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT); + + Path pidPath = new Path(root, GLOBAL_CLEANER_PID); + + + fs.delete(pidPath, false); + LOG.info("Removed the global cleaner pid file at " + pidPath.toString()); + } catch (IOException e) { + LOG.error( + "Unable to remove the global cleaner pid file! The file may need " + + "to be removed manually.", e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerTask.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerTask.java new file mode 100644 index 0000000..7704dda --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerTask.java @@ -0,0 +1,424 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.sharedcache.CacheStructureUtil; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.ResourceReference; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; + +import com.google.common.annotations.VisibleForTesting; + +/** + * The task that runs and cleans up the shared cache area for stale entries and + * orphaned files. It is expected that only one cleaner task runs at any given + * point in time. + */ +class CleanerTask implements Runnable { + public static final String RENAMED_SUFFIX = "-renamed"; + private static final Log LOG = LogFactory.getLog(CleanerTask.class); + + private final String location; + private final int stalenessMinutes; + private final long sleepTime; + private final int nestedLevel; + private final Path root; + private final FileSystem fs; + private final AppChecker appChecker; + private final SCMStore store; + private final SCMContext context; + private final CleanerMetrics metrics; + private final AtomicBoolean cleanerTaskIsRunning; + private final boolean isScheduledTask; + + /** + * Creates a cleaner task based on the configuration. This is provided for + * convenience. + * + * @param conf + * @param appChecker + * @param store + * @param context + * @param metrics + * @param cleanerTaskRunning true if there is another cleaner task currently + * running + * @param isScheduledTask true if the task is a scheduled task + */ + public static CleanerTask create(Configuration conf, AppChecker appChecker, + SCMStore store, SCMContext context, CleanerMetrics metrics, + AtomicBoolean cleanerTaskRunning, boolean isScheduledTask) { + try { + // get the root directory for the shared cache + String location = + conf.get(YarnConfiguration.SHARED_CACHE_ROOT, + YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT); + int stalenessMinutes = + conf.getInt(YarnConfiguration.SCM_CLEANER_STALENESS_MINUTES, + YarnConfiguration.DEFAULT_SCM_CLEANER_STALENESS_MINUTES); + // non-positive value is invalid; use the default + if (stalenessMinutes <= 0) { + LOG.warn("Non-positive staleness value: " + stalenessMinutes + + ". The default will be used instead."); + stalenessMinutes = YarnConfiguration.DEFAULT_SCM_CLEANER_STALENESS_MINUTES; + } + long sleepTime = + conf.getLong(YarnConfiguration.SCM_CLEANER_CLEANING_SLEEP_BETWEEN_CLEAN_MS, + YarnConfiguration.DEFAULT_SCM_CLEANER_CLEANING_SLEEP_BETWEEN_CLEAN_MS); + int nestedLevel = CacheStructureUtil.getCacheDepth(conf); + FileSystem fs = FileSystem.get(conf); + + return new CleanerTask(location, stalenessMinutes, sleepTime, + nestedLevel, fs, appChecker, store, context, metrics, + cleanerTaskRunning, isScheduledTask); + } catch (IOException e) { + LOG.error("Unable to obtain the filesystem for the cleaner service", e); + throw new ExceptionInInitializerError(e); + } + } + + /** + * Creates a cleaner task based on the root directory location and the + * filesystem. + */ + CleanerTask(String location, int stalenessMinutes, long sleepTime, + int nestedLevel, FileSystem fs, AppChecker appChecker, SCMStore store, + SCMContext context, CleanerMetrics metrics, + AtomicBoolean cleanerTaskIsRunning, boolean isScheduledTask) { + this.location = location; + this.stalenessMinutes = stalenessMinutes; + this.sleepTime = sleepTime; + this.nestedLevel = nestedLevel; + this.root = new Path(location); + this.fs = fs; + this.store = store; + this.context = context; + this.appChecker = appChecker; + this.metrics = metrics; + this.cleanerTaskIsRunning = cleanerTaskIsRunning; + this.isScheduledTask = isScheduledTask; + } + + public void run() { + // check if it is a scheduled task + if (isScheduledTask + && !this.cleanerTaskIsRunning.compareAndSet(false, true)) { + // this is a scheduled task and there is already another task running + LOG.warn("A cleaner task is already running. " + + "This scheduled cleaner task will do nothing."); + return; + } + + try { + if (!fs.exists(root)) { + LOG.error("The shared cache root " + location + " was not found. " + + "The cleaner task will do nothing."); + return; + } + + // if we are using the in-memory store, we need to ensure the initial list + // of active apps is exhausted before the cleaner task can really process + // the entries because we do not remember whether any of the active apps + // may have been using the shared cache entries this check is needed + // specifically for the default in-memory store + if (store instanceof InMemorySCMStore && !processInitialActiveApps()) { + return; + } + + // we're now ready to process the shared cache area + process(); + } catch (IOException e) { + LOG.error("Unexpected exception while initializing the cleaner task. " + + "This task will do nothing,", e); + } finally { + // this is set to false regardless of if it is a scheduled or on-demand + // task + this.cleanerTaskIsRunning.set(false); + } + } + + /** + * Goes through the list of the apps, and removes apps that are no longer + * active. + * + * @return true if the list is null or empty; false otherwise + */ + @VisibleForTesting + boolean processInitialActiveApps() { + Collection activeApps = context.getInitialActiveApps(); + if (activeApps == null || activeApps.isEmpty()) { + // we're fine; there are no active apps that were running at the time of + // the service start + return true; + } + LOG.info("Looking into " + activeApps.size() + + " apps to see if they are still active"); + Iterator it = activeApps.iterator(); + while (it.hasNext()) { + ApplicationId id = it.next(); + try { + if (!appChecker.appIsActive(id)) { + // remove it from the list + it.remove(); + } + } catch (IOException e) { + LOG.warn("Exception while checking the app status; will leave the entry in", + e); + // continue + } + } + LOG.info("There are now " + activeApps.size() + " entries in the list"); + return activeApps.isEmpty(); + } + + /** + * Sweeps and processes the shared cache area to clean up stale and orphaned + * files. + */ + void process() { + // mark the beginning of the run in the metrics + metrics.reportCleaningStart(); + try { + // now traverse individual directories and process them + // the directory structure is specified by the nested level parameter + // (e.g. 9/c/d/) + StringBuilder pattern = new StringBuilder(); + for (int i = 0; i < nestedLevel; i++) { + pattern.append("*/"); + } + pattern.append("*"); + FileStatus[] entries = fs.globStatus(new Path(root, pattern.toString())); + int numEntries = entries == null ? 0 : entries.length; + LOG.info("Processing " + numEntries + " entries"); + long beginNano = System.nanoTime(); + if (entries != null) { + for (FileStatus entry: entries) { + // check for interruption so it can abort in a timely manner in case + // of shutdown + if (Thread.currentThread().isInterrupted()) { + LOG.warn("The cleaner task was interrupted. Aborting."); + break; + } + + if (entry.isDirectory()) { + processSingleEntry(entry); + } else { + LOG.warn("Invalid file at path " + entry.getPath().toString() + + " when a directory was expected"); + } + // add sleep time between cleaning each directory if it is non-zero + if (sleepTime > 0) { + Thread.sleep(sleepTime); + } + } + } + long endNano = System.nanoTime(); + long durationMs = TimeUnit.NANOSECONDS.toMillis(endNano - beginNano); + LOG.info("Processed " + numEntries + " entries in " + durationMs + + " ms."); + } catch (IOException e1) { + LOG.error("Unable to complete the cleaner task", e1); + } catch (InterruptedException e2) { + Thread.currentThread().interrupt(); // restore the interrupt + } + } + + /** + * Returns a path for the root directory for the shared cache. + */ + Path getRootPath() { + return root; + } + + /** + * Processes a single shared cache entry directory. + */ + void processSingleEntry(FileStatus entry) { + Path path = entry.getPath(); + // indicates the processing status of the entry + TaskStatus taskStatus = TaskStatus.INIT; + + // first, if the path ends with the renamed suffix, it indicates the + // directory was moved (as stale) but somehow not deleted (probably due to + // SCM failure); delete the directory + if (path.toString().endsWith(RENAMED_SUFFIX)) { + LOG.info("Found a renamed directory that was left undeleted at " + + path.toString() + ". Deleting."); + try { + if (fs.delete(path, true)) { + taskStatus = TaskStatus.DELETED; + } + } catch (IOException e) { + LOG.error("Error while processing an entry: " + path, e); + } + } else { + // this is the path to the cache entry directory + // the directory name is the cache entry key (i.e. checksum) + String key = path.getName(); + + try { + cleanDeadAppIds(key); + } catch (IOException e) { + LOG.error("Exception thrown while removing dead appIds.", e); + } + + if (entryIsStale(key, entry)) { + try { + /* + * As a side note: store.removeKey(key) and + * removeEntryFromCacheFileSystem(path) do not have to be in a + * critical section. We will never receive a notify for this entry + * before the delete from the FS has happened because the rename on + * the node manager will fail. If the node manager uploads the file + * after it is deleted form the FS, we are ok and the key will simply + * get added back to the scm once a notification is received. + */ + // remove the entry from scm + if (store.removeKey(key)) { + // remove the entry from the file system + boolean deleted = removeEntryFromCacheFileSystem(path); + if (deleted) { + taskStatus = TaskStatus.DELETED; + } else { + LOG.error("Failed to remove path from the file system." + + " Skipping this entry: " + path); + taskStatus = TaskStatus.ERROR; + } + } else { + // we did not delete the entry because it contained application ids + taskStatus = TaskStatus.PROCESSED; + } + } catch (IOException e) { + LOG.error( + "Failed to remove path from the file system. Skipping this entry: " + + path, e); + taskStatus = TaskStatus.ERROR; + } + } else { + taskStatus = TaskStatus.PROCESSED; + } + } + + // record the processing + switch (taskStatus) { + case DELETED: + metrics.reportAFileDelete(); + break; + case PROCESSED: + metrics.reportAFileProcess(); + break; + case ERROR: + // TODO add metric for reporting errors in processing an entry + break; + default: + LOG.error( + "Cleaner task ended with an invalid task status: " + taskStatus); + } + } + + boolean entryIsStale(String key, FileStatus entry) { + long staleTime = + System.currentTimeMillis() + - TimeUnit.MINUTES.toMillis(stalenessMinutes); + long accessTime = store.getAccessTime(key); + if (accessTime == -1) { + // check modification time + long modTime = entry.getModificationTime(); + // if modification time is older then SCM startup time, we need to just + // use SCM startup time as the last point of certainty + long lastUse = + modTime < context.getStartTime() ? context.getStartTime() : modTime; + return lastUse < staleTime; + } else { + // check access time + return accessTime < staleTime; + } + } + + private boolean removeEntryFromCacheFileSystem(Path path) throws IOException { + // rename the directory to make the delete atomic + Path renamedPath = new Path(path.toString() + RENAMED_SUFFIX); + if (fs.rename(path, renamedPath)) { + // the directory can be removed safely now + // log the original path + LOG.info("Deleting " + path.toString()); + return fs.delete(renamedPath, true); + } else { + // we were unable to remove it for some reason: it's best to leave + // it at that + LOG.error("We were not able to rename the directory to " + + renamedPath.toString() + ". We will leave it intact."); + } + return false; + } + + /** + * Delete all appIds for apps that are not in an end state (regardless of + * whether the entry is stale or not). If key or appId does not exist in scm + * do nothing. + * + * @param key checksum of entry + * @throws IOException + */ + private void cleanDeadAppIds(String key) throws IOException { + Collection refs = store.getResourceReferences(key); + if (refs != null) { + Set refsToRemove = new HashSet(); + for (ResourceReference r : refs) { + if (!appChecker.appIsActive(r.getAppId())) { + // app is dead + refsToRemove.add(r); + } + } + if (refsToRemove.size() > 0) { + store.removeResourceRefs(key, refsToRemove, false); + } + } + } + + /** + * A status indicating what happened with the processing of a given cache + * entry. + */ + private enum TaskStatus { + INIT, + /** Entry was successfully processed, but not deleted **/ + PROCESSED, + /** Entry was successfully deleted **/ + DELETED, + /** The cleaner task ran into an error while processing the entry **/ + ERROR; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/ClientProtocolService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/ClientProtocolService.java new file mode 100644 index 0000000..cb5b111 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/ClientProtocolService.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.ClientSCMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.sharedcache.CacheStructureUtil; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.ResourceReference; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; + +/** + * This service handles all rpc calls from the client to the shared cache + * manager. + */ +public class ClientProtocolService extends AbstractService implements + ClientSCMProtocol { + + private static final Log LOG = LogFactory.getLog(ClientProtocolService.class); + + private final RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + + private Server server; + InetSocketAddress clientBindAddress; + private final SCMStore store; + private int cacheDepth; + private String cacheRoot; + private ClientSCMMetrics metrics; + + public ClientProtocolService(SCMStore store) { + super(ClientProtocolService.class.getName()); + this.store = store; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.clientBindAddress = getBindAddress(conf); + + this.cacheDepth = CacheStructureUtil.getCacheDepth(conf); + + this.cacheRoot = + conf.get(YarnConfiguration.SHARED_CACHE_ROOT, + YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT); + + super.serviceInit(conf); + } + + InetSocketAddress getBindAddress(Configuration conf) { + return conf.getSocketAddr(YarnConfiguration.SCM_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADDRESS, + YarnConfiguration.DEFAULT_SCM_PORT); + } + + @Override + protected void serviceStart() throws Exception { + Configuration conf = getConfig(); + this.metrics = ClientSCMMetrics.initSingleton(conf); + + YarnRPC rpc = YarnRPC.create(conf); + this.server = + rpc.getServer(ClientSCMProtocol.class, this, + clientBindAddress, + conf, null, // Secret manager null for now (security not supported) + conf.getInt(YarnConfiguration.SCM_CLIENT_THREAD_COUNT, + YarnConfiguration.DEFAULT_SCM_CLIENT_THREAD_COUNT)); + + // TODO: Enable service authorization + + this.server.start(); + clientBindAddress = + conf.updateConnectAddr(YarnConfiguration.SCM_ADDRESS, + server.getListenerAddress()); + + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.server != null) { + this.server.stop(); + } + + super.serviceStop(); + } + + @Override + public UseSharedCacheResourceResponse use( + UseSharedCacheResourceRequest request) throws YarnException, + IOException { + + UseSharedCacheResourceResponse response = + recordFactory.newRecordInstance(UseSharedCacheResourceResponse.class); + + UserGroupInformation callerUGI; + try { + callerUGI = UserGroupInformation.getCurrentUser(); + } catch (IOException ie) { + LOG.info("Error getting UGI ", ie); + throw RPCUtil.getRemoteException(ie); + } + + String fileName = + this.store.addResourceReference(request.getResourceKey(), + request.getAppId(), callerUGI.getShortUserName()); + + if (fileName != null) { + response + .setPath(getCacheEntryFilePath(request.getResourceKey(), fileName)); + this.metrics.incCacheHitCount(); + } else { + this.metrics.incCacheMissCount(); + } + + return response; + } + + @Override + public ReleaseSharedCacheResourceResponse release( + ReleaseSharedCacheResourceRequest request) throws YarnException, + IOException { + + ReleaseSharedCacheResourceResponse response = + recordFactory + .newRecordInstance(ReleaseSharedCacheResourceResponse.class); + + UserGroupInformation callerUGI; + try { + callerUGI = UserGroupInformation.getCurrentUser(); + } catch (IOException ie) { + LOG.info("Error getting UGI ", ie); + throw RPCUtil.getRemoteException(ie); + } + + boolean removed = + this.store.removeResourceReference( + request.getResourceKey(), + new ResourceReference(request.getAppId(), callerUGI + .getShortUserName()), true); + + if (removed) { + this.metrics.incCacheRelease(); + } + + return response; + } + + private String getCacheEntryFilePath(String checksum, String filename) { + return CacheStructureUtil.getCacheEntryPath(this.cacheDepth, + this.cacheRoot, checksum) + Path.SEPARATOR_CHAR + filename; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/NMCacheUploaderSCMProtocolService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/NMCacheUploaderSCMProtocolService.java new file mode 100644 index 0000000..0d17136 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/NMCacheUploaderSCMProtocolService.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.NMCacheUploaderSCMProtocolMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; + +/** + * This service handles all rpc calls from the NodeManager uploader to the + * shared cache manager. + */ +public class NMCacheUploaderSCMProtocolService extends AbstractService implements + NMCacheUploaderSCMProtocol { + + private static final Log LOG = LogFactory.getLog(NMCacheUploaderSCMProtocolService.class); + + private final RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + + private Server server; + InetSocketAddress bindAddress; + private final SCMStore store; + private NMCacheUploaderSCMProtocolMetrics metrics; + + public NMCacheUploaderSCMProtocolService(SCMStore store) { + super(NMCacheUploaderSCMProtocolService.class.getName()); + this.store = store; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.bindAddress = getBindAddress(conf); + + super.serviceInit(conf); + } + + InetSocketAddress getBindAddress(Configuration conf) { + return conf.getSocketAddr(YarnConfiguration.NM_SCM_ADDRESS, + YarnConfiguration.DEFAULT_NM_SCM_ADDRESS, + YarnConfiguration.DEFAULT_NM_SCM_PORT); + } + + @Override + protected void serviceStart() throws Exception { + Configuration conf = getConfig(); + this.metrics = NMCacheUploaderSCMProtocolMetrics.initSingleton(conf); + + YarnRPC rpc = YarnRPC.create(conf); + this.server = + rpc.getServer(NMCacheUploaderSCMProtocol.class, this, bindAddress, + conf, null, // Secret manager null for now (security not supported) + conf.getInt(YarnConfiguration.SCM_NM_THREAD_COUNT, + YarnConfiguration.DEFAULT_SCM_NM_THREAD_COUNT)); + + // TODO: Enable service authorization + + this.server.start(); + bindAddress = + conf.updateConnectAddr(YarnConfiguration.NM_SCM_ADDRESS, + server.getListenerAddress()); + + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.server != null) { + this.server.stop(); + } + + super.serviceStop(); + } + + @Override + public NotifySCMResponse notify(NotifySCMRequest request) + throws YarnException, IOException { + NotifySCMResponse response = recordFactory.newRecordInstance(NotifySCMResponse.class); + + // TODO: Security/authorization + + String filename = store.addKey(request.getResourceKey(), request.getFileName()); + + boolean accepted = filename.equals(request.getFileName()); + + if (accepted) { + this.metrics.incAcceptedUploads(); + } else { + this.metrics.incRejectedUploads(); + } + + response.setAccepted(accepted); + + return response; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/RemoteAppChecker.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/RemoteAppChecker.java new file mode 100644 index 0000000..937fcc7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/RemoteAppChecker.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; + +/** + * An implementation of AppChecker that queries the resource manager remotely to + * determine whether the app is running. + */ +class RemoteAppChecker implements AppChecker { + private static final Log LOG = LogFactory.getLog(RemoteAppChecker.class); + private static final EnumSet ACTIVE_STATES = + EnumSet.complementOf(EnumSet.of(YarnApplicationState.FINISHED, + YarnApplicationState.FAILED, + YarnApplicationState.KILLED)); + + private final ApplicationClientProtocol applicationsManager; + + /** + * Creates an instance of RemoteAppChecker based on the configuration. + */ + public static AppChecker create(Configuration conf) { + // create the RM proxy based on the configuration + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = + conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_PORT); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ApplicationClientProtocol applicationsManager = + (ApplicationClientProtocol)rpc.getProxy( + ApplicationClientProtocol.class, rmAddress, conf); + LOG.info("Connected to ResourceManager at " + rmAddress); + return new RemoteAppChecker(applicationsManager); + } + + RemoteAppChecker(ApplicationClientProtocol applicationsManager) { + this.applicationsManager = applicationsManager; + } + + public boolean appIsActive(ApplicationId id) throws IOException { + GetApplicationReportRequest request = + GetApplicationReportRequest.newInstance(id); + + try { + GetApplicationReportResponse response = + applicationsManager.getApplicationReport(request); + ApplicationReport report = response.getApplicationReport(); + if (report == null) { + // the app does not exist + return false; + } + + return ACTIVE_STATES.contains(report.getYarnApplicationState()); + } catch (YarnException e) { + throw new IOException(e); + } + } + + public Collection getAllActiveApps() throws IOException { + GetApplicationsRequest request = + GetApplicationsRequest.newInstance(ACTIVE_STATES); + + try { + GetApplicationsResponse response = + applicationsManager.getApplications(request); + List activeApps = new ArrayList(); + List apps = response.getApplicationList(); + for (ApplicationReport app: apps) { + activeApps.add(app.getApplicationId()); + } + return activeApps; + } catch (YarnException e) { + throw new IOException(e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java new file mode 100644 index 0000000..1838932 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.SCMAdminProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.ipc.YarnRPC; + +/** + * This service handles all SCMAdminProtocol rpc calls from administrators + * to the shared cache manager. + */ +public class SCMAdminProtocolService extends AbstractService implements + SCMAdminProtocol { + + private static final Log LOG = LogFactory.getLog(SCMAdminProtocolService.class); + + private final RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + + private Server server; + InetSocketAddress clientBindAddress; + private final CleanerService cleanerService; + private AccessControlList adminAcl; + + public SCMAdminProtocolService(CleanerService cleanerService) { + super(SCMAdminProtocolService.class.getName()); + this.cleanerService = cleanerService; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.clientBindAddress = getBindAddress(conf); + adminAcl = new AccessControlList(conf.get( + YarnConfiguration.YARN_ADMIN_ACL, + YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); + + + super.serviceInit(conf); + } + + InetSocketAddress getBindAddress(Configuration conf) { + return conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADMIN_PORT); + } + + @Override + protected void serviceStart() throws Exception { + Configuration conf = getConfig(); + YarnRPC rpc = YarnRPC.create(conf); + this.server = + rpc.getServer(SCMAdminProtocol.class, this, + clientBindAddress, + conf, null, // Secret manager null for now (security not supported) + conf.getInt(YarnConfiguration.SCM_ADMIN_CLIENT_THREAD_COUNT, + YarnConfiguration.DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT)); + + // TODO: Enable service authorization + + this.server.start(); + clientBindAddress = + conf.updateConnectAddr(YarnConfiguration.SCM_ADMIN_ADDRESS, + server.getListenerAddress()); + + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.server != null) { + this.server.stop(); + } + + super.serviceStop(); + } + + private void checkAcls(String method) throws YarnException { + UserGroupInformation user; + try { + user = UserGroupInformation.getCurrentUser(); + } catch (IOException ioe) { + LOG.warn("Couldn't get current user", ioe); + throw RPCUtil.getRemoteException(ioe); + } + + if (!adminAcl.isUserAllowed(user)) { + LOG.warn("User " + user.getShortUserName() + " doesn't have permission" + + " to call '" + method + "'"); + + throw RPCUtil.getRemoteException( + new AccessControlException("User " + user.getShortUserName() + + " doesn't have permission" + " to call '" + method + "'")); + } + LOG.info("SCM Admin: " + method + " invoked by user " + + user.getShortUserName()); + } + + @Override + public RunSharedCacheCleanerTaskResponse runCleanerTask( + RunSharedCacheCleanerTaskRequest request) throws YarnException { + checkAcls("runCleanerTask"); + RunSharedCacheCleanerTaskResponse response = + recordFactory.newRecordInstance(RunSharedCacheCleanerTaskResponse.class); + response.setAccepted(this.cleanerService.runCleanerTask()); + return response; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMContext.java new file mode 100644 index 0000000..6c4f1e5 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMContext.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.util.Collection; +import java.util.Map; + +import org.apache.hadoop.yarn.api.records.ApplicationId; + + +/** + * A context object for the shared cache manager. + */ +public class SCMContext { + private final long startTime; + private final Map initialCachedEntries; + private final Collection initialActiveApps; + + public SCMContext() { + this(null, null); + } + + public SCMContext(Map initialCachedEntries, + Collection initialActiveApps) { + this.startTime = System.currentTimeMillis(); + this.initialCachedEntries = initialCachedEntries; + this.initialActiveApps = initialActiveApps; + } + + /** + * Returns the start time when the SCM context was created. + */ + public long getStartTime() { + return this.startTime; + } + + /** + * Returns the initial cached entries that was set when the SCM context was + * created. Note that the map is not thread safe. The user of this map is + * expected to provide thread safety or use it in a single-threaded manner. + *
+ * The user may remove entries from this map once it is consumed fully. + */ + public Map getInitialCachedEntries() { + return initialCachedEntries; + } + + /** + * Returns the initial active apps that was set when the SCM context was + * created. Note that this list is not thread safe. The user of this list is + * expected to provide thread safety or use it in a single-threaded manner. + *
+ * The user may remove entries from this list as apps are identified as no + * longer active. + */ + public Collection getInitialActiveApps() { + return initialActiveApps; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java new file mode 100644 index 0000000..6e5b1e6 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java @@ -0,0 +1,271 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.source.JvmMetrics; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.sharedcache.CacheStructureUtil; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; +import org.apache.hadoop.yarn.server.sharedcachemanager.webapp.SCMWebServer; + +import com.google.common.annotations.VisibleForTesting; + +/** + * This service maintains the shared cache meta data. It handles claiming and + * releasing of resources, all rpc calls from the client to the shared cache + * manager, and administrative commands. It also persists the shared cache meta + * data to a backend store, and cleans up stale entries on a regular basis. + */ +public class SharedCacheManager extends CompositeService { + /** + * Priority of the SharedCacheManager shutdown hook. + */ + public static final int SHUTDOWN_HOOK_PRIORITY = 30; + + private static final Log LOG = LogFactory.getLog(SharedCacheManager.class); + + private Configuration conf; + private SCMStore store; + final SCMWebServer webServer; + + public SharedCacheManager() { + super("SharedCacheManager"); + this.webServer = new SCMWebServer(); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.conf = conf; + + AppChecker appChecker = RemoteAppChecker.create(conf); + + try { + SCMContext context = createSCMContext(appChecker, conf); + + this.store = createSCMStoreService(conf, context); + addService(store); + + CleanerService cs = createCleanerService(appChecker, store, context); + addService(cs); + + NMCacheUploaderSCMProtocolService nms = + createNMCacheUploaderSCMProtocolService(store); + addService(nms); + + ClientProtocolService cps = createClientProtocolService(store); + addService(cps); + + SCMAdminProtocolService saps = createSCMAdminProtocolService(cs); + addService(saps); + + } catch (IOException e) { + LOG.error("Encountered unexpected exception while initializing the shared cache manager", + e); + throw new YarnRuntimeException(e); + } + + super.serviceInit(conf); + } + + @VisibleForTesting + SCMContext createSCMContext(AppChecker appChecker, Configuration conf) + throws IOException { + // obtain the list of active apps as part of the context + LOG.info("Getting the active app list to initialize the SCM context"); + Collection activeAppList = getActiveAppList(appChecker); + LOG.info(activeAppList.size() + " apps recorded as active at this time"); + + // obtain the list of cached entries from the filesystem + LOG.info("Getting the list of all cached entries from the filesystem"); + FileSystem fs = FileSystem.get(conf); + Map initialCachedEntries = getInitialCachedEntries(fs, conf); + LOG.info(initialCachedEntries.size() + + " entries obtained from the filesystem"); + + SCMContext context = new SCMContext(initialCachedEntries, activeAppList); + return context; + } + + @VisibleForTesting + Collection getActiveAppList(AppChecker appChecker) + throws IOException { + return appChecker.getAllActiveApps(); + } + + @VisibleForTesting + Map getInitialCachedEntries(FileSystem fs, Configuration conf) + throws IOException { + // get the root directory for the shared cache + String location = + conf.get(YarnConfiguration.SHARED_CACHE_ROOT, + YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT); + Path root = new Path(location); + if (!fs.exists(root)) { + String message = "The shared cache root " + location + " was not found"; + LOG.error(message); + throw new IOException(message); + } + + int nestedLevel = CacheStructureUtil.getCacheDepth(conf); + // now traverse individual directories and process them + // the directory structure is specified by the nested level parameter + // (e.g. 9/c/d//file) + StringBuilder pattern = new StringBuilder(); + for (int i = 0; i < nestedLevel+1; i++) { + pattern.append("*/"); + } + pattern.append("*"); + + LOG.info("Querying for all individual cached entry files"); + FileStatus[] entries = fs.globStatus(new Path(root, pattern.toString())); + int numEntries = entries == null ? 0 : entries.length; + LOG.info("Found " + numEntries + " files: processing for one entity per " + + "checksum"); + + Map initialCachedEntries = new HashMap(); + if (entries != null) { + for (FileStatus entry : entries) { + Path file = entry.getPath(); + String fileName = file.getName(); + if (entry.isFile()) { + // get the parent to get the checksum + Path parent = file.getParent(); + if (parent != null) { + // the name of the immediate parent directory is the checksum + String key = parent.getName(); + // make sure we insert only one file per checksum whichever comes + // first + String mapped = initialCachedEntries.get(key); + if (mapped != null) { + LOG.warn("Key " + key + " is already mapped to file " + mapped + + "; file " + fileName + " will not be added"); + } else { + initialCachedEntries.put(key, fileName); + } + } + } + } + } + LOG.info("A total of " + initialCachedEntries.size() + + " files are now mapped"); + return initialCachedEntries; + } + + private static SCMStore createSCMStoreService(Configuration conf, + SCMContext context) { + String className = + conf.get(YarnConfiguration.SCM_STORE_IMPL, + YarnConfiguration.DEFAULT_SCM_STORE_IMPL); + SCMStore store = null; + try { + Class clazz = Class.forName(className); + Constructor cstr = clazz.getConstructor(SCMContext.class); + store = (SCMStore) cstr.newInstance(context); + } catch (Exception e) { + throw new YarnRuntimeException(e); + } + return store; + } + + private CleanerService createCleanerService(AppChecker appChecker, + SCMStore store, SCMContext context) { + return new CleanerService(appChecker, store, context); + } + + private NMCacheUploaderSCMProtocolService createNMCacheUploaderSCMProtocolService( + SCMStore store) { + return new NMCacheUploaderSCMProtocolService(store); + } + + private ClientProtocolService createClientProtocolService(SCMStore store) { + return new ClientProtocolService(store); + } + + private SCMAdminProtocolService createSCMAdminProtocolService( + CleanerService cleanerService) { + return new SCMAdminProtocolService(cleanerService); + } + + @Override + protected void serviceStart() throws Exception { + // Start metrics + DefaultMetricsSystem.initialize("SharedCacheManager"); + JvmMetrics.initSingleton("SharedCacheManager", null); + + // Start the web server + try { + webServer.start(this.conf, this); + } catch (IOException e) { + LOG.error("Cannot start the web server.", e); + } + + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + webServer.stop(); + DefaultMetricsSystem.shutdown(); + super.serviceStop(); + } + + /** + * For testing purposes only. + */ + @VisibleForTesting + SCMStore getSCMStore() { + return this.store; + } + + public static void main(String[] args) { + Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); + StringUtils.startupShutdownMessage(SharedCacheManager.class, args, LOG); + try { + Configuration conf = new YarnConfiguration(); + SharedCacheManager sharedCacheManager = new SharedCacheManager(); + ShutdownHookManager.get().addShutdownHook( + new CompositeServiceShutdownHook(sharedCacheManager), + SHUTDOWN_HOOK_PRIORITY); + sharedCacheManager.init(conf); + sharedCacheManager.start(); + } catch (Throwable t) { + LOG.fatal("Error starting SharedCacheManager", t); + System.exit(-1); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetrics.java new file mode 100644 index 0000000..166d46a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetrics.java @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.sharedcachemanager.metrics; + +import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName; +import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsAnnotations; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MetricsSourceBuilder; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; + +/** + * This class is for maintaining the various Cleaner activity statistics and + * publishing them through the metrics interfaces. + */ +@Metrics(name = "CleanerActivity", about = "Cleaner service metrics", context = "yarn") +public class CleanerMetrics { + public static final Log LOG = LogFactory.getLog(CleanerMetrics.class); + private final MetricsRegistry registry = new MetricsRegistry("cleaner"); + + enum Singleton { + INSTANCE; + + CleanerMetrics impl; + + synchronized CleanerMetrics init(Configuration conf) { + if (impl == null) { + impl = create(conf); + } + return impl; + } + } + + public static CleanerMetrics initSingleton(Configuration conf) { + return Singleton.INSTANCE.init(conf); + } + + public static CleanerMetrics getInstance() { + CleanerMetrics topMetrics = Singleton.INSTANCE.impl; + if (topMetrics == null) + throw new IllegalStateException( + "The CleanerMetics singlton instance is not initialized." + + " Have you called init first?"); + return topMetrics; + } + + @Metric("number of deleted files over all runs") + private MutableCounterLong totalDeletedFiles; + + public long getTotalDeletedFiles() { + return totalDeletedFiles.value(); + } + + private @Metric("number of deleted files in the last run") + MutableGaugeLong deletedFiles; + + public long getDeletedFiles() { + return deletedFiles.value(); + } + + private @Metric("number of processed files over all runs") + MutableCounterLong totalProcessedFiles; + + public long getTotalProcessedFiles() { + return totalProcessedFiles.value(); + } + + private @Metric("number of processed files in the last run") + MutableGaugeLong processedFiles; + + public long getProcessedFiles() { + return processedFiles.value(); + } + + private @Metric("Rate of deleting the files over all runs") + MutableGaugeInt totalDeleteRate; + + public int getTotalDeleteRate() { + return totalDeleteRate.value(); + } + + private @Metric("Rate of deleting the files over the last run") + MutableGaugeInt deleteRate; + + public int getDeleteRate() { + return deleteRate.value(); + } + + private @Metric("Rate of prcessing the files over all runs") + MutableGaugeInt totalProcessRate; + + public int getTotalProcessRate() { + return totalProcessRate.value(); + } + + private @Metric("Rate of prcessing the files over the last run") + MutableGaugeInt processRate; + + public int getProcessRate() { + return processRate.value(); + } + + CleanerMetrics() { + } + + /** + * The metric source obtained after parsing the annotations + */ + MetricsSource metricSource; + + /** + * The start of the last run of cleaner is ms + */ + private AtomicLong lastRunStart = new AtomicLong(System.currentTimeMillis()); + + /** + * The end of the last run of cleaner is ms + */ + private AtomicLong lastRunEnd = new AtomicLong(System.currentTimeMillis()); + + /** + * The sum of the durations of the last runs + */ + private AtomicLong sumOfPastPeriods = new AtomicLong(0); + + public MetricsSource getMetricSource() { + return metricSource; + } + + static CleanerMetrics create(Configuration conf) { + MetricsSystem ms = DefaultMetricsSystem.instance(); + + CleanerMetrics metricObject = new CleanerMetrics(); + MetricsSourceBuilder sb = MetricsAnnotations.newSourceBuilder(metricObject); + final MetricsSource s = sb.build(); + ms.register("cleaner", "The cleaner service of truly shared cache", s); + metricObject.metricSource = s; + return metricObject; + } + + /** + * Report a delete operation at the current system time + */ + public void reportAFileDelete() { + long time = System.currentTimeMillis(); + reportAFileDelete(time); + } + + /** + * Report a delete operation at the specified time. + * Delete implies process as well. + * @param time + */ + public void reportAFileDelete(long time) { + totalProcessedFiles.incr(); + processedFiles.incr(); + totalDeletedFiles.incr(); + deletedFiles.incr(); + updateRates(time); + lastRunEnd.set(time); + } + + /** + * Report a process operation at the current system time + */ + public void reportAFileProcess() { + long time = System.currentTimeMillis(); + reportAFileProcess(time); + } + + /** + * Report a process operation at the specified time + * + * @param time + */ + public void reportAFileProcess(long time) { + totalProcessedFiles.incr(); + processedFiles.incr(); + updateRates(time); + lastRunEnd.set(time); + } + + private void updateRates(long time) { + long startTime = lastRunStart.get(); + long lastPeriod = time - startTime; + long sumPeriods = sumOfPastPeriods.get() + lastPeriod; + float lastRunProcessRate = ((float) processedFiles.value()) / lastPeriod; + processRate.set(ratePerMsToHour(lastRunProcessRate)); + float totalProcessRateMs = ((float) totalProcessedFiles.value()) / sumPeriods; + totalProcessRate.set(ratePerMsToHour(totalProcessRateMs)); + float lastRunDeleteRate = ((float) deletedFiles.value()) / lastPeriod; + deleteRate.set(ratePerMsToHour(lastRunDeleteRate)); + float totalDeleteRateMs = ((float) totalDeletedFiles.value()) / sumPeriods; + totalDeleteRate.set(ratePerMsToHour(totalDeleteRateMs)); + } + + /** + * Report the start a new run of the cleaner at the current system time + */ + public void reportCleaningStart() { + long time = System.currentTimeMillis(); + reportCleaningStart(time); + } + + /** + * Report the start a new run of the cleaner at the specified time + * + * @param time + */ + public void reportCleaningStart(long time) { + long lastPeriod = lastRunEnd.get() - lastRunStart.get(); + if (lastPeriod < 0) { + LOG.error("No operation since last start!"); + lastPeriod = 0; + } + lastRunStart.set(time); + processedFiles.set(0); + deletedFiles.set(0); + processRate.set(0); + deleteRate.set(0); + sumOfPastPeriods.addAndGet(lastPeriod); + registry.tag(SessionId, Long.toString(time), true); + } + + static int ratePerMsToHour(float ratePerMs) { + float ratePerHour = ratePerMs * 1000 * 3600; + return (int) ratePerHour; + } + + public static boolean isCleanerMetricRecord(String name) { + return (name.startsWith("cleaner")); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetricsCollector.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetricsCollector.java new file mode 100644 index 0000000..84672e7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetricsCollector.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.metrics; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MsInfo; + +/** + * Very simple metric collector for the cleaner service + */ +public class CleanerMetricsCollector implements MetricsCollector { + + /** + * A map from reporting period to the list of reported metrics in that period + */ + Map metrics = new HashMap(); + String sessionId = null; + + @Override + public MetricsRecordBuilder addRecord(String name) { + // For the cleaner service we need to record only the metrics destined for the + // cleaner service. We hence ignore the others. + if (CleanerMetrics.isCleanerMetricRecord(name)) { + return new CleanerMetricsRecordBuilder(metrics); + } + else + return new NullMetricsRecordBuilder(); + } + + @Override + public MetricsRecordBuilder addRecord(MetricsInfo info) { + return addRecord(info.name()); + } + + public Map getMetrics() { + return metrics; + } + + public String getSessionId() { + return sessionId; + } + + /** + * A builder which ignores all the added metrics, tags, etc. + */ + class NullMetricsRecordBuilder extends MetricsRecordBuilder { + + @Override + public MetricsRecordBuilder tag(MetricsInfo info, String value) { + return this; + } + + @Override + public MetricsRecordBuilder add(MetricsTag tag) { + return this; + } + + @Override + public MetricsRecordBuilder add(AbstractMetric metric) { + return this; + } + + @Override + public MetricsRecordBuilder setContext(String value) { + return this; + } + + @Override + public MetricsRecordBuilder addCounter(MetricsInfo info, int value) { + return this; + } + + @Override + public MetricsRecordBuilder addCounter(MetricsInfo info, long value) { + return this; + } + + @Override + public MetricsRecordBuilder addGauge(MetricsInfo info, int value) { + return this; + } + + @Override + public MetricsRecordBuilder addGauge(MetricsInfo info, long value) { + return this; + } + + @Override + public MetricsRecordBuilder addGauge(MetricsInfo info, float value) { + return this; + } + + @Override + public MetricsRecordBuilder addGauge(MetricsInfo info, double value) { + return this; + } + + @Override + public MetricsCollector parent() { + return CleanerMetricsCollector.this; + } + } + + /** + * A builder which keeps track of only counters and gauges + */ + class CleanerMetricsRecordBuilder extends NullMetricsRecordBuilder { + Map metricValueMap; + + public CleanerMetricsRecordBuilder(Map metricValueMap) { + this.metricValueMap = metricValueMap; + } + + @Override + public MetricsRecordBuilder tag(MetricsInfo info, String value) { + if (MsInfo.SessionId.equals(info)) + setSessionId(value); + return this; + } + + @Override + public MetricsRecordBuilder add(MetricsTag tag) { + if (MsInfo.SessionId.equals(tag.info())) + setSessionId(tag.value()); + return this; + } + + private void setSessionId(String value) { + sessionId = value; + } + + @Override + public MetricsRecordBuilder add(AbstractMetric metric) { + metricValueMap.put(metric.name(), metric.value()); + return this; + } + + @Override + public MetricsRecordBuilder addCounter(MetricsInfo info, int value) { + metricValueMap.put(info.name(), value); + return this; + } + + @Override + public MetricsRecordBuilder addCounter(MetricsInfo info, long value) { + metricValueMap.put(info.name(),value); + return this; + } + + @Override + public MetricsRecordBuilder addGauge(MetricsInfo info, int value) { + metricValueMap.put(info.name(),value); + return this; + } + + @Override + public MetricsRecordBuilder addGauge(MetricsInfo info, long value) { + metricValueMap.put(info.name(),value); + return this; + } + + @Override + public MetricsRecordBuilder addGauge(MetricsInfo info, float value) { + metricValueMap.put(info.name(),value); + return this; + } + + @Override + public MetricsRecordBuilder addGauge(MetricsInfo info, double value) { + metricValueMap.put(info.name(),value); + return this; + } + + @Override + public MetricsCollector parent() { + return CleanerMetricsCollector.this; + } + + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/ClientSCMMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/ClientSCMMetrics.java new file mode 100644 index 0000000..5707b0c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/ClientSCMMetrics.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.sharedcachemanager.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; + +/** + * This class is for maintaining client requests metrics + * and publishing them through the metrics interfaces. + */ +@InterfaceAudience.Private +@Metrics(about="Client SCM metrics", context="yarn") +public class ClientSCMMetrics { + + static final Log LOG = LogFactory.getLog(ClientSCMMetrics.class); + final MetricsRegistry registry; + + ClientSCMMetrics() { + registry = new MetricsRegistry("clientRequests"); + LOG.debug("Initialized "+ registry); + } + + enum Singleton { + INSTANCE; + + ClientSCMMetrics impl; + + synchronized ClientSCMMetrics init(Configuration conf) { + if (impl == null) { + impl = create(); + } + return impl; + } + } + + public static ClientSCMMetrics initSingleton(Configuration conf) { + return Singleton.INSTANCE.init(conf); + } + + public static ClientSCMMetrics getInstance() { + ClientSCMMetrics topMetrics = Singleton.INSTANCE.impl; + if (topMetrics == null) + throw new IllegalStateException( + "The ClientSCMMetrics singleton instance is not initialized." + + " Have you called init first?"); + return topMetrics; + } + + static ClientSCMMetrics create() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + + ClientSCMMetrics metrics = new ClientSCMMetrics(); + ms.register("clientRequests", null, metrics); + return metrics; + } + + @Metric("Number of cache hits") MutableCounterLong cacheHits; + @Metric("Number of cache misses") MutableCounterLong cacheMisses; + @Metric("Number of cache releases") MutableCounterLong cacheReleases; + + /** + * One cache hit event + */ + public void incCacheHitCount() { + cacheHits.incr(); + } + + /** + * One cache miss event + */ + public void incCacheMissCount() { + cacheMisses.incr(); + } + + /** + * One cache release event + */ + public void incCacheRelease() { + cacheReleases.incr(); + } + + public long getCacheHits() { return cacheHits.value(); } + public long getCacheMisses() { return cacheMisses.value(); } + public long getCacheReleases() { return cacheReleases.value(); } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/NMCacheUploaderSCMProtocolMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/NMCacheUploaderSCMProtocolMetrics.java new file mode 100644 index 0000000..14c7354 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/NMCacheUploaderSCMProtocolMetrics.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.sharedcachemanager.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; + +/** + * This class is for maintaining NM uploader requests metrics + * and publishing them through the metrics interfaces. + */ +@InterfaceAudience.Private +@Metrics(about="NM cache upload metrics", context="yarn") +public class NMCacheUploaderSCMProtocolMetrics { + + static final Log LOG = LogFactory.getLog(NMCacheUploaderSCMProtocolMetrics.class); + final MetricsRegistry registry; + + NMCacheUploaderSCMProtocolMetrics() { + registry = new MetricsRegistry("NMUploadRequests"); + LOG.debug("Initialized "+ registry); + } + + enum Singleton { + INSTANCE; + + NMCacheUploaderSCMProtocolMetrics impl; + + synchronized NMCacheUploaderSCMProtocolMetrics init(Configuration conf) { + if (impl == null) { + impl = create(); + } + return impl; + } + } + + public static NMCacheUploaderSCMProtocolMetrics initSingleton(Configuration conf) { + return Singleton.INSTANCE.init(conf); + } + + public static NMCacheUploaderSCMProtocolMetrics getInstance() { + NMCacheUploaderSCMProtocolMetrics topMetrics = Singleton.INSTANCE.impl; + if (topMetrics == null) + throw new IllegalStateException( + "The NMCacheUploaderSCMProtocolMetrics singleton instance is not initialized." + + " Have you called init first?"); + return topMetrics; + } + + static NMCacheUploaderSCMProtocolMetrics create() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + + NMCacheUploaderSCMProtocolMetrics metrics = new NMCacheUploaderSCMProtocolMetrics(); + ms.register("NMUploaderRequests", null, metrics); + return metrics; + } + + @Metric("Number of accepted uploads") MutableCounterLong acceptedUploads; + @Metric("Number of rejected uploads") MutableCounterLong rejectedUploads; + + /** + * One accepted upload event + */ + public void incAcceptedUploads() { + acceptedUploads.incr(); + } + + /** + * One rejected upload event + */ + public void incRejectedUploads() { + rejectedUploads.incr(); + } + + public long getAcceptedUploads() { return acceptedUploads.value(); } + public long getRejectUploads() { return rejectedUploads.value(); } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/Entry.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/Entry.java new file mode 100644 index 0000000..f527bbc --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/Entry.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.store; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + * Class that encapsulates the cache entry. + * + * The instances are not thread safe. Any operation that uses the entry must + * use thread-safe mechanisms to ensure safe access with the only exception of + * the filename. + */ +class Entry { + private long accessTime; + private final Set refs; + private final String fileName; + + Entry(String fileName) { + this.accessTime = System.currentTimeMillis(); + this.refs = new HashSet(); + this.fileName = fileName; + } + + long getAccessTime() { + return accessTime; + } + + void updateAccessTime() { + accessTime = System.currentTimeMillis(); + } + + String getFileName() { + return this.fileName; + } + + Set getResourceReferences() { + return this.refs; + } + + boolean addReference(ResourceReference ref) { + return this.refs.add(ref); + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/InMemorySCMStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/InMemorySCMStore.java new file mode 100644 index 0000000..ec64e1b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/InMemorySCMStore.java @@ -0,0 +1,254 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.store; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.StringInterner; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.sharedcachemanager.SCMContext; + +/** + * A thread safe version of an in-memory SCM store. The thread safety is + * implemented with two key pieces: (1) at the mapping level a ConcurrentHashMap + * is used to allow concurrency to entries and their mapping, and (2) a key + * level lock is used to ensure mutual exclusion of any operation that accesses + * the entry under the same key. + *
+ * To ensure safe key-level locking, we use the original string key and intern + * it weakly using hadoop's StringInterner. It avoids the pitfalls + * of using built-in String interning. The interned strings are also weakly + * referenced, so it can be garbage collected once it is done. And there is + * little risk of keys being available for other parts of the code so they can + * be used as locks accidentally. + */ +public class InMemorySCMStore extends SCMStore { + private static final Log LOG = LogFactory.getLog(InMemorySCMStore.class); + + private final Map map = new ConcurrentHashMap(); + + public InMemorySCMStore(SCMContext context) { + super(InMemorySCMStore.class.getName(), context); + // bootstrap itself + bootstrap(context); + } + + private String intern(String key) { + return StringInterner.weakIntern(key); + } + + /** + * The in-memory store bootstraps itself from the shared cache entries that + * exist in HDFS. If that information is passed to it via + * SCMContext, it starts with that state. + */ + private void bootstrap(SCMContext context) { + Map initialCachedEntries = context.getInitialCachedEntries(); + if (initialCachedEntries != null) { + LOG.info("Bootstrapping from " + initialCachedEntries.size() + + " entries from the storage"); + Iterator> it = + initialCachedEntries.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry e = it.next(); + String key = intern(e.getKey()); + String fileName = e.getValue(); + Entry entry = new Entry(fileName); + // we don't hold the lock for this as it is done as part of the + // constructor + map.put(key, entry); + // clear out the entry to reduce the footprint + it.remove(); + } + LOG.info("Bootstrapping complete"); + } + } + + /** + * Adds the given resource to the store under the key and the filename. If the + * entry is already found, it returns the existing filename. It represents the + * state of the store at the time of this query. The entry may change or even + * be removed once this method returns. The caller should be prepared to + * handle that situation. + * + * @return the filename of the newly inserted entry or that of the existing + * entry + */ + @Override + public String addKey(String key, String fileName) { + String interned = intern(key); + synchronized (interned) { + Entry entry = map.get(interned); + if (entry == null) { + entry = new Entry(fileName); + map.put(interned, entry); + } + return entry.getFileName(); + } + } + + /** + * Adds the provided resource reference to the cache entry under the key, and + * updates the access time. If it returns a non-null value, the caller may + * safely assume that the entry will not be removed at least until the app in + * this resource reference has terminated. + * + * @return the filename associated with the cache entry, or null if the file + * is not found + */ + @Override + public String addResourceReference(String key, ApplicationId id, + String shortUserName) { + String interned = intern(key); + synchronized (interned) { + Entry entry = map.get(interned); + if (entry == null) { // it's not mapped + return null; + } + entry.addReference(new ResourceReference(id, shortUserName)); + entry.updateAccessTime(); + return entry.getFileName(); + } + } + + /** + * Returns the list of resource references currently registered under the + * cache entry. If the list is empty, it returns an empty collection. The + * returned collection is unmodifiable and a snapshot of the information at + * the time of the query. The state may change after this query returns. The + * caller should handle the situation that some or all of these resource + * references are no longer relevant. + * + * @return the collection that contains the resource references associated + * with the cache entry; or an empty collection if no resource + * references are registered under this entry + */ + @Override + public Collection getResourceReferences(String key) { + String interned = intern(key); + synchronized (interned) { + Entry entry = map.get(interned); + if (entry == null) { + return Collections.emptySet(); + } + Set refs = + new HashSet(entry.getResourceReferences()); + return Collections.unmodifiableSet(refs); + } + } + + /** + * Removes the provided resource reference from the entry. If the entry is + * removed (i.e. does not exist), nothing will be done. + */ + @Override + public boolean removeResourceReference(String key, ResourceReference ref, + boolean updateAccessTime) { + String interned = intern(key); + synchronized (interned) { + boolean removed = false; + Entry entry = map.get(interned); + if (entry != null) { + Set entryRefs = entry.getResourceReferences(); + removed = entryRefs.remove(ref); + if (updateAccessTime) { + entry.updateAccessTime(); + } + } + return removed; + } + } + + /** + * Removes the provided collection of resource references from the entry. If + * the entry is removed (i.e. does not exist), nothing will be done. + */ + @Override + public void removeResourceRefs(String key, + Collection refs, boolean updateAccessTime) { + String interned = intern(key); + synchronized (interned) { + Entry entry = map.get(interned); + if (entry != null) { + Set entryRefs = entry.getResourceReferences(); + entryRefs.removeAll(refs); + if (updateAccessTime) { + entry.updateAccessTime(); + } + } + } + } + + /** + * Removes the given key from the store. Returns true if the key is found and + * removed or if the key is not found. Returns false if it was unable to + * remove it because the resource reference list was not empty. + */ + public boolean removeKey(String key) { + String interned = intern(key); + synchronized (interned) { + Entry entry = map.get(interned); + if (entry == null) { + return true; + } + + if (!entry.getResourceReferences().isEmpty()) { + return false; + } + // no users + map.remove(interned); + return true; + } + } + + /** + * Obtains the access time for the entry of the given key. It represents the + * view of the entry at the time of the query. The value may have been updated + * at a later point. + * + * @return the access time of the entry if found; -1 if the entry is not found + */ + @Override + public long getAccessTime(String key) { + String interned = intern(key); + synchronized (interned) { + Entry entry = map.get(interned); + return entry == null ? -1 : entry.getAccessTime(); + } + } + + /** + * This is to be used only for a test purpose. It must not be used in a real + * production situation. + */ + @Override + public void clearCache() { + // TODO remove this method and rewrite the tests + // just clear it + map.clear(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/ResourceReference.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/ResourceReference.java new file mode 100644 index 0000000..c585fce --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/ResourceReference.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.store; + +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + * This is an object that represents the meta data associated with a reference + * to a resource in the shared cache. + */ +public class ResourceReference { + private final ApplicationId appId; + private final String shortUserName; + + /** + * Create a resource reference. + * + * @param appId ApplicationId that is referencing a resource. + * @param shortUserName ShortUserName of the user that created + * the reference. + */ + public ResourceReference(ApplicationId appId, String shortUserName) { + this.appId = appId; + this.shortUserName = shortUserName; + } + + public ApplicationId getAppId() { + return this.appId; + } + + public String getShortUserName() { + return this.shortUserName; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((appId == null) ? 0 : appId.hashCode()); + result = + prime * result + + ((shortUserName == null) ? 0 : shortUserName.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ResourceReference other = (ResourceReference) obj; + if (appId == null) { + if (other.appId != null) + return false; + } else if (!appId.equals(other.appId)) + return false; + if (shortUserName == null) { + if (other.shortUserName != null) + return false; + } else if (!shortUserName.equals(other.shortUserName)) + return false; + return true; + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStore.java new file mode 100644 index 0000000..c8995ea --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStore.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.store; + +import java.util.Collection; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.sharedcachemanager.SCMContext; + + +/** + * An abstract class for the data store used by the shared cache manager + * service. All implementations of methods in this interface need to be thread + * safe and atomic. + *
+ * All implementations of SCMStore must provide a constructor that + * takes SCMContext as a sole argument. + */ +@Private +@Unstable +public abstract class SCMStore extends AbstractService { + private final SCMContext context; + + protected SCMStore(String name, SCMContext context) { + super(name); + this.context = context; + } + + /** + * Add a key to the shared cache, along with the associated filename. If the + * key already exists no action is taken. If the key did not exist, the key is + * added and the access time is set. + * + * @param key checksum of the cache entry + * @param fileName string of the actual file + * @return the file name of the file in the cache entry + */ + public abstract String addKey(String key, String fileName); + + + /** + * Remove an entry from the cache. + * + * @param key checksum of the cache entry + * @return true if the entry was removed or did not exist, false if the entry + * contained ApplicationIds and was not removed + */ + public abstract boolean removeKey(String key); + + /** + * Add a ResourceReference to a cache entry and update the + * entry's access time. + * + * @param key checksum of the cache entry + * @param id ApplicationId to add + * @param shortUsername the short username of the user responsible for adding + * the resource reference + * @return String name of the file corresponding to the cache entry if the id + * was added or already existed, null if the cache entry did not exist + */ + public abstract String addResourceReference(String key, ApplicationId id, + String shortUsername); + + /** + * Get all the ApplicationIds associated with the cache entry + * + * @param key checksum of the cache entry + * @return an unmodifiable collection of ResourceReferences. If + * the key does not exist, an empty set is returned. + */ + public abstract Collection getResourceReferences(String key); + + /** + * Get the access time for a cache entry. + * + * @param key checksum of the cache entry + * @return the access time in milliseconds of the cache entry, -1 if the entry + * does not exist + */ + public abstract long getAccessTime(String key); + + /** + * Remove a ResourceReference from a cache entry. + * + * @param key checksum of the cache entry + * @param ref the ResourceReference to remove + * @param updateAccessTime true if the call should update the access time for + * the entry + * @return true if the entry was removed, false otherwise + */ + public abstract boolean removeResourceReference(String key, + ResourceReference ref, boolean updateAccessTime); + + /** + * Remove a collection of ResourceReferences from a cache entry. + * + * @param key checksum of the cache entry + * @param refs the collection of ResourceReferences to remove + * @param updateAccessTime true if the call should update the access time for + * the entry + */ + public abstract void removeResourceRefs(String key, + Collection refs, boolean updateAccessTime); + + /** + * Permanently removes all entries from the cache. + *
+ * This method exists solely to support some tests, and must not be used in a + * production situation. + */ + public abstract void clearCache(); + + /** + * Returns the SCMContext. + * + * @return SCMContext + */ + protected SCMContext getSCMContext() { + return context; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/CleanerPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/CleanerPage.java new file mode 100644 index 0000000..30047be --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/CleanerPage.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit; + +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Map.Entry; +import java.util.SortedMap; +import java.util.TreeMap; + + +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetricsCollector; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.BODY; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.THEAD; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; + +import com.google.common.base.Joiner; + +public class CleanerPage extends TwoColumnLayout { + + private static final String TABLE_PREFIX = "table"; + + static class MetricTable extends HashMap { + private static final long serialVersionUID = -6664614528433518964L; + + public int rowCount() { + int rowCount = 0; + for (MetricColumn col : values()) + rowCount = Math.max(rowCount, col.size()); + return rowCount; + } + } + + static class MetricColumn extends LinkedList> { + private static final long serialVersionUID = 1163979877649143564L; + } + + /** + * Improvisation. TODO: send the CleanerList pointer as a constructor + * parameter to other classes + */ + static SortedMap cleanerTables = + new TreeMap(new Comparator() { + //reverse the order to have more recent tables on front (Long is Date in sec) + @Override + public int compare(Long arg0, Long arg1) { + return -1 * arg0.compareTo(arg1); + } + }); + + @Override + protected void preHead(Page.HTML<_> html) { + loadCleanerLists(); + commonPreHead(html); + setTitle("Cleaner Summary " + new Date()); + String tableNames = TABLE_PREFIX + Joiner.on(" " + TABLE_PREFIX).join(cleanerTables.keySet()); + set(DATATABLES_ID, tableNames); + set(initID(DATATABLES, tableNames), + tableInit().append(", aoColumns:[null, {bSearchable:false}]} ") + .toString()); + for (Long tableName : cleanerTables.keySet()) + setTableStyles(html, TABLE_PREFIX + Long.toString(tableName)); + } + + private void loadCleanerLists() { + CleanerMetricsCollector collector = new CleanerMetricsCollector(); + CleanerMetrics.getInstance().getMetricSource() + .getMetrics(collector, true); + String sessionId = collector.getSessionId(); + Long sessionIdLong = sessionId == null? 0: Long.parseLong(sessionId); + MetricTable cleanerTable = parseCleanerMetrics(collector.getMetrics(), sessionIdLong); + cleanerTables.put(sessionIdLong, cleanerTable); + } + + MetricTable parseCleanerMetrics(Map metricValueMap, Long sessionId) { + MetricTable cleanerTable = new MetricTable(); + MetricColumn metricCol = new MetricColumn(); + cleanerTable.put("Cleaner Started at: " + new Date(sessionId) + " " + "metric/value", metricCol); + for (Entry metricEntry : metricValueMap.entrySet()) { + metricCol.add(metricEntry); + } + return cleanerTable; + } + + protected void commonPreHead(Page.HTML<_> html) { + set(ACCORDION_ID, "nav"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + } + + @Override + protected Class nav() { + return CleanerNavBlock.class; + } + + @Override + protected Class content() { + return CleanerContentBlock.class; + } + private static class CleanerNavBlock extends HtmlBlock { + @Override + protected void render(Block html) { + html.div("#nav").h3("Tools").ul().li().a("/conf", "Configuration")._() + .li().a("/stacks", "Thread dump")._().li().a("/logs", "Logs")._() + .li().a("/metrics", "Metrics")._()._()._(); + } + } + + private static class CleanerContentBlock extends HtmlBlock { + + @Override + protected void render(Block html) { + for (Entry entry: CleanerPage.cleanerTables.entrySet()) + renderATable(html, TABLE_PREFIX + entry.getKey(), entry.getValue()); + } + + /** + * Render a table of top list in an html table. Each table represents an + * operation. + * + * @param html + * @param tableName + * @param metricTable + */ + protected void renderATable(Block html, String tableName, + MetricTable metricTable) { + TR>>> hr = + html.body().table('#' + tableName).thead().tr(); + for (String colName : metricTable.keySet()) + hr.td()._(colName)._(); + TBODY>> tableBody = hr._()._().tbody(); + + int rowCount = metricTable.rowCount(); + for (int rowId = 0; rowId < rowCount; rowId++) { + TR>>> tr = tableBody.tr(); + for (Entry colEntry : metricTable.entrySet()) { + MetricColumn col = colEntry.getValue(); + Entry cellEntry = col.get(rowId); + String cellContent = + cellEntry.getKey() + " = " + cellEntry.getValue(); + tr.td(cellContent); + } + tr._(); + } + tableBody._()._()._(); + } + + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java new file mode 100644 index 0000000..00e1e31 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import org.apache.hadoop.yarn.webapp.Controller; +import org.apache.hadoop.yarn.webapp.YarnWebParams; + +public class SCMController extends Controller implements + YarnWebParams { + @Override + public void index() { + setTitle("Shared Cache Manager"); + } + + /** + * It is referenced in SCMWebServer.SCMWebApp.setup() + */ + @SuppressWarnings("unused") + public void cleaner() { + render(CleanerPage.class); + } + + /** + * It is referenced in SCMWebServer.SCMWebApp.setup() + */ + @SuppressWarnings("unused") + public void overview() { + render(SCMOverviewPage.class); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java new file mode 100644 index 0000000..0611d47 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java @@ -0,0 +1,45 @@ +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.NMCacheUploaderSCMProtocolMetrics; + +// This class is used to summarize useful SCM metrics for webUI display +@XmlRootElement(name = "SCMMetrics") +@XmlAccessorType(XmlAccessType.FIELD) +public class SCMMetricsInfo { + protected long totalDeletedFiles; + protected long totalProcessedFiles; + protected long cacheHits; + protected long cacheMisses; + protected long cacheReleases; + protected long acceptedUploads; + protected long rejectedUploads; + + public SCMMetricsInfo() { + } + + public SCMMetricsInfo(CleanerMetrics cleanerMetrics, + ClientSCMMetrics clientSCMMetrics, + NMCacheUploaderSCMProtocolMetrics nmCacheUploaderSCMProtocolMetrics) { + totalDeletedFiles = cleanerMetrics.getTotalDeletedFiles(); + totalProcessedFiles = cleanerMetrics.getTotalProcessedFiles(); + cacheHits = clientSCMMetrics.getCacheHits(); + cacheMisses = clientSCMMetrics.getCacheMisses(); + cacheReleases = clientSCMMetrics.getCacheReleases(); + acceptedUploads = nmCacheUploaderSCMProtocolMetrics.getAcceptedUploads(); + rejectedUploads = nmCacheUploaderSCMProtocolMetrics.getRejectUploads(); + } + + public long getTotalDeletedFiles() { return totalDeletedFiles; } + public long getTotalProcessedFiles() { return totalProcessedFiles; } + public long getCacheHits() { return cacheHits; } + public long getCacheMisses() { return cacheMisses; } + public long getCacheReleases() { return cacheReleases; } + public long getAcceptedUploads() { return acceptedUploads; } + public long getRejectUploads() { return rejectedUploads; } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java new file mode 100644 index 0000000..fa8f575 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java @@ -0,0 +1,87 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import com.google.inject.Inject; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.NMCacheUploaderSCMProtocolMetrics; +import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.InfoBlock; +import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; + +public class SCMOverviewPage extends TwoColumnLayout { + + @Override protected void preHead(Page.HTML<_> html) { + set(ACCORDION_ID, "nav"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + } + + @Override protected Class content() { + return SCMOverviewBlock.class; + } + + @Override + protected Class nav() { + return SCMOverviewNavBlock.class; + } + + static private class SCMOverviewNavBlock extends HtmlBlock { + @Override + protected void render(Block html) { + html.div("#nav").h3("Tools").ul().li().a("/conf", "Configuration")._() + .li().a("/stacks", "Thread dump")._().li().a("/logs", "Logs")._() + .li().a("/metrics", "Metrics")._()._()._(); + } + } + + static private class SCMOverviewBlock extends HtmlBlock { + final SharedCacheManager scm; + + @Inject + SCMOverviewBlock(SharedCacheManager scm, ViewContext ctx) { + super(ctx); + this.scm = scm; + } + + @Override + protected void render(Block html) { + SCMMetricsInfo metricsInfo = new SCMMetricsInfo( + CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(), + NMCacheUploaderSCMProtocolMetrics.getInstance()); + info("Shared Cache Manager overview"). + _("Started on:", Times.format(scm.getStartTime())). + _("Cache hits: ", metricsInfo.getCacheHits()). + _("Cache misses: ", metricsInfo.getCacheMisses()). + _("Cache releases: ", metricsInfo.getCacheReleases()). + _("Accepted uploads: ", metricsInfo.getAcceptedUploads()). + _("Rejected uploads: ", metricsInfo.getRejectUploads()). + _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()). + _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles()); + html._(InfoBlock.class); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java new file mode 100644 index 0000000..033ffbd --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebApps; + +/** + * A very simple web interface for the metric reported by + * {@link org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager} + */ +public class SCMWebServer { + private static final Log LOG = LogFactory.getLog(SCMWebServer.class); + + private WebApp webApp; + + public void start(Configuration conf, SharedCacheManager scm) throws IOException { + String bindAddress = conf.get( + YarnConfiguration.SCM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_SCM_WEBAPP_ADDRESS); + try { + SCMWebApp scmWebApp = new SCMWebApp(scm); + this.webApp = WebApps.$for("cluster").at(bindAddress).start(scmWebApp); + LOG.info("Instantiated " + SCMWebApp.class.getName() + " at " + bindAddress); + } catch (Exception e) { + String msg = SCMWebApp.class.getName() + " failed to start."; + LOG.error(msg, e); + throw new IOException(msg); + } + } + + public void stop() { + if (this.webApp != null) { + this.webApp.stop(); + } + } + + private static class SCMWebApp extends WebApp { + + private final SharedCacheManager scm; + + public SCMWebApp(SharedCacheManager scm) { + this.scm = scm; + } + + @Override + public void setup() { + if (scm != null) { + bind(SharedCacheManager.class).toInstance(scm); + } + route("/", SCMController.class, "overview"); + route("/cleaner", SCMController.class, "cleaner"); + } + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java new file mode 100644 index 0000000..6bffcc6 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java @@ -0,0 +1,244 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; +import org.junit.Test; + +public class TestCleanerTask { + private static final String ROOT = + YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT; + private static final int STALENESS_MINUTES = + YarnConfiguration.DEFAULT_SCM_CLEANER_STALENESS_MINUTES; + private static final long SLEEP_TIME = + YarnConfiguration.DEFAULT_SCM_CLEANER_CLEANING_SLEEP_BETWEEN_CLEAN_MS; + private static final int NESTED_LEVEL = + YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL; + + @Test + public void testNonExistentRoot() throws Exception { + FileSystem fs = mock(FileSystem.class); + AppChecker appChecker = mock(AppChecker.class); + CleanerMetrics metrics = mock(CleanerMetrics.class); + SCMStore store = mock(SCMStore.class); + SCMContext context = mock(SCMContext.class); + + CleanerTask task = + createSpiedTask(fs, appChecker, store, context, metrics, + new AtomicBoolean()); + // the shared cache root does not exist + when(fs.exists(task.getRootPath())).thenReturn(false); + + task.run(); + + // process() should not be called + verify(task, never()).process(); + } + + @Test + public void testProcessFreshEntry() throws Exception { + FileSystem fs = mock(FileSystem.class); + AppChecker appChecker = mock(AppChecker.class); + CleanerMetrics metrics = mock(CleanerMetrics.class); + SCMStore store = mock(SCMStore.class); + SCMContext context = mock(SCMContext.class); + + CleanerTask task = + createSpiedTask(fs, appChecker, store, context, metrics, + new AtomicBoolean()); + + // mock an entry whose modification timestamp is fresh + long staleModificationTime = + System.currentTimeMillis() - + TimeUnit.MINUTES.toMillis(STALENESS_MINUTES - 1); + FileStatus entry = mock(FileStatus.class); + when(fs.exists(new Path(ROOT))).thenReturn(true); + when(entry.getPath()).thenReturn(new Path(ROOT + "/a/b/c/abc")); + when(entry.getModificationTime()).thenReturn(staleModificationTime); + when(store.getAccessTime(isA(String.class))).thenReturn(-1L); + + // process the entry + task.processSingleEntry(entry); + + // the directory should not be renamed + verify(fs, never()).rename(eq(entry.getPath()), isA(Path.class)); + // metrics should record a processed file (but not delete) + verify(metrics).reportAFileProcess(); + verify(metrics, never()).reportAFileDelete(); + } + + @Test + public void testProcessStaleEntry() throws Exception { + FileSystem fs = mock(FileSystem.class); + AppChecker appChecker = mock(AppChecker.class); + when(appChecker.appIsActive(isA(ApplicationId.class))).thenReturn(false); + CleanerMetrics metrics = mock(CleanerMetrics.class); + SCMStore store = mock(SCMStore.class); + SCMContext context = mock(SCMContext.class); + + CleanerTask task = + createSpiedTask(fs, appChecker, store, context, metrics, + new AtomicBoolean()); + + // mock an entry whose modification timestamp is stale + long staleModificationTime = + System.currentTimeMillis() - + TimeUnit.MINUTES.toMillis(STALENESS_MINUTES + 1); + FileStatus entry = mock(FileStatus.class); + String file = ROOT + "/a/b/c/abc"; + when(entry.getPath()).thenReturn(new Path(file)); + when(entry.getModificationTime()).thenReturn(staleModificationTime); + when(store.getAccessTime(isA(String.class))).thenReturn(-1L); + // mock the task to determine that this entry is in use + doReturn(true).when(task).entryIsStale(isA(String.class), + isA(FileStatus.class)); + when(store.removeKey(isA(String.class))).thenReturn(true); + // rename succeeds + when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(true); + // delete returns true + when(fs.delete(isA(Path.class), anyBoolean())).thenReturn(true); + + // process the entry + task.processSingleEntry(entry); + + // the directory should be renamed + verify(fs).rename(eq(entry.getPath()), isA(Path.class)); + // metrics should record a deleted file + verify(metrics).reportAFileDelete(); + verify(metrics, never()).reportAFileProcess(); + } + + private CleanerTask createSpiedTask(FileSystem fs, AppChecker appChecker, + SCMStore store, SCMContext context, CleanerMetrics metrics, + AtomicBoolean isCleanerRunning) { + return spy(new CleanerTask(ROOT, STALENESS_MINUTES, SLEEP_TIME, + NESTED_LEVEL, fs, appChecker, store, context, metrics, + isCleanerRunning, true)); + } + + @Test + public void testEntryIsInUseEntryIsRefreshed() throws Exception { + FileSystem fs = mock(FileSystem.class); + AppChecker appChecker = mock(AppChecker.class); + CleanerMetrics metrics = mock(CleanerMetrics.class); + SCMStore store = mock(SCMStore.class); + SCMContext context = mock(SCMContext.class); + + // have store say the entry does not exist + when(store.getAccessTime(isA(String.class))).thenReturn(-1L); + // have the filesystem return a fresh modification timestamp + FileStatus status = mock(FileStatus.class); + when(status.getModificationTime()).thenReturn(System.currentTimeMillis()); + + CleanerTask task = + createSpiedTask(fs, appChecker, store, context, metrics, + new AtomicBoolean()); + + // call entryIsInUse() + assertFalse(task.entryIsStale("fooKey", status)); + } + + @Test + public void testEntryIsInUseHasAnActiveApp() throws Exception { + FileSystem fs = mock(FileSystem.class); + AppChecker appChecker = mock(AppChecker.class); + CleanerMetrics metrics = mock(CleanerMetrics.class); + SCMStore store = mock(SCMStore.class); + SCMContext context = mock(SCMContext.class); + + long staleModificationTime = + System.currentTimeMillis() - + TimeUnit.MINUTES.toMillis(STALENESS_MINUTES + 1); + FileStatus entry = mock(FileStatus.class); + when(entry.getPath()).thenReturn(new Path(ROOT + "/a/b/c/abc")); + when(entry.getModificationTime()).thenReturn(staleModificationTime); + when(store.getAccessTime(isA(String.class))).thenReturn(-1L); + + CleanerTask task = + createSpiedTask(fs, appChecker, store, context, metrics, + new AtomicBoolean()); + + when(store.removeKey(isA(String.class))).thenReturn(false); + + // process the entry + task.processSingleEntry(entry); + + // metrics should record a processed file (but not delete) + verify(metrics).reportAFileProcess(); + verify(metrics, never()).reportAFileDelete(); + } + + @Test + public void testProcessInitialActiveApps() throws Exception { + FileSystem fs = mock(FileSystem.class); + AppChecker appChecker = mock(AppChecker.class); + CleanerMetrics metrics = mock(CleanerMetrics.class); + + // prepare an initial active app list + List ids = new ArrayList(); + // app id for which app checker will return active + ApplicationId id1 = mock(ApplicationId.class); + when(appChecker.appIsActive(id1)).thenReturn(true); + ids.add(id1); + // app id for which app checker will return inactive + ApplicationId id2 = mock(ApplicationId.class); + when(appChecker.appIsActive(id2)).thenReturn(false); + ids.add(id2); + SCMContext context = mock(SCMContext.class); + when(context.getInitialActiveApps()).thenReturn(ids); + // create the store + SCMStore store = new InMemorySCMStore(context); + + CleanerTask task = + createSpiedTask(fs, appChecker, store, context, metrics, + new AtomicBoolean()); + when(fs.exists(task.getRootPath())).thenReturn(true); + + task.run(); + + // process() should not be called + verify(task, never()).process(); + // the initial active app list should be down to 1 + assertSame(1, ids.size()); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java new file mode 100644 index 0000000..eebc985 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java @@ -0,0 +1,250 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ClientSCMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + + +/** + * Basic unit tests for the Client to SCM Protocol Service. + */ +public class TestClientSCMProtocolService { + + static ClientProtocolService service; + static ClientSCMProtocol clientSCMProxy; + static SCMStore store; + private final RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + + @BeforeClass + public static void startUp() { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.SCM_STORE_IMPL, InMemorySCMStore.class.getName()); + store = new InMemorySCMStore(new SCMContext()); + store.init(conf); + store.start(); + + service = new ClientProtocolService(store); + service.init(conf); + service.start(); + + YarnRPC rpc = YarnRPC.create(new Configuration()); + + InetSocketAddress scmAddress = + conf.getSocketAddr(YarnConfiguration.SCM_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADDRESS, + YarnConfiguration.DEFAULT_SCM_PORT); + + clientSCMProxy = + (ClientSCMProtocol) rpc.getProxy(ClientSCMProtocol.class, scmAddress, + conf); + } + + @AfterClass + public static void cleanUp() { + if (store != null) { + store.stop(); + } + + if (service != null) { + service.stop(); + } + + if (clientSCMProxy != null) { + RPC.stopProxy(clientSCMProxy); + } + } + + @After + public void cleanUpTest() { + store.clearCache(); + } + + @Test + public void testUse_MissingEntry() throws Exception { + long misses = ClientSCMMetrics.getInstance().getCacheMisses(); + UseSharedCacheResourceRequest request = + recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class); + request.setResourceKey("key1"); + request.setAppId(createAppId(1, 1L)); + assertNull(clientSCMProxy.use(request).getPath()); + assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics + .getInstance().getCacheMisses() - misses); + } + + @Test + public void testUse_ExistingEntry_NoAppIds() throws Exception { + // Pre-populate the SCM with one cache entry + store.addKey("key1", "foo.jar"); + + long hits = ClientSCMMetrics.getInstance().getCacheHits(); + + UseSharedCacheResourceRequest request = + recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class); + request.setResourceKey("key1"); + request.setAppId(createAppId(2, 2L)); + // Expecting default depth of 3 and rootdir of /sharedcache + String expectedPath = "/sharedcache/k/e/y/key1/foo.jar"; + assertEquals(expectedPath, clientSCMProxy.use(request).getPath()); + assertEquals(1, store.getResourceReferences("key1").size()); + assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics + .getInstance().getCacheHits() - hits); + + } + + @Test + public void testUse_ExistingEntry_OneId() throws Exception { + // Pre-populate the SCM with one cache entry + store.addKey("key1", "foo.jar"); + store.addResourceReference("key1", createAppId(1, 1L), "user"); + assertEquals(1, store.getResourceReferences("key1").size()); + long hits = ClientSCMMetrics.getInstance().getCacheHits(); + + // Add a new distinct appId + UseSharedCacheResourceRequest request = + recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class); + request.setResourceKey("key1"); + request.setAppId(createAppId(2, 2L)); + + // Expecting default depth of 3 and rootdir of /sharedcache + String expectedPath = "/sharedcache/k/e/y/key1/foo.jar"; + assertEquals(expectedPath, clientSCMProxy.use(request).getPath()); + assertEquals(2, store.getResourceReferences("key1").size()); + assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics + .getInstance().getCacheHits() - hits); + } + + @Test + public void testUse_ExistingEntry_DupId() throws Exception { + // Pre-populate the SCM with one cache entry + store.addKey("key1", "foo.jar"); + UserGroupInformation testUGI = UserGroupInformation.getCurrentUser(); + store.addResourceReference("key1", createAppId(1, 1L), + testUGI.getShortUserName()); + assertEquals(1, store.getResourceReferences("key1").size()); + + long hits = ClientSCMMetrics.getInstance().getCacheHits(); + + // Add a new duplicate appId + UseSharedCacheResourceRequest request = + recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class); + request.setResourceKey("key1"); + request.setAppId(createAppId(1, 1L)); + + // Expecting default depth of 3 and rootdir of /sharedcache + String expectedPath = "/sharedcache/k/e/y/key1/foo.jar"; + assertEquals(expectedPath, clientSCMProxy.use(request).getPath()); + assertEquals(1, store.getResourceReferences("key1").size()); + + assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics + .getInstance().getCacheHits() - hits); + } + + @Test + public void testRelease_ExistingEntry_NonExistantAppId() throws Exception { + // Pre-populate the SCM with one cache entry + store.addKey("key1", "foo.jar"); + store.addResourceReference("key1", createAppId(1, 1L), "user"); + assertEquals(1, store.getResourceReferences("key1").size()); + + long releases = ClientSCMMetrics.getInstance().getCacheReleases(); + + ReleaseSharedCacheResourceRequest request = + recordFactory + .newRecordInstance(ReleaseSharedCacheResourceRequest.class); + request.setResourceKey("key1"); + request.setAppId(createAppId(2, 2L)); + clientSCMProxy.release(request); + assertEquals(1, store.getResourceReferences("key1").size()); + + assertEquals( + "Client SCM metrics were updated when a release did not happen", 0, + ClientSCMMetrics.getInstance().getCacheReleases() - releases); + + } + + @Test + public void testRelease_ExistingEntry_WithAppId() throws Exception { + // Pre-populate the SCM with one cache entry + store.addKey("key1", "foo.jar"); + UserGroupInformation testUGI = UserGroupInformation.getCurrentUser(); + store.addResourceReference("key1", createAppId(1, 1L), + testUGI.getShortUserName()); + assertEquals(1, store.getResourceReferences("key1").size()); + + long releases = ClientSCMMetrics.getInstance().getCacheReleases(); + + ReleaseSharedCacheResourceRequest request = + recordFactory + .newRecordInstance(ReleaseSharedCacheResourceRequest.class); + request.setResourceKey("key1"); + request.setAppId(createAppId(1, 1L)); + clientSCMProxy.release(request); + assertEquals(0, store.getResourceReferences("key1").size()); + + assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics + .getInstance().getCacheReleases() - releases); + + } + + @Test + public void testRelease_MissingEntry() throws Exception { + + long releases = ClientSCMMetrics.getInstance().getCacheReleases(); + + ReleaseSharedCacheResourceRequest request = + recordFactory + .newRecordInstance(ReleaseSharedCacheResourceRequest.class); + request.setResourceKey("key2"); + request.setAppId(createAppId(2, 2L)); + clientSCMProxy.release(request); + assertNotNull(store.getResourceReferences("key2")); + assertEquals(0, store.getResourceReferences("key2").size()); + assertEquals( + "Client SCM metrics were updated when a release did not happen.", 0, + ClientSCMMetrics.getInstance().getCacheReleases() - releases); + } + + private ApplicationId createAppId(int id, long timestamp) { + return ApplicationId.newInstance(timestamp, id); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestNMCacheUploaderSCMProtocolService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestNMCacheUploaderSCMProtocolService.java new file mode 100644 index 0000000..a649a10 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestNMCacheUploaderSCMProtocolService.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.net.InetSocketAddress; +import java.util.Collection; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.NMCacheUploaderSCMProtocolMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.ResourceReference; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + + +/** + * Basic unit tests for the NodeManger to SCM Protocol Service. + */ +public class TestNMCacheUploaderSCMProtocolService { + + static NMCacheUploaderSCMProtocolService service; + static NMCacheUploaderSCMProtocol proxy; + static SCMStore store; + private final RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + + @BeforeClass + public static void startUp() { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.SCM_STORE_IMPL, InMemorySCMStore.class.getName()); + store = new InMemorySCMStore(new SCMContext()); + store.init(conf); + store.start(); + + service = new NMCacheUploaderSCMProtocolService(store); + service.init(conf); + service.start(); + + YarnRPC rpc = YarnRPC.create(new Configuration()); + + InetSocketAddress scmAddress = + conf.getSocketAddr(YarnConfiguration.NM_SCM_ADDRESS, + YarnConfiguration.DEFAULT_NM_SCM_ADDRESS, + YarnConfiguration.DEFAULT_NM_SCM_PORT); + + proxy = + (NMCacheUploaderSCMProtocol) rpc.getProxy( + NMCacheUploaderSCMProtocol.class, scmAddress, conf); + } + + @AfterClass + public static void cleanUp() { + if (store != null) { + store.stop(); + } + + if (service != null) { + service.stop(); + } + + if (proxy != null) { + RPC.stopProxy(proxy); + } + } + + @After + public void cleanUpTest() { + store.clearCache(); + } + + @Test + public void testNotify_noEntry() throws Exception { + long accepted = + NMCacheUploaderSCMProtocolMetrics.getInstance().getAcceptedUploads(); + + NotifySCMRequest request = + recordFactory.newRecordInstance(NotifySCMRequest.class); + request.setResourceKey("key1"); + request.setFilename("foo.jar"); + assertTrue(proxy.notify(request).getAccepted()); + Collection set = store.getResourceReferences("key1"); + assertNotNull(set); + assertEquals(0, set.size()); + + assertEquals( + "NM upload metrics aren't updated.", 1, + NMCacheUploaderSCMProtocolMetrics.getInstance().getAcceptedUploads() - accepted); + + } + + @Test + public void testNotify_entryExists_differentName() throws Exception { + + long rejected = + NMCacheUploaderSCMProtocolMetrics.getInstance().getRejectUploads(); + + store.addKey("key1", "foo.jar"); + NotifySCMRequest request = + recordFactory.newRecordInstance(NotifySCMRequest.class); + request.setResourceKey("key1"); + request.setFilename("foobar.jar"); + assertFalse(proxy.notify(request).getAccepted()); + Collection set = store.getResourceReferences("key1"); + assertNotNull(set); + assertEquals(0, set.size()); + assertEquals( + "NM upload metrics aren't updated.", 1, + NMCacheUploaderSCMProtocolMetrics.getInstance().getRejectUploads() - rejected); + + } + + @Test + public void testNotify_entryExists_sameName() throws Exception { + + long accepted = + NMCacheUploaderSCMProtocolMetrics.getInstance().getAcceptedUploads(); + + store.addKey("key1", "foo.jar"); + NotifySCMRequest request = + recordFactory.newRecordInstance(NotifySCMRequest.class); + request.setResourceKey("key1"); + request.setFilename("foo.jar"); + assertTrue(proxy.notify(request).getAccepted()); + Collection set = store.getResourceReferences("key1"); + assertNotNull(set); + assertEquals(0, set.size()); + assertEquals( + "NM upload metrics aren't updated.", 1, + NMCacheUploaderSCMProtocolMetrics.getInstance().getAcceptedUploads() - accepted); + + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java new file mode 100644 index 0000000..6768d9d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.junit.Test; + +public class TestRemoteAppChecker { + + @Test + public void testNonExistentApp() throws Exception { + ApplicationClientProtocol applicationsManager = mock(ApplicationClientProtocol.class); + GetApplicationReportResponse response = mock(GetApplicationReportResponse.class); + // mock the response so that it returns a null application report + when(response.getApplicationReport()).thenReturn(null); + when(applicationsManager.getApplicationReport(isA(GetApplicationReportRequest.class))). + thenReturn(response); + + AppChecker appChecker = new RemoteAppChecker(applicationsManager); + assertFalse(appChecker.appIsActive(isA(ApplicationId.class))); + } + + @Test + public void testRunningApp() throws Exception { + ApplicationClientProtocol applicationsManager = mock(ApplicationClientProtocol.class); + GetApplicationReportResponse response = mock(GetApplicationReportResponse.class); + ApplicationReport report = mock(ApplicationReport.class); + when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.ACCEPTED); + // mock the response so that it returns an application report in the + // accepted state + when(response.getApplicationReport()).thenReturn(report); + when(applicationsManager.getApplicationReport(isA(GetApplicationReportRequest.class))). + thenReturn(response); + + AppChecker appChecker = new RemoteAppChecker(applicationsManager); + assertTrue(appChecker.appIsActive(isA(ApplicationId.class))); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java new file mode 100644 index 0000000..d18ddea --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.SCMAdminProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore; +import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Basic unit tests for the SCM Admin Protocol Service. + */ +public class TestSCMAdminProtocolService { + + static SCMAdminProtocolService service; + static SCMAdminProtocol SCMAdminProxy; + static SCMStore store; + static CleanerService cleaner; + private final RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + + @Before + public void startUp() { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.SCM_STORE_IMPL, InMemorySCMStore.class.getName()); + + cleaner = mock(CleanerService.class); + + service = spy(new SCMAdminProtocolService(cleaner)); + service.init(conf); + service.start(); + + YarnRPC rpc = YarnRPC.create(new Configuration()); + + InetSocketAddress scmAddress = + conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_SCM_ADMIN_PORT); + + SCMAdminProxy = + (SCMAdminProtocol) rpc.getProxy(SCMAdminProtocol.class, scmAddress, + conf); + } + + @After + public void cleanUpTest() { + if (service != null) { + service.stop(); + } + + if (SCMAdminProxy != null) { + RPC.stopProxy(SCMAdminProxy); + } + } + + @Test + public void testRunCleanerTask() throws Exception { + when(cleaner.runCleanerTask()).thenReturn(true); + RunSharedCacheCleanerTaskRequest request = + recordFactory.newRecordInstance(RunSharedCacheCleanerTaskRequest.class); + RunSharedCacheCleanerTaskResponse response = SCMAdminProxy.runCleanerTask(request); + Assert.assertTrue("cleaner task request isn't accepted", response.getAccepted()); + verify(service, times(1)).runCleanerTask(any(RunSharedCacheCleanerTaskRequest.class)); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheManager.java new file mode 100644 index 0000000..4be80d5 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheManager.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager; + +import static org.apache.hadoop.fs.CreateFlag.CREATE; +import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.sharedcache.CacheStructureUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestSharedCacheManager { + private static final Configuration conf = new YarnConfiguration(); + + private static final FileSystem fs; + + private static final String KEY1 = "abcdefgh"; + private static final String FILE1 = "foo"; + private static final String KEY2 = "ijklmnop"; + private static final String FILE2 = "bar"; + + private static final Path baseDir; + private static final String root; + private static final String path1; + private static final String path2; + + static { + try { + fs = FileSystem.getLocal(conf); + } catch (IOException e) { + throw new RuntimeException(e); + } + + baseDir = + new Path("target", TestSharedCacheManager.class.getSimpleName()) + .makeQualified(fs.getUri(), fs.getWorkingDirectory()); + root = baseDir.toUri().getPath(); + // set the shared cache root + conf.set(YarnConfiguration.SHARED_CACHE_ROOT, root); + + path1 = getFullPath(KEY1, FILE1); + path2 = getFullPath(KEY2, FILE2); + } + + private static String getFullPath(String key, String fileName) { + int cacheDepth = YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL; + return CacheStructureUtil.getCacheEntryPath(cacheDepth, root, key) + + Path.SEPARATOR + fileName; + } + + @Before + public void setUp() throws IOException { + FileContext files = FileContext.getLocalFSFileContext(); + files.mkdir(baseDir, null, true); + // add a couple of directories and files + createFile(new Path(path1), files); + createFile(new Path(path2), files); + } + + @After + public void shutDown() throws IOException { + FileContext files = FileContext.getLocalFSFileContext(); + files.delete(baseDir, true); + } + + private void createFile(Path path, FileContext files) throws IOException { + Path parent = path.getParent(); + files.mkdir(parent, null, true); + FSDataOutputStream out = null; + try { + out = files.create(path, EnumSet.of(CREATE, OVERWRITE)); + out.writeUTF("This is a test"); + } finally { + if (out != null) { + out.close(); + } + } + } + + @Test + public void testCreateSCMContext() throws Exception { + SharedCacheManager scm = new SharedCacheManager(); + + AppChecker appChecker = mock(AppChecker.class); + List ids = new ArrayList(); + ids.add(createAppId(1, 1L)); + ids.add(createAppId(2, 1L)); + ids.add(createAppId(3, 1L)); + when(appChecker.getAllActiveApps()).thenReturn(ids); + + SCMContext context = scm.createSCMContext(appChecker, conf); + Map initialCachedEntries = context.getInitialCachedEntries(); + // verify the initial cached entries + assertSame(2, initialCachedEntries.size()); + assertEquals(FILE1, initialCachedEntries.get(KEY1)); + assertEquals(FILE2, initialCachedEntries.get(KEY2)); + + // verify the active apps + assertSame(ids, context.getInitialActiveApps()); + } + + private ApplicationId createAppId(int id, long timestamp) { + return ApplicationId.newInstance(timestamp, id); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java new file mode 100644 index 0000000..0aaa3bf --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.sharedcachemanager.metrics; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Before; +import org.junit.Test; + +public class TestCleanerMetrics { + + Configuration conf = new Configuration(); + long currTime = System.currentTimeMillis(); + // it is float to allow floating point rate calculation + float lastDuration = 0; + float totalDurations = 0; + CleanerMetrics cleanerMetrics; + + @Before + public void init() { + CleanerMetrics.initSingleton(conf); + cleanerMetrics = CleanerMetrics.getInstance(); + } + + @Test + public void testMetricsOverMultiplePeriods() { + simulateACleanerRun(); + assertMetrics(4, 4, 1, 1); + currTime += 1300; + simulateACleanerRun(); + assertMetrics(4, 8, 1, 2); + } + + public void simulateACleanerRun() { + long startTime = currTime; + cleanerMetrics.reportCleaningStart(currTime); + currTime += 9; + cleanerMetrics.reportAFileProcess(currTime); + cleanerMetrics.reportAFileDelete(currTime); + currTime++; + cleanerMetrics.reportAFileProcess(currTime); + cleanerMetrics.reportAFileProcess(currTime); + lastDuration = currTime - startTime; + totalDurations += lastDuration; + } + + void assertMetrics(int proc, int totalProc, int del, int totalDel) { + assertEquals( + "Processed files in the last period are not measured correctly", proc, + cleanerMetrics.getProcessedFiles()); + assertEquals("Total processed files are not measured correctly", + totalProc, cleanerMetrics.getTotalProcessedFiles()); + assertEquals( + "Deleted files in the last period are not measured correctly", del, + cleanerMetrics.getDeletedFiles()); + assertEquals("Total deleted files are not measured correctly", + totalDel, cleanerMetrics.getTotalDeletedFiles()); + + assertEquals( + "Rate of processed files in the last period are not measured correctly", + CleanerMetrics.ratePerMsToHour(proc / lastDuration), + cleanerMetrics.getProcessRate()); + assertEquals( + "Rate of total processed files are not measured correctly", + CleanerMetrics.ratePerMsToHour(totalProc / totalDurations), + cleanerMetrics.getTotalProcessRate()); + assertEquals( + "Rate of deleted files in the last period are not measured correctly", + CleanerMetrics.ratePerMsToHour(del / lastDuration), + cleanerMetrics.getDeleteRate()); + assertEquals( + "Rate of total deleted files are not measured correctly", + CleanerMetrics.ratePerMsToHour(totalDel / totalDurations), + cleanerMetrics.getTotalDeleteRate()); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java new file mode 100644 index 0000000..c3c8a7c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java @@ -0,0 +1,271 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.sharedcachemanager.SCMContext; +import org.junit.Test; + +public class TestInMemorySCMStore { + @Test + public void testAddKeyConcurrency() throws Exception { + final SCMStore store = new InMemorySCMStore(new SCMContext()); + final String key = "key1"; + int count = 5; + ExecutorService exec = Executors.newFixedThreadPool(count); + List> futures = new ArrayList>(count); + final CountDownLatch start = new CountDownLatch(1); + for (int i = 0; i < count; i++) { + final String fileName = "foo-" + i + ".jar"; + Callable task = new Callable() { + public String call() throws Exception { + start.await(); + String result = store.addKey(key, fileName); + System.out.println("fileName: " + fileName + ", result: " + result); + return result; + } + }; + futures.add(exec.submit(task)); + } + // start them all at the same time + start.countDown(); + // check the result; they should all agree with the value + Set results = new HashSet(); + for (Future future: futures) { + results.add(future.get()); + } + assertSame(1, results.size()); + exec.shutdown(); + store.close(); + } + + @Test + public void testAddAppIdNonExistentKey() throws Exception { + SCMStore store = new InMemorySCMStore(new SCMContext()); + String key = "key1"; + ApplicationId id = createAppId(1, 1L); + // try adding an app id without adding the key first + assertNull(store.addResourceReference(key, id, "user")); + store.close(); + } + + @Test + public void testRemoveKeyEmptyAppIds() throws Exception { + SCMStore store = new InMemorySCMStore(new SCMContext()); + String key = "key1"; + String fileName = "foo.jar"; + // first add key + store.addKey(key, fileName); + // try removing the key; it should return true + assertTrue(store.removeKey(key)); + store.close(); + } + + @Test + public void testAddAppIdRemoveKey() throws Exception { + SCMStore store = new InMemorySCMStore(new SCMContext()); + String key = "key1"; + ApplicationId id = createAppId(1, 1L); + String user = "user"; + // add the key, and then add an app id + store.addKey(key, "foo.jar"); + store.addResourceReference(key, id, user); + // removeKey should return false + assertTrue(!store.removeKey(key)); + // the entry and the app id should be intact + Collection refs = store.getResourceReferences(key); + assertTrue(refs != null); + assertEquals(Collections.singleton(new ResourceReference(id, user)), refs); + store.close(); + } + + @Test + public void testAddAppIdConcurrency() throws Exception { + final SCMStore store = new InMemorySCMStore(new SCMContext()); + final String key = "key1"; + final String user = "user"; + String fileName = "foo.jar"; + + // first add the key + store.addKey(key, fileName); + + // make concurrent addAppId calls (clients) + int count = 5; + ExecutorService exec = Executors.newFixedThreadPool(count); + List> futures = new ArrayList>(count); + final CountDownLatch start = new CountDownLatch(1); + for (int i = 0; i < count; i++) { + final ApplicationId id = createAppId(i, i); + Callable task = new Callable() { + public String call() throws Exception { + start.await(); + return store.addResourceReference(key, id, user); + } + }; + futures.add(exec.submit(task)); + } + // start them all at the same time + start.countDown(); + // check the result + Set results = new HashSet(); + for (Future future: futures) { + results.add(future.get()); + } + // they should all have the same file name + assertSame(1, results.size()); + assertEquals(Collections.singleton(fileName), results); + // there should be 5 app ids as a result + Collection refs = store.getResourceReferences(key); + assertSame(count, refs.size()); + exec.shutdown(); + store.close(); + } + + @Test + public void testAddAppIdAddKeyConcurrency() throws Exception { + final SCMStore store = new InMemorySCMStore(new SCMContext()); + final String key = "key1"; + final String fileName = "foo.jar"; + final String user = "user"; + final ApplicationId id = createAppId(1, 1L); + // add the key and add the id at the same time + ExecutorService exec = Executors.newFixedThreadPool(2); + final CountDownLatch start = new CountDownLatch(1); + Callable addKeyTask = new Callable() { + public String call() throws Exception { + start.await(); + return store.addKey(key, fileName); + } + }; + Callable addAppIdTask = new Callable() { + public String call() throws Exception { + start.await(); + return store.addResourceReference(key, id, user); + } + }; + Future addAppIdFuture = exec.submit(addAppIdTask); + Future addKeyFuture = exec.submit(addKeyTask); + // start them at the same time + start.countDown(); + // get the results + String addKeyResult = addKeyFuture.get(); + String addAppIdResult = addAppIdFuture.get(); + assertEquals(fileName, addKeyResult); + System.out.println("addAppId() result: " + addAppIdResult); + // it may be null or the fileName depending on the timing + assertTrue(addAppIdResult == null || addAppIdResult.equals(fileName)); + exec.shutdown(); + store.close(); + } + + @Test + public void testRemoveAppIds() throws Exception { + SCMStore store = new InMemorySCMStore(new SCMContext()); + String key = "key1"; + String fileName = "foo.jar"; + String user = "user"; + // first add the key + store.addKey(key, fileName); + // add an app id + ApplicationId id = createAppId(1, 1L); + ResourceReference myRef = new ResourceReference(id, user); + String result = store.addResourceReference(key, id, user); + assertEquals(fileName, result); + Collection refs = store.getResourceReferences(key); + assertSame(1, refs.size()); + assertEquals(Collections.singleton(myRef), refs); + // remove the same key + store.removeResourceRefs(key, Collections.singleton(myRef), true); + Collection newRefs = store.getResourceReferences(key); + assertTrue(newRefs == null || newRefs.isEmpty()); + store.close(); + } + + @Test + public void testRemoveAppId() throws Exception { + SCMStore store = new InMemorySCMStore(new SCMContext()); + String key = "key1"; + String fileName = "foo.jar"; + String user = "user"; + // first add the key + store.addKey(key, fileName); + // add an app id + ApplicationId id = createAppId(1, 1L); + ResourceReference myRef = new ResourceReference(id, user); + String result = store.addResourceReference(key, id, user); + assertEquals(fileName, result); + Collection refs = store.getResourceReferences(key); + assertSame(1, refs.size()); + assertEquals(Collections.singleton(myRef), refs); + // remove the same key + store.removeResourceReference(key, myRef, true); + Collection newRefs = store.getResourceReferences(key); + assertTrue(newRefs == null || newRefs.isEmpty()); + store.close(); + } + + @Test + public void testBootstrapping() throws Exception { + Map initialCachedEntries = new HashMap(); + int count = 10; + for (int i = 0; i < count; i++) { + String key = String.valueOf(i); + String fileName = key + ".jar"; + initialCachedEntries.put(key, fileName); + } + + SCMContext context = new SCMContext(initialCachedEntries, null); + SCMStore store = new InMemorySCMStore(context); + ApplicationId id = createAppId(1, 1L); + // the entries from the cached entries should now exist + for (int i = 0; i < count; i++) { + String key = String.valueOf(i); + String fileName = key + ".jar"; + String result = store.addResourceReference(key, id, "user"); + // the value should not be null (i.e. it has the key) and the filename should match + assertEquals(fileName, result); + // the initial input should be emptied + assertTrue(initialCachedEntries.isEmpty()); + } + store.close(); + } + + private ApplicationId createAppId(int id, long timestamp) { + return ApplicationId.newInstance(timestamp, id); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml index b635d10..886773a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -39,6 +39,7 @@ hadoop-yarn-server-nodemanager hadoop-yarn-server-web-proxy hadoop-yarn-server-resourcemanager + hadoop-yarn-server-sharedcachemanager hadoop-yarn-server-tests hadoop-yarn-server-applicationhistoryservice