diff --git hadoop-project/pom.xml hadoop-project/pom.xml index 9c91a96..6215fea 100644 --- hadoop-project/pom.xml +++ hadoop-project/pom.xml @@ -470,7 +470,7 @@ commons-codec commons-codec - 1.4 + 1.8 commons-net diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java index 1552cdf..77616ec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java @@ -66,5 +66,9 @@ * in #{@link LocalResource}. Currently only jars support pattern, all * others will be treated like a #{@link LocalResourceType#ARCHIVE}. */ - PATTERN + PATTERN, + /** + * File that need to be broadcasted using BitTorrent + */ + BTFILE } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c3d121a..58455fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -161,6 +161,7 @@ enum LocalResourceTypeProto { ARCHIVE = 1; FILE = 2; PATTERN = 3; + BTFILE = 4; } message LocalResourceProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/pom.xml new file mode 100644 index 0000000..f3a3382 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/pom.xml @@ -0,0 +1,58 @@ + + + + + hadoop-yarn-applications + org.apache.hadoop + 2.9.0-SNAPSHOT + + 4.0.0 + org.apache.hadoop + hadoop-yarn-applications-broadcast-demo + 2.9.0-SNAPSHOT + Apache Hadoop YARN Broadcast Demo + + + + ${project.parent.parent.basedir} + + + + + + org.apache.hadoop + hadoop-yarn-server-broadcast + 2.9.0-SNAPSHOT + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-yarn-api + 2.9.0-SNAPSHOT + + + org.apache.hadoop + hadoop-yarn-client + 2.9.0-SNAPSHOT + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/src/main/java/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/src/main/java/ApplicationMaster.java new file mode 100644 index 0000000..b6747c8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/src/main/java/ApplicationMaster.java @@ -0,0 +1,310 @@ +package org.apache.hadoop.yarn.applications.broadcast; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.Options; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.records.*; +import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.client.api.async.NMClientAsync; +import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.broadcast.service.BroadcastClient; +import org.apache.hadoop.yarn.util.ConverterUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class ApplicationMaster { + private AtomicInteger numCompletedContainers = new AtomicInteger(); + protected AtomicInteger numRequestedContainers = new AtomicInteger(); + private AtomicInteger numFailedContainers = new AtomicInteger(); + protected AtomicInteger numAllocatedContainers = new AtomicInteger(); + public int numTotalContainers; + + @SuppressWarnings("rawtypes") + private AMRMClientAsync amRMClient; + private NMClientAsync nmClientAsync; + private NMCallbackHandler containerListener; + private FileSystem hdfs; + private volatile boolean done; + private List launchThreads = new ArrayList(); + private String appId; + + private Configuration conf; + private int containerMemory; + private int containerVCores; + private int requestPriority; + + private String appName = "demo"; + private String dataLocalPath = "data"; + private String dataRemotePath = "data"; + private String torrentStr = null; + + private Map shellEnv = new HashMap(); + private String downloadType; + private int downloadSize; + + public static void main(String[] args) throws Exception { + ApplicationMaster appMaster = new ApplicationMaster(); + appMaster.init(args); + appMaster.run(); + appMaster.finish(); + } + + private void init(String[] args) throws Exception { + conf = new YarnConfiguration(); + try { + hdfs = FileSystem.get(conf); + } catch (IOException e) { + e.printStackTrace(); + } + containerMemory = 500; + containerVCores = 1; + + Options opts = new Options(); + opts.addOption("appId", true, "application id"); + opts.addOption("dtype", true, "download type"); + opts.addOption("dsize", true, "download block size"); + opts.addOption("nc", true, "# containers"); + CommandLine cliParser = new GnuParser().parse(opts, args); + + this.appId = cliParser.getOptionValue("appId"); + this.downloadType = cliParser.getOptionValue("dtype"); + this.downloadSize = Integer.parseInt(cliParser.getOptionValue("dsize")); + this.numTotalContainers = Integer.parseInt(cliParser.getOptionValue("nc")); + } + + private boolean finish() throws Exception { + while (!done && (numCompletedContainers.get() != numTotalContainers)) + Thread.sleep(2000); + for (Thread lauThread : launchThreads) + lauThread.join(1000); + nmClientAsync.stop(); + + FinalApplicationStatus appStatus = (numFailedContainers.get() == 0 && numCompletedContainers.get() == numTotalContainers) ? FinalApplicationStatus.SUCCEEDED : FinalApplicationStatus.FAILED; + amRMClient.unregisterApplicationMaster(appStatus, "", null); + amRMClient.stop(); + return appStatus == FinalApplicationStatus.SUCCEEDED; + } + + @SuppressWarnings({ "unchecked", "unchecked" }) + private void run() throws Exception { + /* begin broadcast */ + if (downloadType.equals("torrent")) + torrentStr = BroadcastClient.beginBroadcast((new File(dataLocalPath)).getAbsolutePath(), appId, downloadSize); + if (downloadType.equals("hdfs")) { + Path hdfsPath = new Path(hdfs.getHomeDirectory(), appName + "/" + appId + "/" + dataRemotePath); + hdfs.copyFromLocalFile(new Path(dataLocalPath), hdfsPath); + } + + /* setup am-rm client */ + AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); + amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); + amRMClient.init(conf); + amRMClient.start(); + + /* setup am-nm client */ + containerListener = new NMCallbackHandler(this); + nmClientAsync = new NMClientAsyncImpl(containerListener); + nmClientAsync.init(conf); + nmClientAsync.start(); + + /* register on rm */ + RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(NetUtils.getHostname(), -1, ""); + List previousAAMRunningContainers = response.getContainersFromPreviousAttempts(); + numAllocatedContainers.addAndGet(previousAAMRunningContainers.size()); + int numTotalContainersToRequest = numTotalContainers - previousAAMRunningContainers.size(); // FIXME + for (int i = 0; i < numTotalContainersToRequest; i++) + amRMClient.addContainerRequest(setupContainerAskForRM()); + numRequestedContainers.set(numTotalContainers); + } + + private ContainerRequest setupContainerAskForRM() { + Priority pri = Priority.newInstance(2); + Resource capacity = Resource.newInstance(containerMemory, containerVCores); + return new ContainerRequest(capacity, null, null, pri, true); + } + + private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler { + + @Override + public float getProgress() { + return (float) (numCompletedContainers.get() * 1.0 / numTotalContainers); + } + + @Override + public void onContainersAllocated(List allocatedContainers) { + numAllocatedContainers.addAndGet(allocatedContainers.size()); + for (Container allocatedContainer : allocatedContainers) { + LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(allocatedContainer, containerListener); + Thread launchThread = new Thread(runnableLaunchContainer); + + launchThreads.add(launchThread); + launchThread.start(); + } + } + + @SuppressWarnings("unchecked") + @Override + public void onContainersCompleted( + List completedContainers) { + for (ContainerStatus containerStatus : completedContainers) { + int exitStatus = containerStatus.getExitStatus(); + if (exitStatus != 0) { + if (exitStatus == ContainerExitStatus.ABORTED) { // FIXME + numCompletedContainers.incrementAndGet(); + numFailedContainers.incrementAndGet(); + } else { + numAllocatedContainers.decrementAndGet(); + numRequestedContainers.decrementAndGet(); + } + } else { + numCompletedContainers.incrementAndGet(); + } + } + + int askCount = numTotalContainers - numRequestedContainers.get(); + numRequestedContainers.addAndGet(askCount); + if (askCount > 0) + for (int i = 0; i 0) + System.out.println("ask for container again"); + + + if (numCompletedContainers.get() >= numTotalContainers) + done = true; + } + + @Override + public void onError(Throwable arg0) { // FIXME + done = true; + amRMClient.stop(); + } + + @Override + public void onNodesUpdated(List arg0) { // FIXME + } + + @Override + public void onShutdownRequest() { // FIXME + done = true; + } + + } + + static class NMCallbackHandler implements NMClientAsync.CallbackHandler { + private final ApplicationMaster applicationMaster; + private ConcurrentHashMap containers = new ConcurrentHashMap(); + + public NMCallbackHandler(ApplicationMaster applicationMaster) { + this.applicationMaster = applicationMaster; + } + + @Override + public void onContainerStarted(ContainerId containerId, + Map allServiceResponse) { + } + + @Override + public void onContainerStatusReceived(ContainerId arg0, + ContainerStatus arg1) { + } + + @Override + public void onContainerStopped(ContainerId containerId) { + } + + @Override + public void onGetContainerStatusError(ContainerId arg0, Throwable arg1) { + // TODO Auto-generated method stub + + } + + @Override + public void onStartContainerError(ContainerId containerId, Throwable t) { + containers.remove(containerId); + applicationMaster.numCompletedContainers.incrementAndGet(); // FIXME + applicationMaster.numFailedContainers.incrementAndGet(); + } + + @Override + public void onStopContainerError(ContainerId containerId, Throwable t) { + // TODO Auto-generated method stub + containers.remove(containerId); + } + + public void addContainer(ContainerId id, Container container) { + containers.put(id, container); + } + } + + private class LaunchContainerRunnable implements Runnable { + Container container; + NMCallbackHandler containerListener; + + public LaunchContainerRunnable(Container allocatedContainer, + NMCallbackHandler containerListener) { + this.container = allocatedContainer; + this.containerListener = containerListener; + } + + @Override + public void run() { + Map localResources = new HashMap(); + try { + if (downloadType.equals("torrent")) + addToLocalResourcesTorrent(torrentStr, dataLocalPath, localResources); + else + addToLocalResourcesHdfs(dataLocalPath, dataRemotePath, localResources); + } catch (Exception e) { + e.printStackTrace(); + } + Vector vargs = new Vector(5); + vargs.add("md5sum data"); + vargs.add("1>"+ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); + vargs.add("2>"+ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); + StringBuilder command = new StringBuilder(); + for (CharSequence str : vargs) + command.append(str).append(" "); + List commands = new ArrayList(); + commands.add(command.toString()); + + ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(localResources, shellEnv, commands, null, null, null); + containerListener.addContainer(container.getId(), container); + nmClientAsync.startContainerAsync(container, ctx); + } + + private void addToLocalResourcesHdfs(String fileLocalPath, String fileRemotePath, Map localResource) throws Exception { + Path hdfsPath = new Path(hdfs.getHomeDirectory(), appName + "/" + appId + "/" + fileRemotePath); + FileStatus fileStatus = hdfs.getFileStatus(hdfsPath); + LocalResource rsrc = LocalResource.newInstance( + ConverterUtils.getYarnUrlFromPath(hdfsPath), LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION, fileStatus.getLen(), + fileStatus.getModificationTime()); + localResource.put(fileRemotePath, rsrc); + } + + private void addToLocalResourcesTorrent(String torrentStr, + String fileRemotePath, Map localResource) throws Exception { + Path hdfsPath = new Path("hdfs://localhost:9000"+torrentStr); + + LocalResource rsrc = LocalResource.newInstance( + ConverterUtils.getYarnUrlFromPath(hdfsPath), LocalResourceType.BTFILE, + LocalResourceVisibility.APPLICATION, 0, 0); + localResource.put(fileRemotePath, rsrc); + } + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/src/main/java/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/src/main/java/Client.java new file mode 100644 index 0000000..443c684 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-broadcast-demo/src/main/java/Client.java @@ -0,0 +1,194 @@ +package org.apache.hadoop.yarn.applications.broadcast; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.records.*; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.ConverterUtils; + +import java.io.IOException; +import java.util.*; + +public class Client { + /* system related */ + private YarnClient yarnClient; + private Configuration conf; + private FileSystem hdfs; + + + /* app related */ + private String appName = "demo"; + private ApplicationId appId; + private String appMasterJarLocalPath = "/hadoop/dsh.jar"; + private String appMasterJarRemotePath = "dsh.jar"; + private String dataLocalPath = "/hadoop/data"; + private String dataRemotePath = "data"; + + private int amMemory = 1024; + private CharSequence appMasterMainClass = "org.apache.hadoop.yarn.applications.broadcast.ApplicationMaster"; + private int amVCores = 1; + private int amPriority = 0; + private String amQueue = "default"; + private int amNumContainers = 1; + + /* am args */ + private String downloadType; + private String downloadBlockSize; + private String numContainers; + + + public Client() { + conf = new YarnConfiguration(); + yarnClient = YarnClient.createYarnClient(); + yarnClient.init(conf); + try { + hdfs = FileSystem.get(conf); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) throws Exception { + Client client = new Client(); + client.run(args); + } + + private void run(String[] args) throws Exception { + if (args.length != 5 || (!args[2].equals("hdfs") && !args[2].equals("torrent"))) { + System.out.println("need args: <#container>"); + System.exit(-1); + } + appMasterJarLocalPath = args[0]; + dataLocalPath = args[1]; + downloadType = args[2]; + downloadBlockSize = args[3]; + numContainers = args[4]; + + yarnClient.start(); + YarnClientApplication app = yarnClient.createApplication(); + GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); + appId = appResponse.getApplicationId(); + ApplicationSubmissionContext appContext = app + .getApplicationSubmissionContext(); + appContext.setKeepContainersAcrossApplicationAttempts(false); + appContext.setApplicationName(appName); + + /* local resources */ + Map localResources = new HashMap(); + addToLocalResources(appMasterJarLocalPath, appMasterJarRemotePath, localResources); + addToLocalResources(dataLocalPath, dataRemotePath, localResources); + + /* env var */ + Map env = new HashMap(); + int numHashThread; + numHashThread = Math.min(amMemory/Integer.parseInt(downloadBlockSize)-1, Runtime.getRuntime().availableProcessors()); + numHashThread = Math.max(1, numHashThread-1); + env.put("TTORRENT_HASHING_THREADS", String.valueOf(numHashThread)); + StringBuilder classPathEnv = new StringBuilder( + Environment.CLASSPATH.$$()).append( + ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); + for (String c : conf + .getStrings( + YarnConfiguration.YARN_APPLICATION_CLASSPATH, + YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { + classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); + classPathEnv.append(c.trim()); + } + env.put("CLASSPATH", classPathEnv.toString()); + System.out.println("******"); + System.out.println(conf.toString()); + System.out.println("******"); + + /* cmd to exec appMaster */ + Vector vargs = new Vector(30); + vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); + vargs.add("-Xmx" + amMemory + "m"); + vargs.add(appMasterMainClass); + /* + vargs.add("--container_memory " + amMemory); + vargs.add("--container_vcores " + amVCores); + vargs.add("--num_containers " + amNumContainers); + vargs.add("--priority " + amPriority); + */ + vargs.add("--appId " + appId); + vargs.add("--dtype " + downloadType); + vargs.add("--dsize " + downloadBlockSize); + vargs.add("--nc " + numContainers); + + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + + "/AppMaster.stdout"); + vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + + "/AppMaster.stderr"); + StringBuilder command = new StringBuilder(); + for (CharSequence str : vargs) { + command.append(str).append(" "); + } + List commands = new ArrayList(); + commands.add(command.toString()); + + /* container launch context */ + ContainerLaunchContext amContainer = ContainerLaunchContext + .newInstance(localResources, env, commands, null, null, null); + + /* app submission context */ + Resource capability = Resource.newInstance(amMemory, amVCores); + ResourceRequest rr = ResourceRequest.newInstance(Priority.newInstance(amPriority), "*", capability, amNumContainers, false); + appContext.setAMContainerResourceRequest(rr); + appContext.setAMContainerSpec(amContainer); + appContext.setQueue(amQueue); + + yarnClient.submitApplication(appContext); + + monitorApplication(); + } + + private void addToLocalResources(String fileLocalPath, + String fileRemotePath, Map localResource) throws Exception { + Path hdfsPath = new Path(hdfs.getHomeDirectory(), appName + "/" + appId + "/" + fileRemotePath); + Path localPath = new Path(fileLocalPath); + hdfs.copyFromLocalFile(localPath, hdfsPath); + FileStatus fileStatus = hdfs.getFileStatus(hdfsPath); + LocalResource rsrc = LocalResource.newInstance( + ConverterUtils.getYarnUrlFromPath(hdfsPath), LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION, fileStatus.getLen(), + fileStatus.getModificationTime()); + localResource.put(fileRemotePath, rsrc); + } + + private void monitorApplication() throws Exception { + while (true) { + Thread.sleep(1000); + ApplicationReport report = yarnClient.getApplicationReport(appId); + System.out.println("Got application report from ASM for" + + ", appId=" + appId.getId() + ", clientToAMToken=" + + report.getClientToAMToken() + ", appDiagnostics=" + + report.getDiagnostics() + ", appMasterHost=" + + report.getHost() + ", appQueue=" + report.getQueue() + + ", appMasterRpcPort=" + report.getRpcPort() + + ", appStartTime=" + report.getStartTime() + + ", yarnAppState=" + + report.getYarnApplicationState().toString() + + ", distributedFinalState=" + + report.getFinalApplicationStatus().toString() + + ", appTrackingUrl=" + report.getTrackingUrl() + + ", appUser=" + report.getUser()); + + YarnApplicationState state = report.getYarnApplicationState(); + if (state == YarnApplicationState.FINISHED) { + System.out.println("succeeded"); + return; + } else if (state == YarnApplicationState.KILLED + || state == YarnApplicationState.FAILED) { + System.out.println("failed"); + return; + } + } + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml index e8ae42b..66d471a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml @@ -37,6 +37,7 @@ hadoop-yarn-applications-distributedshell hadoop-yarn-applications-unmanaged-am-launcher + hadoop-yarn-applications-broadcast-demo diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/pom.xml new file mode 100644 index 0000000..0ab6675 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/pom.xml @@ -0,0 +1,129 @@ + + + + + hadoop-yarn-server + org.apache.hadoop + 2.9.0-SNAPSHOT + + 4.0.0 + org.apache.hadoop + hadoop-yarn-server-broadcast + 2.9.0-SNAPSHOT + Apache Hadoop YARN Server Broadcast + + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-hdfs + + + org.apache.hadoop + hadoop-yarn-common + + + org.mockito + mockito-all + 1.8.5 + + + + com.google.guava + guava + + + + commons-io + commons-io + + + org.simpleframework + simple + 4.1.21 + + + org.slf4j + slf4j-api + + + io.netty + netty + + + org.apache.httpcomponents + httpmime + 4.2.5 + + + junit + junit + 4.8.2 + test + + + org.apache.hadoop + hadoop-common + test-jar + test + + + + + + + maven-antrun-plugin + + + pre-site + + run + + + + + + + + + + + + org.apache.hadoop + hadoop-maven-plugins + + + maven-clean-plugin + + + + src/site/resources + + configuration.xsl + + false + + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BDecoder.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BDecoder.java new file mode 100644 index 0000000..16ed348 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BDecoder.java @@ -0,0 +1,323 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.bcodec; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.EOFException; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.input.AutoCloseInputStream; + + +/** + * B-encoding decoder. + * + *

+ * A b-encoded byte stream can represent byte arrays, numbers, lists and maps + * (dictionaries). This class implements a decoder of such streams into + * {@link BEValue}s. + *

+ * + *

+ * Inspired by Snark's implementation. + *

+ * + * @see B-encoding specification + */ +public class BDecoder { + + // The InputStream to BDecode. + private final InputStream in; + + // The last indicator read. + // Zero if unknown. + // '0'..'9' indicates a byte[]. + // 'i' indicates an Number. + // 'l' indicates a List. + // 'd' indicates a Map. + // 'e' indicates end of Number, List or Map (only used internally). + // -1 indicates end of stream. + // Call getNextIndicator to get the current value (will never return zero). + private int indicator = 0; + + /** + * Initializes a new BDecoder. + * + *

+ * Nothing is read from the given InputStream yet. + *

+ * + * @param in The input stream to read from. + */ + public BDecoder(InputStream in) { + this.in = in; + } + + /** + * Decode a B-encoded stream. + * + *

+ * Automatically instantiates a new BDecoder for the provided input stream + * and decodes its root member. + *

+ * + * @param in The input stream to read from. + */ + public static BEValue bdecode(InputStream in) throws IOException { + return new BDecoder(in).bdecode(); + } + + /** + * Decode a B-encoded byte buffer. + * + *

+ * Automatically instantiates a new BDecoder for the provided buffer and + * decodes its root member. + *

+ * + * @param data The {@link ByteBuffer} to read from. + */ + public static BEValue bdecode(ByteBuffer data) throws IOException { + return BDecoder.bdecode(new AutoCloseInputStream( + new ByteArrayInputStream(data.array()))); + } + + /** + * Returns what the next b-encoded object will be on the stream or -1 + * when the end of stream has been reached. + * + *

+ * Can return something unexpected (not '0' .. '9', 'i', 'l' or 'd') when + * the stream isn't b-encoded. + *

+ * + * This might or might not read one extra byte from the stream. + */ + private int getNextIndicator() throws IOException { + if (this.indicator == 0) { + this.indicator = in.read(); + } + return this.indicator; + } + + /** + * Gets the next indicator and returns either null when the stream + * has ended or b-decodes the rest of the stream and returns the + * appropriate BEValue encoded object. + */ + public BEValue bdecode() throws IOException { + if (this.getNextIndicator() == -1) + return null; + + if (this.indicator >= '0' && this.indicator <= '9') + return this.bdecodeBytes(); + else if (this.indicator == 'i') + return this.bdecodeNumber(); + else if (this.indicator == 'l') + return this.bdecodeList(); + else if (this.indicator == 'd') + return this.bdecodeMap(); + else + throw new InvalidBEncodingException + ("Unknown indicator '" + this.indicator + "'"); + } + + /** + * Returns the next b-encoded value on the stream and makes sure it is a + * byte array. + * + * @throws InvalidBEncodingException If it is not a b-encoded byte array. + */ + public BEValue bdecodeBytes() throws IOException { + int c = this.getNextIndicator(); + int num = c - '0'; + if (num < 0 || num > 9) + throw new InvalidBEncodingException("Number expected, not '" + + (char)c + "'"); + this.indicator = 0; + + c = this.read(); + int i = c - '0'; + while (i >= 0 && i <= 9) { + // This can overflow! + num = num*10 + i; + c = this.read(); + i = c - '0'; + } + + if (c != ':') { + throw new InvalidBEncodingException("Colon expected, not '" + + (char)c + "'"); + } + + return new BEValue(read(num)); + } + + /** + * Returns the next b-encoded value on the stream and makes sure it is a + * number. + * + * @throws InvalidBEncodingException If it is not a number. + */ + public BEValue bdecodeNumber() throws IOException { + int c = this.getNextIndicator(); + if (c != 'i') { + throw new InvalidBEncodingException("Expected 'i', not '" + + (char)c + "'"); + } + this.indicator = 0; + + c = this.read(); + if (c == '0') { + c = this.read(); + if (c == 'e') + return new BEValue(BigInteger.ZERO); + else + throw new InvalidBEncodingException("'e' expected after zero," + + " not '" + (char)c + "'"); + } + + // We don't support more the 255 char big integers + char[] chars = new char[256]; + int off = 0; + + if (c == '-') { + c = this.read(); + if (c == '0') + throw new InvalidBEncodingException("Negative zero not allowed"); + chars[off] = '-'; + off++; + } + + if (c < '1' || c > '9') + throw new InvalidBEncodingException("Invalid Integer start '" + + (char)c + "'"); + chars[off] = (char)c; + off++; + + c = this.read(); + int i = c - '0'; + while (i >= 0 && i <= 9) { + chars[off] = (char)c; + off++; + c = read(); + i = c - '0'; + } + + if (c != 'e') + throw new InvalidBEncodingException("Integer should end with 'e'"); + + String s = new String(chars, 0, off); + return new BEValue(new BigInteger(s)); + } + + /** + * Returns the next b-encoded value on the stream and makes sure it is a + * list. + * + * @throws InvalidBEncodingException If it is not a list. + */ + public BEValue bdecodeList() throws IOException { + int c = this.getNextIndicator(); + if (c != 'l') { + throw new InvalidBEncodingException("Expected 'l', not '" + + (char)c + "'"); + } + this.indicator = 0; + + List result = new ArrayList(); + c = this.getNextIndicator(); + while (c != 'e') { + result.add(this.bdecode()); + c = this.getNextIndicator(); + } + this.indicator = 0; + + return new BEValue(result); + } + + /** + * Returns the next b-encoded value on the stream and makes sure it is a + * map (dictionary). + * + * @throws InvalidBEncodingException If it is not a map. + */ + public BEValue bdecodeMap() throws IOException { + int c = this.getNextIndicator(); + if (c != 'd') { + throw new InvalidBEncodingException("Expected 'd', not '" + + (char)c + "'"); + } + this.indicator = 0; + + Map result = new HashMap(); + c = this.getNextIndicator(); + while (c != 'e') { + // Dictionary keys are always strings. + String key = this.bdecode().getString(); + + BEValue value = this.bdecode(); + result.put(key, value); + + c = this.getNextIndicator(); + } + this.indicator = 0; + + return new BEValue(result); + } + + /** + * Returns the next byte read from the InputStream (as int). + * + * @throws EOFException If InputStream.read() returned -1. + */ + private int read() throws IOException { + int c = this.in.read(); + if (c == -1) + throw new EOFException(); + return c; + } + + /** + * Returns a byte[] containing length valid bytes starting at offset zero. + * + * @throws EOFException If InputStream.read() returned -1 before all + * requested bytes could be read. Note that the byte[] returned might be + * bigger then requested but will only contain length valid bytes. The + * returned byte[] will be reused when this method is called again. + */ + private byte[] read(int length) throws IOException { + byte[] result = new byte[length]; + + int read = 0; + while (read < length) + { + int i = this.in.read(result, read, length - read); + if (i == -1) + throw new EOFException(); + read += i; + } + + return result; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BEValue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BEValue.java new file mode 100644 index 0000000..232acfe --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BEValue.java @@ -0,0 +1,181 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.bcodec; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * A type-agnostic container for B-encoded values. + * + */ +public class BEValue { + + /** + * The B-encoded value can be a byte array, a Number, a List or a Map. + * Lists and Maps contains BEValues too. + */ + private final Object value; + + public BEValue(byte[] value) { + this.value = value; + } + + public BEValue(String value) throws UnsupportedEncodingException { + this.value = value.getBytes("UTF-8"); + } + + public BEValue(String value, String enc) + throws UnsupportedEncodingException { + this.value = value.getBytes(enc); + } + + public BEValue(int value) { + this.value = new Integer(value); + } + + public BEValue(long value) { + this.value = new Long(value); + } + + public BEValue(Number value) { + this.value = value; + } + + public BEValue(List value) { + this.value = value; + } + + public BEValue(Map value) { + this.value = value; + } + + public Object getValue() { + return this.value; + } + + /** + * Returns this BEValue as a String, interpreted as UTF-8. + * @throws InvalidBEncodingException If the value is not a byte[]. + */ + public String getString() throws InvalidBEncodingException { + return this.getString("UTF-8"); + } + + /** + * Returns this BEValue as a String, interpreted with the specified + * encoding. + * + * @param encoding The encoding to interpret the bytes as when converting + * them into a {@link String}. + * @throws InvalidBEncodingException If the value is not a byte[]. + */ + public String getString(String encoding) throws InvalidBEncodingException { + try { + return new String(this.getBytes(), encoding); + } catch (ClassCastException cce) { + throw new InvalidBEncodingException(cce.toString()); + } catch (UnsupportedEncodingException uee) { + throw new InternalError(uee.toString()); + } + } + + /** + * Returns this BEValue as a byte[]. + * + * @throws InvalidBEncodingException If the value is not a byte[]. + */ + public byte[] getBytes() throws InvalidBEncodingException { + try { + return (byte[])this.value; + } catch (ClassCastException cce) { + throw new InvalidBEncodingException(cce.toString()); + } + } + + /** + * Returns this BEValue as a Number. + * + * @throws InvalidBEncodingException If the value is not a {@link Number}. + */ + public Number getNumber() throws InvalidBEncodingException { + try { + return (Number)this.value; + } catch (ClassCastException cce) { + throw new InvalidBEncodingException(cce.toString()); + } + } + + /** + * Returns this BEValue as short. + * + * @throws InvalidBEncodingException If the value is not a {@link Number}. + */ + public short getShort() throws InvalidBEncodingException { + return this.getNumber().shortValue(); + } + + /** + * Returns this BEValue as int. + * + * @throws InvalidBEncodingException If the value is not a {@link Number}. + */ + public int getInt() throws InvalidBEncodingException { + return this.getNumber().intValue(); + } + + /** + * Returns this BEValue as long. + * + * @throws InvalidBEncodingException If the value is not a {@link Number}. + */ + public long getLong() throws InvalidBEncodingException { + return this.getNumber().longValue(); + } + + /** + * Returns this BEValue as a List of BEValues. + * + * @throws InvalidBEncodingException If the value is not an + * {@link ArrayList}. + */ + @SuppressWarnings("unchecked") + public List getList() throws InvalidBEncodingException { + if (this.value instanceof ArrayList) { + return (ArrayList)this.value; + } else { + throw new InvalidBEncodingException("Excepted List !"); + } + } + + /** + * Returns this BEValue as a Map of String keys and BEValue values. + * + * @throws InvalidBEncodingException If the value is not a {@link HashMap}. + */ + @SuppressWarnings("unchecked") + public Map getMap() throws InvalidBEncodingException { + if (this.value instanceof HashMap) { + return (Map)this.value; + } else { + throw new InvalidBEncodingException("Expected Map !"); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BEncoder.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BEncoder.java new file mode 100644 index 0000000..779386c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/BEncoder.java @@ -0,0 +1,121 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.bcodec; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * B-encoding encoder. + * + *

+ * This class provides utility methods to encode objects and + * {@link BEValue}s to B-encoding into a provided output stream. + *

+ * + *

+ * Inspired by Snark's implementation. + *

+ * + * @see B-encoding specification + */ +public class BEncoder { + + @SuppressWarnings("unchecked") + public static void bencode(Object o, OutputStream out) + throws IOException, IllegalArgumentException { + if (o instanceof BEValue) { + o = ((BEValue)o).getValue(); + } + + if (o instanceof String) { + bencode((String)o, out); + } else if (o instanceof byte[]) { + bencode((byte[])o, out); + } else if (o instanceof Number) { + bencode((Number)o, out); + } else if (o instanceof List) { + bencode((List)o, out); + } else if (o instanceof Map) { + bencode((Map)o, out); + } else { + throw new IllegalArgumentException("Cannot bencode: " + + o.getClass()); + } + } + + public static void bencode(String s, OutputStream out) throws IOException { + byte[] bs = s.getBytes("UTF-8"); + bencode(bs, out); + } + + public static void bencode(Number n, OutputStream out) throws IOException { + out.write('i'); + String s = n.toString(); + out.write(s.getBytes("UTF-8")); + out.write('e'); + } + + public static void bencode(List l, OutputStream out) + throws IOException { + out.write('l'); + for (BEValue value : l) { + bencode(value, out); + } + out.write('e'); + } + + public static void bencode(byte[] bs, OutputStream out) throws IOException { + String l = Integer.toString(bs.length); + out.write(l.getBytes("UTF-8")); + out.write(':'); + out.write(bs); + } + + public static void bencode(Map m, OutputStream out) + throws IOException { + out.write('d'); + + // Keys must be sorted. + Set s = m.keySet(); + List l = new ArrayList(s); + Collections.sort(l); + + for (String key : l) { + Object value = m.get(key); + bencode(key, out); + bencode(value, out); + } + + out.write('e'); + } + + public static ByteBuffer bencode(Map m) + throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BEncoder.bencode(m, baos); + baos.close(); + return ByteBuffer.wrap(baos.toByteArray()); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/InvalidBEncodingException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/InvalidBEncodingException.java new file mode 100644 index 0000000..21a493b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/bcodec/InvalidBEncodingException.java @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.broadcast.bcodec; + +import java.io.IOException; + + +/** + * Exception thrown when a B-encoded stream cannot be decoded. + * + */ +public class InvalidBEncodingException extends IOException { + + public static final long serialVersionUID = -1; + + public InvalidBEncodingException(String message) { + super(message); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Client.java new file mode 100644 index 0000000..8382407 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Client.java @@ -0,0 +1,990 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.client.announce.Announce; +import org.apache.hadoop.yarn.server.broadcast.client.announce.AnnounceException; +import org.apache.hadoop.yarn.server.broadcast.client.announce.AnnounceResponseListener; +import org.apache.hadoop.yarn.server.broadcast.client.peer.PeerActivityListener; +import org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * A pure-java BitTorrent client. + * + *

+ * A BitTorrent client in its bare essence shares a given torrent. If the + * torrent is not complete locally, it will continue to download it. If or + * after the torrent is complete, the client may eventually continue to seed it + * for other clients. + *

+ * + *

+ * This BitTorrent client implementation is made to be simple to embed and + * simple to use. First, initialize a ShareTorrent object from a torrent + * meta-info source (either a file or a byte array, see + * com.turn.ttorrent.SharedTorrent for how to create a SharedTorrent object). + * Then, instantiate your Client object with this SharedTorrent and call one of + * {@link #download} to simply download the torrent, or {@link #share} to + * download and continue seeding for the given amount of time after the + * download completes. + *

+ * + */ +public class Client extends Observable implements Runnable, + AnnounceResponseListener, IncomingConnectionListener, PeerActivityListener { + + private static final Logger logger = + LoggerFactory.getLogger(Client.class); + + /** Peers unchoking frequency, in seconds. Current BitTorrent specification + * recommends 10 seconds to avoid choking fibrilation. */ + private static final int UNCHOKING_FREQUENCY = 3; + + /** Optimistic unchokes are done every 2 loop iterations, i.e. every + * 2*UNCHOKING_FREQUENCY seconds. */ + private static final int OPTIMISTIC_UNCHOKE_ITERATIONS = 3; + + private static final int RATE_COMPUTATION_ITERATIONS = 2; + private static final int MAX_DOWNLOADERS_UNCHOKE = 4; + + public Client(InetAddress address, SharedTorrent sharedTorrent) throws IOException { + this.torrent = sharedTorrent; + this.state = ClientState.WAITING; + + String id = Client.BITTORRENT_ID_PREFIX + UUID.randomUUID() + .toString().split("-")[4]; + + // Initialize the incoming connection handler and register ourselves to + // it. + this.service = new ConnectionHandler(this.torrent, id, address); + this.service.register(this); + + this.self = new Peer( + this.service.getSocketAddress() + .getAddress().getHostAddress(), + (short)this.service.getSocketAddress().getPort(), + ByteBuffer.wrap(id.getBytes(Torrent.BYTE_ENCODING))); + + // Initialize the announce request thread, and register ourselves to it + // as well. + this.announce = new Announce(this.torrent, this.self); + this.announce.register(this); + + logger.info("BitTorrent client [{}] for {} started and " + + "listening at {}:{}...", + new Object[] { + this.self.getShortHexPeerId(), + this.torrent.getName(), + this.self.getIp(), + this.self.getPort() + }); + + this.peers = new ConcurrentHashMap(); + this.connected = new ConcurrentHashMap(); + this.random = new Random(System.currentTimeMillis()); + } + + + public enum ClientState { + WAITING, + VALIDATING, + SHARING, + SEEDING, + ERROR, + DONE + } + + private static final String BITTORRENT_ID_PREFIX = "-TO0042-"; + + private SharedTorrent torrent; + private ClientState state; + private Peer self; + + private Thread thread; + private boolean stop; + private long seed; + + private ConnectionHandler service; + private Announce announce; + private ConcurrentMap peers; + private ConcurrentMap connected; + + private Random random; + + + /** + * Initialize the BitTorrent client. + * + * @param address The address to bind to + * @param torrentPath path to torrent file + * @param dirPath path to the dir of future data file + */ + public Client(InetAddress address, String torrentPath, String dirPath) throws IOException { + this(address, SharedTorrent.fromFile(new File(torrentPath), new File(dirPath).getCanonicalFile())); + } + + public Client(InetAddress address, byte[] torrentData, String dirPath) throws IOException { + this(address, new SharedTorrent(torrentData, new File(dirPath))); + } + + /** + * Set the maximum download rate (in kB/second) for this + * torrent. A setting of <= 0.0 disables rate limiting. + * + * @param rate The maximum download rate + */ + public void setMaxDownloadRate(double rate) { + this.torrent.setMaxDownloadRate(rate); + } + + /** + * Set the maximum upload rate (in kB/second) for this + * torrent. A setting of <= 0.0 disables rate limiting. + * + * @param rate The maximum upload rate + */ + public void setMaxUploadRate(double rate) { + this.torrent.setMaxUploadRate(rate); + } + + /** + * Get this client's peer specification. + */ + public Peer getPeerSpec() { + return this.self; + } + + /** + * Return the torrent this client is exchanging on. + */ + public SharedTorrent getTorrent() { + return this.torrent; + } + + /** + * Returns the set of known peers. + */ + public Set getPeers() { + return new HashSet(this.peers.values()); + } + + /** + * Change this client's state and notify its observers. + * + *

+ * If the state has changed, this client's observers will be notified. + *

+ * + * @param state The new client state. + */ + private synchronized void setState(ClientState state) { + if (this.state != state) { + this.setChanged(); + } + this.state = state; + this.notifyObservers(this.state); + } + + /** + * Return the current state of this BitTorrent client. + */ + public ClientState getState() { + return this.state; + } + + /** + * Download the torrent without seeding after completion. + */ + public void download() { + this.share(0); + } + + /** + * Download and share this client's torrent until interrupted. + */ + public void share() { + this.share(-1); + } + + /** + * Download and share this client's torrent. + * + * @param seed Seed time in seconds after the download is complete. Pass + * 0 to immediately stop after downloading. + */ + public synchronized void share(int seed) { + this.seed = seed; + this.stop = false; + + if (this.thread == null || !this.thread.isAlive()) { + this.thread = new Thread(this); + this.thread.setName("bt-client(" + + this.self.getShortHexPeerId() + ")"); + this.thread.setDaemon(true); + this.thread.start(); + } + } + + /** + * Immediately but gracefully stop this client. + */ + public void stop() { + this.stop(true); + } + + /** + * Immediately but gracefully stop this client. + * + * @param wait Whether to wait for the client execution thread to complete + * or not. This allows for the client's state to be settled down in one of + * the DONE or ERROR states when this method returns. + */ + public void stop(boolean wait) { + this.stop = true; + + if (this.thread != null && this.thread.isAlive()) { + this.thread.interrupt(); + if (wait) { + this.waitForCompletion(); + } + } + + this.thread = null; + } + + /** + * Wait for downloading (and seeding, if requested) to complete. + */ + public void waitForCompletion() { + if (this.thread != null && this.thread.isAlive()) { + try { + this.thread.join(); + } catch (InterruptedException ie) { + logger.error(ie.getMessage(), ie); + } + } + } + + /** + * Tells whether we are a seed for the torrent we're sharing. + */ + public boolean isSeed() { + return this.torrent.isComplete(); + } + + /** + * Main client loop. + * + *

+ * The main client download loop is very simple: it starts the announce + * request thread, the incoming connection handler service, and loops + * unchoking peers every UNCHOKING_FREQUENCY seconds until told to stop. + * Every OPTIMISTIC_UNCHOKE_ITERATIONS, an optimistic unchoke will be + * attempted to try out other peers. + *

+ * + *

+ * Once done, it stops the announce and connection services, and returns. + *

+ */ + @Override + public void run() { + // First, analyze the torrent's local data. + try { + this.setState(ClientState.VALIDATING); + this.torrent.init(); + } catch (IOException ioe) { + logger.warn("Error while initializing torrent data: {}!", + ioe.getMessage(), ioe); + } catch (InterruptedException ie) { + logger.warn("Client was interrupted during initialization. " + + "Aborting right away."); + } finally { + if (!this.torrent.isInitialized()) { + try { + this.service.close(); + } catch (IOException ioe) { + logger.warn("Error while releasing bound channel: {}!", + ioe.getMessage(), ioe); + } + + this.setState(ClientState.ERROR); + this.torrent.close(); + return; + } + } + + // Initial completion test + if (this.torrent.isComplete()) { + this.seed(); + } else { + this.setState(ClientState.SHARING); + } + + // Detect early stop + if (this.stop) { + logger.info("Download is complete and no seeding was requested."); + this.finish(); + return; + } + + this.announce.start(); + this.service.start(); + + int optimisticIterations = 0; + int rateComputationIterations = 0; + + while (!this.stop) { + optimisticIterations = + (optimisticIterations == 0 ? + Client.OPTIMISTIC_UNCHOKE_ITERATIONS : + optimisticIterations - 1); + + rateComputationIterations = + (rateComputationIterations == 0 ? + Client.RATE_COMPUTATION_ITERATIONS : + rateComputationIterations - 1); + + try { + this.unchokePeers(optimisticIterations == 0); + this.info(); + if (rateComputationIterations == 0) { + this.resetPeerRates(); + } + } catch (Exception e) { + logger.error("An exception occurred during the BitTorrent " + + "client main loop execution!", e); + } + + try { + Thread.sleep(Client.UNCHOKING_FREQUENCY*1000); + } catch (InterruptedException ie) { + logger.trace("BitTorrent main loop interrupted."); + } + } + + logger.debug("Stopping BitTorrent client connection service " + + "and announce threads..."); + + this.service.stop(); + try { + this.service.close(); + } catch (IOException ioe) { + logger.warn("Error while releasing bound channel: {}!", + ioe.getMessage(), ioe); + } + + this.announce.stop(); + + // Close all peer connections + logger.debug("Closing all remaining peer connections..."); + for (SharingPeer peer : this.connected.values()) { + peer.unbind(false); + } + + this.finish(); + } + + /** + * Close torrent and set final client state before signing off. + */ + private void finish() { + this.torrent.close(); + + // Determine final state + if (this.torrent.isFinished()) { + this.setState(ClientState.DONE); + } else { + this.setState(ClientState.ERROR); + } + + logger.info("BitTorrent client signing off."); + } + + /** + * Display information about the BitTorrent client state. + * + *

+ * This emits an information line in the log about this client's state. It + * includes the number of choked peers, number of connected peers, number + * of known peers, information about the torrent availability and + * completion and current transmission rates. + *

+ */ + public synchronized void info() { + float dl = 0; + float ul = 0; + for (SharingPeer peer : this.connected.values()) { + dl += peer.getDLRate().get(); + ul += peer.getULRate().get(); + } + + logger.info("{} {}/{} pieces ({}%) [{}/{}] with {}/{} peers at {}/{} kB/s.", + new Object[] { + this.getState().name(), + this.torrent.getCompletedPieces().cardinality(), + this.torrent.getPieceCount(), + String.format("%.2f", this.torrent.getCompletion()), + this.torrent.getAvailablePieces().cardinality(), + this.torrent.getRequestedPieces().cardinality(), + this.connected.size(), + this.peers.size(), + String.format("%.2f", dl/1024.0), + String.format("%.2f", ul/1024.0), + }); + for (SharingPeer peer : this.connected.values()) { + Piece piece = peer.getRequestedPiece(); + logger.debug(" | {} {}", + peer, + piece != null + ? "(downloading " + piece + ")" + : "" + ); + } + } + + /** + * Reset peers download and upload rates. + * + *

+ * This method is called every RATE_COMPUTATION_ITERATIONS to reset the + * download and upload rates of all peers. This contributes to making the + * download and upload rate computations rolling averages every + * UNCHOKING_FREQUENCY * RATE_COMPUTATION_ITERATIONS seconds (usually 20 + * seconds). + *

+ */ + private synchronized void resetPeerRates() { + for (SharingPeer peer : this.connected.values()) { + peer.getDLRate().reset(); + peer.getULRate().reset(); + } + } + + /** + * Retrieve a SharingPeer object from the given peer specification. + * + *

+ * This function tries to retrieve an existing peer object based on the + * provided peer specification or otherwise instantiates a new one and adds + * it to our peer repository. + *

+ * + * @param search The {@link Peer} specification. + */ + private SharingPeer getOrCreatePeer(Peer search) { + SharingPeer peer; + + synchronized (this.peers) { + logger.trace("Searching for {}...", search); + if (search.hasPeerId()) { + peer = this.peers.get(search.getHexPeerId()); + if (peer != null) { + logger.trace("Found peer (by peer ID): {}.", peer); + this.peers.put(peer.getHostIdentifier(), peer); + this.peers.put(search.getHostIdentifier(), peer); + return peer; + } + } + + peer = this.peers.get(search.getHostIdentifier()); + if (peer != null) { + if (search.hasPeerId()) { + logger.trace("Recording peer ID {} for {}.", + search.getHexPeerId(), peer); + peer.setPeerId(search.getPeerId()); + this.peers.put(search.getHexPeerId(), peer); + } + + logger.debug("Found peer (by host ID): {}.", peer); + return peer; + } + + peer = new SharingPeer(search.getIp(), search.getPort(), + search.getPeerId(), this.torrent); + logger.trace("Created new peer: {}.", peer); + + this.peers.put(peer.getHostIdentifier(), peer); + if (peer.hasPeerId()) { + this.peers.put(peer.getHexPeerId(), peer); + } + + return peer; + } + } + + /** + * Retrieve a peer comparator. + * + *

+ * Returns a peer comparator based on either the download rate or the + * upload rate of each peer depending on our state. While sharing, we rely + * on the download rate we get from each peer. When our download is + * complete and we're only seeding, we use the upload rate instead. + *

+ * + * @return A SharingPeer comparator that can be used to sort peers based on + * the download or upload rate we get from them. + */ + private Comparator getPeerRateComparator() { + if (ClientState.SHARING.equals(this.state)) { + return new SharingPeer.DLRateComparator(); + } else if (ClientState.SEEDING.equals(this.state)) { + return new SharingPeer.ULRateComparator(); + } else { + throw new IllegalStateException("Client is neither sharing nor " + + "seeding, we shouldn't be comparing peers at this point."); + } + } + + /** + * Unchoke connected peers. + * + *

+ * This is one of the "clever" places of the BitTorrent client. Every + * OPTIMISTIC_UNCHOKING_FREQUENCY seconds, we decide which peers should be + * unchocked and authorized to grab pieces from us. + *

+ * + *

+ * Reciprocation (tit-for-tat) and upload capping is implemented here by + * carefully choosing which peers we unchoke, and which peers we choke. + *

+ * + *

+ * The four peers with the best download rate and are interested in us get + * unchoked. This maximizes our download rate as we'll be able to get data + * from there four "best" peers quickly, while allowing these peers to + * download from us and thus reciprocate their generosity. + *

+ * + *

+ * Peers that have a better download rate than these four downloaders but + * are not interested get unchoked too, we want to be able to download from + * them to get more data more quickly. If one becomes interested, it takes + * a downloader's place as one of the four top downloaders (i.e. we choke + * the downloader with the worst upload rate). + *

+ * + * @param optimistic Whether to perform an optimistic unchoke as well. + */ + private synchronized void unchokePeers(boolean optimistic) { + // Build a set of all connected peers, we don't care about peers we're + // not connected to. + TreeSet bound = new TreeSet( + this.getPeerRateComparator()); + bound.addAll(this.connected.values()); + + if (bound.size() == 0) { + logger.trace("No connected peers, skipping unchoking."); + return; + } else { + logger.trace("Running unchokePeers() on {} connected peers.", + bound.size()); + } + + int downloaders = 0; + Set choked = new HashSet(); + + // We're interested in the top downloaders first, so use a descending + // set. + for (SharingPeer peer : bound.descendingSet()) { + if (downloaders < Client.MAX_DOWNLOADERS_UNCHOKE) { + // Unchoke up to MAX_DOWNLOADERS_UNCHOKE interested peers + if (peer.isChoking()) { + if (peer.isInterested()) { + downloaders++; + } + + peer.unchoke(); + } + } else { + // Choke everybody else + choked.add(peer); + } + } + + // Actually choke all chosen peers (if any), except the eventual + // optimistic unchoke. + if (choked.size() > 0) { + SharingPeer randomPeer = choked.toArray( + new SharingPeer[0])[this.random.nextInt(choked.size())]; + + for (SharingPeer peer : choked) { + if (optimistic && peer == randomPeer) { + logger.debug("Optimistic unchoke of {}.", peer); + continue; + } + + peer.choke(); + } + } + } + + + /** AnnounceResponseListener handler(s). **********************************/ + + /** + * Handle an announce response event. + * + * @param interval The announce interval requested by the tracker. + * @param complete The number of seeders on this torrent. + * @param incomplete The number of leechers on this torrent. + */ + @Override + public void handleAnnounceResponse(int interval, int complete, + int incomplete) { + this.announce.setInterval(interval); + } + + /** + * Handle the discovery of new peers. + * + * @param peers The list of peers discovered (from the announce response or + * any other means like DHT/PEX, etc.). + */ + @Override + public void handleDiscoveredPeers(List peers) { + if (peers == null || peers.isEmpty()) { + // No peers returned by the tracker. Apparently we're alone on + // this one for now. + return; + } + + logger.info("Got {} peer(s) in tracker response.", peers.size()); + + if (!this.service.isAlive()) { + logger.warn("Connection handler service is not available."); + return; + } + + for (Peer peer : peers) { + // Attempt to connect to the peer if and only if: + // - We're not already connected or connecting to it; + // - We're not a seeder (we leave the responsibility + // of connecting to peers that need to download + // something). + SharingPeer match = this.getOrCreatePeer(peer); + if (this.isSeed()) { + continue; + } + + synchronized (match) { + if (!match.isConnected()) { + this.service.connect(match); + } + } + } + } + + + /** IncomingConnectionListener handler(s). ********************************/ + + /** + * Handle a new peer connection. + * + *

+ * This handler is called once the connection has been successfully + * established and the handshake exchange made. This generally simply means + * binding the peer to the socket, which will put in place the communication + * thread and logic with this peer. + *

+ * + * @param channel The connected socket channel to the remote peer. Note + * that if the peer somehow rejected our handshake reply, this socket might + * very soon get closed, but this is handled down the road. + * @param peerId The byte-encoded peerId extracted from the peer's + * handshake, after validation. + * @see org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer + */ + @Override + public void handleNewPeerConnection(SocketChannel channel, byte[] peerId) { + Peer search = new Peer( + channel.socket().getInetAddress().getHostAddress(), + channel.socket().getPort(), + (peerId != null + ? ByteBuffer.wrap(peerId) + : null)); + + logger.info("Handling new peer connection with {}...", search); + SharingPeer peer = this.getOrCreatePeer(search); + + try { + synchronized (peer) { + if (peer.isConnected()) { + logger.info("Already connected with {}, closing link.", + peer); + channel.close(); + return; + } + + peer.register(this); + peer.bind(channel); + } + + this.connected.put(peer.getHexPeerId(), peer); + peer.register(this.torrent); + logger.debug("New peer connection with {} [{}/{}].", + new Object[] { + peer, + this.connected.size(), + this.peers.size() + }); + } catch (Exception e) { + this.connected.remove(peer.getHexPeerId()); + logger.warn("Could not handle new peer connection " + + "with {}: {}", peer, e.getMessage()); + } + } + + /** + * Handle a failed peer connection. + * + *

+ * If an outbound connection failed (could not connect, invalid handshake, + * etc.), remove the peer from our known peers. + *

+ * + * @param peer The peer we were trying to connect with. + * @param cause The exception encountered when connecting with the peer. + */ + @Override + public void handleFailedConnection(SharingPeer peer, Throwable cause) { + logger.warn("Could not connect to {}: {}.", peer, cause.getMessage()); + this.peers.remove(peer.getHostIdentifier()); + if (peer.hasPeerId()) { + this.peers.remove(peer.getHexPeerId()); + } + } + + /** PeerActivityListener handler(s). **************************************/ + + @Override + public void handlePeerChoked(SharingPeer peer) { /* Do nothing */ } + + @Override + public void handlePeerReady(SharingPeer peer) { /* Do nothing */ } + + @Override + public void handlePieceAvailability(SharingPeer peer, + Piece piece) { /* Do nothing */ } + + @Override + public void handleBitfieldAvailability(SharingPeer peer, + BitSet availablePieces) { /* Do nothing */ } + + @Override + public void handlePieceSent(SharingPeer peer, + Piece piece) { /* Do nothing */ } + + /** + * Piece download completion handler. + * + *

+ * When a piece is completed, and valid, we announce to all connected peers + * that we now have this piece. + *

+ * + *

+ * We use this handler to identify when all of the pieces have been + * downloaded. When that's the case, we can start the seeding period, if + * any. + *

+ * + * @param peer The peer we got the piece from. + * @param piece The piece in question. + */ + @Override + public void handlePieceCompleted(SharingPeer peer, Piece piece) + throws IOException { + synchronized (this.torrent) { + if (piece.isValid()) { + // Make sure the piece is marked as completed in the torrent + // Note: this is required because the order the + // PeerActivityListeners are called is not defined, and we + // might be called before the torrent's piece completion + // handler is. + this.torrent.markCompleted(piece); + logger.debug("Completed download of {} from {}. " + + "We now have {}/{} pieces", + new Object[] { + piece, + peer, + this.torrent.getCompletedPieces().cardinality(), + this.torrent.getPieceCount() + }); + + // Send a HAVE message to all connected peers + PeerMessage have = PeerMessage.HaveMessage.craft(piece.getIndex()); + for (SharingPeer remote : this.connected.values()) { + remote.send(have); + } + + // Force notify after each piece is completed to propagate download + // completion information (or new seeding state) + this.setChanged(); + this.notifyObservers(this.state); + } else { + logger.warn("Downloaded piece#{} from {} was not valid ;-(", + piece.getIndex(), peer); + } + + if (this.torrent.isComplete()) { + logger.info("Last piece validated and completed, finishing download..."); + + // Cancel all remaining outstanding requests + for (SharingPeer remote : this.connected.values()) { + if (remote.isDownloading()) { + int requests = remote.cancelPendingRequests().size(); + logger.info("Cancelled {} remaining pending requests on {}.", + requests, remote); + } + } + + this.torrent.finish(); + + try { + this.announce.getCurrentTrackerClient() + .announce(TrackerMessage + .AnnounceRequestMessage + .RequestEvent.COMPLETED, true); + } catch (AnnounceException ae) { + logger.warn("Error announcing completion event to " + + "tracker: {}", ae.getMessage()); + } + + logger.info("Download is complete and finalized."); + this.seed(); + } + } + } + + @Override + public void handlePeerDisconnected(SharingPeer peer) { + if (this.connected.remove(peer.hasPeerId() + ? peer.getHexPeerId() + : peer.getHostIdentifier()) != null) { + logger.debug("Peer {} disconnected, [{}/{}].", + new Object[] { + peer, + this.connected.size(), + this.peers.size() + }); + } + + peer.reset(); + } + + @Override + public void handleIOException(SharingPeer peer, IOException ioe) { + logger.warn("I/O error while exchanging data with {}, " + + "closing connection with it!", peer, ioe.getMessage()); + peer.unbind(true); + } + + + /** Post download seeding. ************************************************/ + + /** + * Start the seeding period, if any. + * + *

+ * This method is called when all the pieces of our torrent have been + * retrieved. This may happen immediately after the client starts if the + * torrent was already fully download or we are the initial seeder client. + *

+ * + *

+ * When the download is complete, the client switches to seeding mode for + * as long as requested in the share() call, if seeding was + * requested. If not, the {@link ClientShutdown} will execute + * immediately to stop the client's main loop. + *

+ * + * @see ClientShutdown + */ + private synchronized void seed() { + // Silently ignore if we're already seeding. + if (ClientState.SEEDING.equals(this.getState())) { + return; + } + + logger.info("Download of {} pieces completed.", + this.torrent.getPieceCount()); + + this.setState(ClientState.SEEDING); + if (this.seed < 0) { + logger.info("Seeding indefinetely..."); + return; + } + + // In case seeding for 0 seconds we still need to schedule the task in + // order to call stop() from different thread to avoid deadlock + logger.info("Seeding for {} seconds...", this.seed); + Timer timer = new Timer(); + timer.schedule(new ClientShutdown(this, timer), this.seed*1000); + } + + /** + * Timer task to stop seeding. + * + *

+ * This TimerTask will be called by a timer set after the download is + * complete to stop seeding from this client after a certain amount of + * requested seed time (might be 0 for immediate termination). + *

+ * + *

+ * This task simply contains a reference to this client instance and calls + * its stop() method to interrupt the client's main loop. + *

+ * + * @author mpetazzoni + */ + public static class ClientShutdown extends TimerTask { + + private final Client client; + private final Timer timer; + + public ClientShutdown(Client client, Timer timer) { + this.client = client; + this.timer = timer; + } + + @Override + public void run() { + this.client.stop(); + if (this.timer != null) { + this.timer.cancel(); + } + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/ConnectionHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/ConnectionHandler.java new file mode 100644 index 0000000..922549a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/ConnectionHandler.java @@ -0,0 +1,516 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.ByteBuffer; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.text.ParseException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Incoming peer connections service. + * + *

+ * Every BitTorrent client, BitTorrent being a peer-to-peer protocol, listens + * on a httpPort for incoming connections from other peers sharing the same + * torrent. + *

+ * + *

+ * This ConnectionHandler implements this service and starts a listening socket + * in the first available httpPort in the default BitTorrent client httpPort range + * 6881-6889. When a peer connects to it, it expects the BitTorrent handshake + * message, parses it and replies with our own handshake. + *

+ * + *

+ * Outgoing connections to other peers are also made through this service, + * which handles the handshake procedure with the remote peer. Regardless of + * the direction of the connection, once this handshake is successful, all + * {@link IncomingConnectionListener}s are notified and passed the connected + * socket and the remote peer ID. + *

+ * + *

+ * This class does nothing more. All further peer-to-peer communication happens + * in the {@link org.apache.hadoop.yarn.server.broadcast.client.peer.PeerExchange PeerExchange} + * class. + *

+ * + * @see BitTorrent handshake specification + */ +public class ConnectionHandler implements Runnable { + + private static final Logger logger = + LoggerFactory.getLogger(ConnectionHandler.class); + + public static final int PORT_RANGE_START = 30000; + public static final int PORT_RANGE_END = 40000; + + private static final int OUTBOUND_CONNECTIONS_POOL_SIZE = 20; + private static final int OUTBOUND_CONNECTIONS_THREAD_KEEP_ALIVE_SECS = 10; + + private static final int CLIENT_KEEP_ALIVE_MINUTES = 3; + + private SharedTorrent torrent; + private String id; + private ServerSocketChannel channel; + private InetSocketAddress address; + + private Set listeners; + private ExecutorService executor; + private Thread thread; + private boolean stop; + + /** + * Create and start a new listening service for out torrent, reporting + * with our peer ID on the given address. + * + *

+ * This binds to the first available httpPort in the client httpPort range + * PORT_RANGE_START to PORT_RANGE_END. + *

+ * + * @param torrent The torrent shared by this client. + * @param id This client's peer ID. + * @param address The address to bind to. + * @throws IOException When the service can't be started because no httpPort in + * the defined range is available or usable. + */ + ConnectionHandler(SharedTorrent torrent, String id, InetAddress address) + throws IOException { + this.torrent = torrent; + this.id = id; + + // Bind to the first available httpPort in the range + // [PORT_RANGE_START; PORT_RANGE_END]. + for (int port = ConnectionHandler.PORT_RANGE_START; + port <= ConnectionHandler.PORT_RANGE_END; + port++) { + InetSocketAddress tryAddress = + new InetSocketAddress(address, port); + + try { + this.channel = ServerSocketChannel.open(); + this.channel.socket().bind(tryAddress); + this.channel.configureBlocking(false); + this.address = tryAddress; + break; + } catch (IOException ioe) { + // Ignore, try next httpPort + logger.warn("Could not bind to {}, trying next httpPort...", tryAddress); + } + } + + if (this.channel == null || !this.channel.socket().isBound()) { + throw new IOException("No available httpPort for the BitTorrent client!"); + } + + logger.info("Listening for incoming connections on {}.", this.address); + + this.listeners = new HashSet(); + this.executor = null; + this.thread = null; + } + + /** + * Return the full socket address this service is bound to. + */ + public InetSocketAddress getSocketAddress() { + return this.address; + } + + /** + * Register a new incoming connection listener. + * + * @param listener The listener who wants to receive connection + * notifications. + */ + public void register(IncomingConnectionListener listener) { + this.listeners.add(listener); + } + + /** + * Start accepting new connections in a background thread. + */ + public void start() { + if (this.channel == null) { + throw new IllegalStateException( + "Connection handler cannot be recycled!"); + } + + this.stop = false; + + if (this.executor == null || this.executor.isShutdown()) { + this.executor = new ThreadPoolExecutor( + OUTBOUND_CONNECTIONS_POOL_SIZE, + OUTBOUND_CONNECTIONS_POOL_SIZE, + OUTBOUND_CONNECTIONS_THREAD_KEEP_ALIVE_SECS, + TimeUnit.SECONDS, + new LinkedBlockingQueue(), + new ConnectorThreadFactory()); + } + + if (this.thread == null || !this.thread.isAlive()) { + this.thread = new Thread(this); + this.thread.setName("bt-serve"); + this.thread.start(); + } + } + + /** + * Stop accepting connections. + * + *

+ * Note: the underlying socket remains open and bound. + *

+ */ + public void stop() { + this.stop = true; + + if (this.thread != null && this.thread.isAlive()) { + try { + this.thread.join(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + } + + if (this.executor != null && !this.executor.isShutdown()) { + this.executor.shutdownNow(); + } + + this.executor = null; + this.thread = null; + } + + /** + * Close this connection handler to release the httpPort it is bound to. + * + * @throws IOException If the channel could not be closed. + */ + public void close() throws IOException { + if (this.channel != null) { + this.channel.close(); + this.channel = null; + } + } + + /** + * The main service loop. + * + *

+ * The service waits for new connections for 250ms, then waits 100ms so it + * can be interrupted. + *

+ */ + @Override + public void run() { + while (!this.stop) { + try { + SocketChannel client = this.channel.accept(); + if (client != null) { + this.accept(client); + } + } catch (SocketTimeoutException ste) { + // Ignore and go back to sleep + } catch (IOException ioe) { + logger.warn("Unrecoverable error in connection handler", ioe); + this.stop(); + } + + try { + Thread.sleep(100); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * Return a human-readable representation of a connected socket channel. + * + * @param channel The socket channel to represent. + * @return A textual representation (host:httpPort) of the given + * socket. + */ + private String socketRepr(SocketChannel channel) { + Socket s = channel.socket(); + return String.format("%s:%d%s", + s.getInetAddress().getHostName(), + s.getPort(), + channel.isConnected() ? "+" : "-"); + } + + /** + * Accept the next incoming connection. + * + *

+ * When a new peer connects to this service, wait for it to send its + * handshake. We then parse and check that the handshake advertises the + * torrent hash we expect, then reply with our own handshake. + *

+ * + *

+ * If everything goes according to plan, notify the + * IncomingConnectionListeners with the connected socket and + * the parsed peer ID. + *

+ * + * @param client The accepted client's socket channel. + */ + private void accept(SocketChannel client) + throws IOException, SocketTimeoutException { + try { + logger.debug("New incoming connection, waiting for handshake..."); + Handshake hs = this.validateHandshake(client, null); + int sent = this.sendHandshake(client); + logger.trace("Replied to {} with handshake ({} bytes).", + this.socketRepr(client), sent); + + // Go to non-blocking mode for peer interaction + client.configureBlocking(false); + client.socket().setSoTimeout(CLIENT_KEEP_ALIVE_MINUTES*60*1000); + this.fireNewPeerConnection(client, hs.getPeerId()); + } catch (ParseException pe) { + logger.info("Invalid handshake from {}: {}", + this.socketRepr(client), pe.getMessage()); + IOUtils.closeQuietly(client); + } catch (IOException ioe) { + logger.warn("An error occured while reading an incoming " + + "handshake: {}", ioe.getMessage()); + if (client.isConnected()) { + IOUtils.closeQuietly(client); + } + } + } + + /** + * Tells whether the connection handler is running and can be used to + * handle new peer connections. + */ + public boolean isAlive() { + return this.executor != null && + !this.executor.isShutdown() && + !this.executor.isTerminated(); + } + + /** + * Connect to the given peer and perform the BitTorrent handshake. + * + *

+ * Submits an asynchronous connection task to the outbound connections + * executor to connect to the given peer. + *

+ * + * @param peer The peer to connect to. + */ + public void connect(SharingPeer peer) { + if (!this.isAlive()) { + throw new IllegalStateException( + "Connection handler is not accepting new peers at this time!"); + } + + this.executor.submit(new ConnectorTask(this, peer)); + } + + /** + * Validate an expected handshake on a connection. + * + *

+ * Reads an expected handshake message from the given connected socket, + * parses it and validates that the torrent hash_info corresponds to the + * torrent we're sharing, and that the peerId matches the peer ID we expect + * to see coming from the remote peer. + *

+ * + * @param channel The connected socket channel to the remote peer. + * @param peerId The peer ID we expect in the handshake. If null, + * any peer ID is accepted (this is the case for incoming connections). + * @return The validated handshake message object. + */ + private Handshake validateHandshake(SocketChannel channel, byte[] peerId) + throws IOException, ParseException { + ByteBuffer len = ByteBuffer.allocate(1); + ByteBuffer data; + + // Read the handshake from the wire + logger.trace("Reading handshake size (1 byte) from {}...", this.socketRepr(channel)); + if (channel.read(len) < len.capacity()) { + throw new IOException("Handshake size read underrrun"); + } + + len.rewind(); + int pstrlen = len.get(); + + data = ByteBuffer.allocate(Handshake.BASE_HANDSHAKE_LENGTH + pstrlen); + data.put((byte)pstrlen); + int expected = data.remaining(); + int read = channel.read(data); + if (read < expected) { + throw new IOException("Handshake data read underrun (" + + read + " < " + expected + " bytes)"); + } + + // Parse and check the handshake + data.rewind(); + Handshake hs = Handshake.parse(data); + if (!Arrays.equals(hs.getInfoHash(), this.torrent.getInfoHash())) { + throw new ParseException("Handshake for unknow torrent " + + Torrent.byteArrayToHexString(hs.getInfoHash()) + + " from " + this.socketRepr(channel) + ".", pstrlen + 9); + } + + if (peerId != null && !Arrays.equals(hs.getPeerId(), peerId)) { + throw new ParseException("Announced peer ID " + + Torrent.byteArrayToHexString(hs.getPeerId()) + + " did not match expected peer ID " + + Torrent.byteArrayToHexString(peerId) + ".", pstrlen + 29); + } + + return hs; + } + + /** + * Send our handshake message to the socket. + * + * @param channel The socket channel to the remote peer. + */ + private int sendHandshake(SocketChannel channel) throws IOException { + return channel.write( + Handshake.craft( + this.torrent.getInfoHash(), + this.id.getBytes(Torrent.BYTE_ENCODING)).getData()); + } + + /** + * Trigger the new peer connection event on all registered listeners. + * + * @param channel The socket channel to the newly connected peer. + * @param peerId The peer ID of the connected peer. + */ + private void fireNewPeerConnection(SocketChannel channel, byte[] peerId) { + for (IncomingConnectionListener listener : this.listeners) { + listener.handleNewPeerConnection(channel, peerId); + } + } + + private void fireFailedConnection(SharingPeer peer, Throwable cause) { + for (IncomingConnectionListener listener : this.listeners) { + listener.handleFailedConnection(peer, cause); + } + } + + + /** + * A simple thread factory that returns appropriately named threads for + * outbound connector threads. + * + * @author mpetazzoni + */ + private static class ConnectorThreadFactory implements ThreadFactory { + + private int number = 0; + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setName("bt-connect-" + ++this.number); + return t; + } + } + + + /** + * An outbound connection task. + * + *

+ * These tasks are fed to the thread executor in charge of processing + * outbound connection requests. It attempts to connect to the given peer + * and proceeds with the BitTorrent handshake. If the handshake is + * successful, the new peer connection event is fired to all incoming + * connection listeners. Otherwise, the failed connection event is fired. + *

+ * + * @author mpetazzoni + */ + private static class ConnectorTask implements Runnable { + + private final ConnectionHandler handler; + private final SharingPeer peer; + + private ConnectorTask(ConnectionHandler handler, SharingPeer peer) { + this.handler = handler; + this.peer = peer; + } + + @Override + public void run() { + InetSocketAddress address = + new InetSocketAddress(this.peer.getIp(), this.peer.getPort()); + SocketChannel channel = null; + + try { + logger.info("Connecting to {}...", this.peer); + channel = SocketChannel.open(address); + while (!channel.isConnected()) { + Thread.sleep(10); + } + + logger.debug("Connected. Sending handshake to {}...", this.peer); + channel.configureBlocking(true); + int sent = this.handler.sendHandshake(channel); + logger.debug("Sent handshake ({} bytes), waiting for response...", sent); + Handshake hs = this.handler.validateHandshake(channel, + (this.peer.hasPeerId() + ? this.peer.getPeerId().array() + : null)); + logger.info("Handshaked with {}, peer ID is {}.", + this.peer, Torrent.byteArrayToHexString(hs.getPeerId())); + + // Go to non-blocking mode for peer interaction + channel.configureBlocking(false); + this.handler.fireNewPeerConnection(channel, hs.getPeerId()); + } catch (Exception e) { + if (channel != null && channel.isConnected()) { + IOUtils.closeQuietly(channel); + } + this.handler.fireFailedConnection(this.peer, e); + } + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Handshake.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Handshake.java new file mode 100644 index 0000000..018cc5e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Handshake.java @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.text.ParseException; + + +/** + * Peer handshake handler. + * + */ +public class Handshake { + + public static final String BITTORRENT_PROTOCOL_IDENTIFIER = "BitTorrent protocol"; + public static final int BASE_HANDSHAKE_LENGTH = 49; + + ByteBuffer data; + ByteBuffer infoHash; + ByteBuffer peerId; + + private Handshake(ByteBuffer data, ByteBuffer infoHash, + ByteBuffer peerId) { + this.data = data; + this.data.rewind(); + + this.infoHash = infoHash; + this.peerId = peerId; + } + + public ByteBuffer getData() { + return this.data; + } + + public byte[] getInfoHash() { + return this.infoHash.array(); + } + + public byte[] getPeerId() { + return this.peerId.array(); + } + + public static Handshake parse(ByteBuffer buffer) + throws ParseException, UnsupportedEncodingException { + int pstrlen = Byte.valueOf(buffer.get()).intValue(); + if (pstrlen < 0 || + buffer.remaining() != BASE_HANDSHAKE_LENGTH + pstrlen - 1) { + throw new ParseException("Incorrect handshake message length " + + "(pstrlen=" + pstrlen + ") !", 0); + } + + // Check the protocol identification string + byte[] pstr = new byte[pstrlen]; + buffer.get(pstr); + + if (!Handshake.BITTORRENT_PROTOCOL_IDENTIFIER.equals( + new String(pstr, Torrent.BYTE_ENCODING))) { + throw new ParseException("Invalid protocol identifier!", 1); + } + + // Ignore reserved bytes + byte[] reserved = new byte[8]; + buffer.get(reserved); + + byte[] infoHash = new byte[20]; + buffer.get(infoHash); + byte[] peerId = new byte[20]; + buffer.get(peerId); + return new Handshake(buffer, ByteBuffer.wrap(infoHash), + ByteBuffer.wrap(peerId)); + } + + public static Handshake craft(byte[] torrentInfoHash, + byte[] clientPeerId) { + try { + ByteBuffer buffer = ByteBuffer.allocate( + Handshake.BASE_HANDSHAKE_LENGTH + + Handshake.BITTORRENT_PROTOCOL_IDENTIFIER.length()); + + byte[] reserved = new byte[8]; + ByteBuffer infoHash = ByteBuffer.wrap(torrentInfoHash); + ByteBuffer peerId = ByteBuffer.wrap(clientPeerId); + + buffer.put((byte)Handshake + .BITTORRENT_PROTOCOL_IDENTIFIER.length()); + buffer.put(Handshake + .BITTORRENT_PROTOCOL_IDENTIFIER.getBytes(Torrent.BYTE_ENCODING)); + buffer.put(reserved); + buffer.put(infoHash); + buffer.put(peerId); + + return new Handshake(buffer, infoHash, peerId); + } catch (UnsupportedEncodingException uee) { + return null; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/IncomingConnectionListener.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/IncomingConnectionListener.java new file mode 100644 index 0000000..71caa44 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/IncomingConnectionListener.java @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer; + +import java.nio.channels.SocketChannel; +import java.util.EventListener; + +/** + * EventListener interface for objects that want to handle incoming peer + * connections. + * + */ +public interface IncomingConnectionListener extends EventListener { + + public void handleNewPeerConnection(SocketChannel channel, byte[] peerId); + + public void handleFailedConnection(SharingPeer peer, Throwable cause); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Piece.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Piece.java new file mode 100644 index 0000000..c2dec3a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Piece.java @@ -0,0 +1,312 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer; +import org.apache.hadoop.yarn.server.broadcast.client.storage.TorrentByteStorage; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.concurrent.Callable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * A torrent piece. + * + *

+ * This class represents a torrent piece. Torrents are made of pieces, which + * are in turn made of blocks that are exchanged using the peer protocol. + * The piece length is defined at the torrent level, but the last piece that + * makes the torrent might be smaller. + *

+ * + *

+ * If the torrent has multiple files, pieces can spread across file boundaries. + * The TorrentByteStorage abstracts this problem to give Piece objects the + * impression of a contiguous, linear byte storage. + *

+ * + */ +public class Piece implements Comparable { + + private static final Logger logger = + LoggerFactory.getLogger(Piece.class); + + private final TorrentByteStorage bucket; + private final int index; + private final long offset; + private final long length; + private final byte[] hash; + private final boolean seeder; + + private volatile boolean valid; + private int seen; + private ByteBuffer data; + + /** + * Initialize a new piece in the byte bucket. + * + * @param bucket The underlying byte storage bucket. + * @param index This piece index in the torrent. + * @param offset This piece offset, in bytes, in the storage. + * @param length This piece length, in bytes. + * @param hash This piece 20-byte SHA1 hash sum. + * @param seeder Whether we're seeding this torrent or not (disables piece + * validation). + */ + public Piece(TorrentByteStorage bucket, int index, long offset, + long length, byte[] hash, boolean seeder) { + this.bucket = bucket; + this.index = index; + this.offset = offset; + this.length = length; + this.hash = hash; + this.seeder = seeder; + + // Piece is considered invalid until first check. + this.valid = false; + + // Piece start unseen + this.seen = 0; + + this.data = null; + } + + /** + * Tells whether this piece's data is valid or not. + */ + public boolean isValid() { + return this.valid; + } + + /** + * Returns the index of this piece in the torrent. + */ + public int getIndex() { + return this.index; + } + + /** + * Returns the size, in bytes, of this piece. + * + *

+ * All pieces, except the last one, are expected to have the same size. + *

+ */ + public long size() { + return this.length; + } + + /** + * Tells whether this piece is available in the current connected peer swarm. + */ + public boolean available() { + return this.seen > 0; + } + + /** + * Mark this piece as being seen at the given peer. + * + * @param peer The sharing peer this piece has been seen available at. + */ + public void seenAt(SharingPeer peer) { + this.seen++; + } + + /** + * Mark this piece as no longer being available at the given peer. + * + * @param peer The sharing peer from which the piece is no longer available. + */ + public void noLongerAt(SharingPeer peer) { + this.seen--; + } + + /** + * Validates this piece. + * + * @return Returns true if this piece, as stored in the underlying byte + * storage, is valid, i.e. its SHA1 sum matches the one from the torrent + * meta-info. + */ + public synchronized boolean validate() throws IOException { + if (this.seeder) { + logger.trace("Skipping validation of {} (seeder mode).", this); + this.valid = true; + return true; + } + + logger.trace("Validating {}...", this); + this.valid = false; + + ByteBuffer buffer = this._read(0, this.length); + byte[] data = new byte[(int)this.length]; + buffer.get(data); + this.valid = Arrays.equals(Torrent.hash(data), this.hash); + + return this.isValid(); + } + + /** + * Internal piece data read function. + * + *

+ * This function will read the piece data without checking if the piece has + * been validated. It is simply meant at factoring-in the common read code + * from the validate and read functions. + *

+ * + * @param offset Offset inside this piece where to start reading. + * @param length Number of bytes to read from the piece. + * @return A byte buffer containing the piece data. + * @throws IllegalArgumentException If offset + length goes over + * the piece boundary. + * @throws IOException If the read can't be completed (I/O error, or EOF + * reached, which can happen if the piece is not complete). + */ + private ByteBuffer _read(long offset, long length) throws IOException { + if (offset + length > this.length) { + throw new IllegalArgumentException("Piece#" + this.index + + " overrun (" + offset + " + " + length + " > " + + this.length + ") !"); + } + + // TODO: remove cast to int when large ByteBuffer support is + // implemented in Java. + ByteBuffer buffer = ByteBuffer.allocate((int)length); + int bytes = this.bucket.read(buffer, this.offset + offset); + buffer.rewind(); + buffer.limit(bytes >= 0 ? bytes : 0); + return buffer; + } + + /** + * Read a piece block from the underlying byte storage. + * + *

+ * This is the public method for reading this piece's data, and it will + * only succeed if the piece is complete and valid on disk, thus ensuring + * any data that comes out of this function is valid piece data we can send + * to other peers. + *

+ * + * @param offset Offset inside this piece where to start reading. + * @param length Number of bytes to read from the piece. + * @return A byte buffer containing the piece data. + * @throws IllegalArgumentException If offset + length goes over + * the piece boundary. + * @throws IllegalStateException If the piece is not valid when attempting + * to read it. + * @throws IOException If the read can't be completed (I/O error, or EOF + * reached, which can happen if the piece is not complete). + */ + public ByteBuffer read(long offset, int length) + throws IllegalArgumentException, IllegalStateException, IOException { + if (!this.valid) { + throw new IllegalStateException("Attempting to read an " + + "known-to-be invalid piece!"); + } + + return this._read(offset, length); + } + + /** + * Record the given block at the given offset in this piece. + * + *

+ * Note: this has synchronized access to the underlying byte storage. + *

+ * + * @param block The ByteBuffer containing the block data. + * @param offset The block offset in this piece. + */ + public synchronized void record(ByteBuffer block, int offset) + throws IOException { + if (this.data == null || offset == 0) { + // TODO: remove cast to int when large ByteBuffer support is + // implemented in Java. + this.data = ByteBuffer.allocate((int)this.length); + } + + int pos = block.position(); + this.data.position(offset); + this.data.put(block); + block.position(pos); + + if (block.remaining() + offset == this.length) { + this.data.rewind(); + logger.trace("Recording {}...", this); + this.bucket.write(this.data, this.offset); + this.data = null; + } + } + + /** + * Return a human-readable representation of this piece. + */ + public String toString() { + return String.format("piece#%4d%s", + this.index, + this.isValid() ? "+" : "-"); + } + + /** + * Piece comparison function for ordering pieces based on their + * availability. + * + * @param other The piece to compare with, should not be null. + */ + public int compareTo(Piece other) { + if (this.seen != other.seen) { + return this.seen < other.seen ? -1 : 1; + } + return this.index == other.index ? 0 : + (this.index < other.index ? -1 : 1); + } + + /** + * A {@link Callable} to call the piece validation function. + * + *

+ * This {@link Callable} implementation allows for the calling of the piece + * validation function in a controlled context like a thread or an + * executor. It returns the piece it was created for. Results of the + * validation can easily be extracted from the {@link Piece} object after + * it is returned. + *

+ * + * @author mpetazzoni + */ + public static class CallableHasher implements Callable { + + private final Piece piece; + + public CallableHasher(Piece piece) { + this.piece = piece; + } + + @Override + public Piece call() throws IOException { + this.piece.validate(); + return this.piece; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/SharedTorrent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/SharedTorrent.java new file mode 100644 index 0000000..62dd8a2 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/SharedTorrent.java @@ -0,0 +1,879 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.bcodec.InvalidBEncodingException; +import org.apache.hadoop.yarn.server.broadcast.client.peer.PeerActivityListener; +import org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer; +import org.apache.hadoop.yarn.server.broadcast.client.storage.FileCollectionStorage; +import org.apache.hadoop.yarn.server.broadcast.client.storage.FileStorage; +import org.apache.hadoop.yarn.server.broadcast.client.storage.TorrentByteStorage; +import org.apache.hadoop.yarn.server.broadcast.client.strategy.RequestStrategy; +import org.apache.hadoop.yarn.server.broadcast.client.strategy.RequestStrategyImplRarest; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + + +/** + * A torrent shared by the BitTorrent client. + * + *

+ * The {@link SharedTorrent} class extends the Torrent class with all the data + * and logic required by the BitTorrent client implementation. + *

+ * + *

+ * Note: this implementation currently only supports single-file + * torrents. + *

+ * + */ +public class SharedTorrent extends Torrent implements PeerActivityListener { + + private static final Logger logger = + LoggerFactory.getLogger(SharedTorrent.class); + + /** End-game trigger ratio. + * + *

+ * Eng-game behavior (requesting already requested pieces from available + * and ready peers to try to speed-up the end of the transfer) will only be + * enabled when the ratio of completed pieces over total pieces in the + * torrent is over this value. + *

+ */ + private static final float ENG_GAME_COMPLETION_RATIO = 0.95f; + + /** Default Request Strategy. + * + * Use the rarest-first strategy by default. + */ + private static final RequestStrategy DEFAULT_REQUEST_STRATEGY = new RequestStrategyImplRarest(); + + private boolean stop; + + private long uploaded; + private long downloaded; + private long left; + + private final TorrentByteStorage bucket; + + private final int pieceLength; + private final ByteBuffer piecesHashes; + + private boolean initialized; + private Piece[] pieces; + private SortedSet rarest; + private BitSet completedPieces; + private BitSet requestedPieces; + private RequestStrategy requestStrategy; + + private double maxUploadRate = 0.0; + private double maxDownloadRate = 0.0; + /** + * Create a new shared torrent from a base Torrent object. + * + *

+ * This will recreate a SharedTorrent object from the provided Torrent + * object's encoded meta-info data. + *

+ * + * @param torrent The Torrent object. + * @param destDir The destination directory or location of the torrent + * files. + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(Torrent torrent, File destDir) + throws FileNotFoundException, IOException { + this(torrent, destDir, false); + } + + /** + * Create a new shared torrent from a base Torrent object. + * + *

+ * This will recreate a SharedTorrent object from the provided Torrent + * object's encoded meta-info data. + *

+ * + * @param torrent The Torrent object. + * @param destDir The destination directory or location of the torrent + * files. + * @param seeder Whether we're a seeder for this torrent or not (disables + * validation). + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(Torrent torrent, File destDir, boolean seeder) + throws FileNotFoundException, IOException { + this(torrent.getEncoded(), destDir, seeder, DEFAULT_REQUEST_STRATEGY); + } + + /** + * Create a new shared torrent from a base Torrent object. + * + *

+ * This will recreate a SharedTorrent object from the provided Torrent + * object's encoded meta-info data. + *

+ * + * @param torrent The Torrent object. + * @param destDir The destination directory or location of the torrent + * files. + * @param seeder Whether we're a seeder for this torrent or not (disables + * validation). + * @param requestStrategy The request strategy implementation. + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(Torrent torrent, File destDir, boolean seeder, + RequestStrategy requestStrategy) + throws FileNotFoundException, IOException { + this(torrent.getEncoded(), destDir, seeder, requestStrategy); + } + + /** + * Create a new shared torrent from meta-info binary data. + * + * @param torrent The meta-info byte data. + * @param destDir The destination directory or location of the torrent + * files. + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(byte[] torrent, File destDir) + throws FileNotFoundException, IOException { + this(torrent, destDir, false); + } + + /** + * Create a new shared torrent from meta-info binary data. + * + * @param torrent The meta-info byte data. + * @param parent The parent directory or location the torrent files. + * @param seeder Whether we're a seeder for this torrent or not (disables + * validation). + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(byte[] torrent, File parent, boolean seeder) + throws FileNotFoundException, IOException { + this(torrent, parent, seeder, DEFAULT_REQUEST_STRATEGY); + } + + /** + * Create a new shared torrent from meta-info binary data. + * + * @param torrent The meta-info byte data. + * @param parent The parent directory or location the torrent files. + * @param seeder Whether we're a seeder for this torrent or not (disables + * validation). + * @param requestStrategy The request strategy implementation. + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(byte[] torrent, File parent, boolean seeder, + RequestStrategy requestStrategy) + throws FileNotFoundException, IOException { + super(torrent, seeder); + + if (parent == null || !parent.isDirectory()) { + throw new IllegalArgumentException("Invalid parent directory!"); + } + + String parentPath = parent.getCanonicalPath(); + + try { + this.pieceLength = this.decoded_info.get("piece length").getInt(); + this.piecesHashes = ByteBuffer.wrap(this.decoded_info.get("pieces") + .getBytes()); + + if (this.piecesHashes.capacity() / Torrent.PIECE_HASH_SIZE * + (long)this.pieceLength < this.getSize()) { + throw new IllegalArgumentException("Torrent size does not " + + "match the number of pieces and the piece size!"); + } + } catch (InvalidBEncodingException ibee) { + throw new IllegalArgumentException( + "Error reading torrent meta-info fields!"); + } + + List files = new LinkedList(); + long offset = 0L; + for (Torrent.TorrentFile file : this.files) { + File actual = new File(parent, file.file.getPath()); + +/* if (!actual.getCanonicalPath().startsWith(parentPath)) { + throw new SecurityException("Torrent file path attempted " + + "to break directory jail!"); + }*/ + + actual.getParentFile().mkdirs(); + files.add(new FileStorage(actual, offset, file.size)); + offset += file.size; + } + this.bucket = new FileCollectionStorage(files, this.getSize()); + + this.stop = false; + + this.uploaded = 0; + this.downloaded = 0; + this.left = this.getSize(); + + this.initialized = false; + this.pieces = new Piece[0]; + this.rarest = Collections.synchronizedSortedSet(new TreeSet()); + this.completedPieces = new BitSet(); + this.requestedPieces = new BitSet(); + + //TODO: should switch to guice + this.requestStrategy = requestStrategy; + } + + /** + * Create a new shared torrent from the given torrent file. + * + * @param source The .torrent file to read the torrent + * meta-info from. + * @param parent The parent directory or location of the torrent files. + * @throws IOException When the torrent file cannot be read or decoded. + */ + public static SharedTorrent fromFile(File source, File parent) + throws IOException { + byte[] data = FileUtils.readFileToByteArray(source); + return new SharedTorrent(data, parent); + } + + public double getMaxUploadRate() { + return this.maxUploadRate; + } + + /** + * Set the maximum upload rate (in kB/second) for this + * torrent. A setting of <= 0.0 disables rate limiting. + * + * @param rate The maximum upload rate + */ + public void setMaxUploadRate(double rate) { + this.maxUploadRate = rate; + } + + public double getMaxDownloadRate() { + return this.maxDownloadRate; + } + + /** + * Set the maximum download rate (in kB/second) for this + * torrent. A setting of <= 0.0 disables rate limiting. + * + * @param rate The maximum download rate + */ + public void setMaxDownloadRate(double rate) { + this.maxDownloadRate = rate; + } + + /** + * Get the number of bytes uploaded for this torrent. + */ + public long getUploaded() { + return this.uploaded; + } + + /** + * Get the number of bytes downloaded for this torrent. + * + *

+ * Note: this could be more than the torrent's length, and should + * not be used to determine a completion percentage. + *

+ */ + public long getDownloaded() { + return this.downloaded; + } + + /** + * Get the number of bytes left to download for this torrent. + */ + public long getLeft() { + return this.left; + } + + /** + * Tells whether this torrent has been fully initialized yet. + */ + public boolean isInitialized() { + return this.initialized; + } + + /** + * Stop the torrent initialization as soon as possible. + */ + public void stop() { + this.stop = true; + } + + /** + * Build this torrent's pieces array. + * + *

+ * Hash and verify any potentially present local data and create this + * torrent's pieces array from their respective hash provided in the + * torrent meta-info. + *

+ * + *

+ * This function should be called soon after the constructor to initialize + * the pieces array. + *

+ */ + public synchronized void init() throws InterruptedException, IOException { + if (this.isInitialized()) { + throw new IllegalStateException("Torrent was already initialized!"); + } + + int threads = getHashingThreadsCount(); + int nPieces = (int) (Math.ceil( + (double)this.getSize() / this.pieceLength)); + int step = 10; + + this.pieces = new Piece[nPieces]; + this.completedPieces = new BitSet(nPieces); + this.piecesHashes.clear(); + + ExecutorService executor = Executors.newFixedThreadPool(threads); + List> results = new LinkedList>(); + + try { + logger.info("Analyzing local data for {} with {} threads ({} pieces)...", + new Object[] { this.getName(), threads, nPieces }); + for (int idx=0; idx hasher = new Piece.CallableHasher(this.pieces[idx]); + results.add(executor.submit(hasher)); + + if (results.size() >= threads) { + this.validatePieces(results); + } + + if (idx / (float)nPieces * 100f > step) { + logger.info(" ... {}% complete", step); + step += 10; + } + } + + this.validatePieces(results); + } finally { + // Request orderly executor shutdown and wait for hashing tasks to + // complete. + executor.shutdown(); + while (!executor.isTerminated()) { + if (this.stop) { + throw new InterruptedException("Torrent data analysis " + + "interrupted."); + } + + Thread.sleep(10); + } + } + + logger.debug("{}: we have {}/{} bytes ({}%) [{}/{} pieces].", + new Object[] { + this.getName(), + (this.getSize() - this.left), + this.getSize(), + String.format("%.1f", (100f * (1f - this.left / (float)this.getSize()))), + this.completedPieces.cardinality(), + this.pieces.length + }); + this.initialized = true; + } + + /** + * Process the pieces enqueued for hash validation so far. + * + * @param results The list of {@link Future}s of pieces to process. + */ + private void validatePieces(List> results) + throws IOException { + try { + for (Future task : results) { + Piece piece = task.get(); + if (this.pieces[piece.getIndex()].isValid()) { + this.completedPieces.set(piece.getIndex()); + this.left -= piece.size(); + } + } + + results.clear(); + } catch (Exception e) { + throw new IOException("Error while hashing a torrent piece!", e); + } + } + + + public synchronized void close() { + try { + this.bucket.close(); + } catch (IOException ioe) { + logger.error("Error closing torrent byte storage: {}", + ioe.getMessage()); + } + } + + /** + * Retrieve a piece object by index. + * + * @param index The index of the piece in this torrent. + */ + public Piece getPiece(int index) { + if (this.pieces == null) { + throw new IllegalStateException("Torrent not initialized yet."); + } + + if (index >= this.pieces.length) { + throw new IllegalArgumentException("Invalid piece index!"); + } + + return this.pieces[index]; + } + + /** + * Get the number of pieces in this torrent. + */ + public int getPieceCount() { + if (this.pieces == null) { + throw new IllegalStateException("Torrent not initialized yet."); + } + + return this.pieces.length; + } + + + /** + * Return a copy of the bit field of available pieces for this torrent. + * + *

+ * Available pieces are pieces available in the swarm, and it does not + * include our own pieces. + *

+ */ + public BitSet getAvailablePieces() { + if (!this.isInitialized()) { + throw new IllegalStateException("Torrent not yet initialized!"); + } + + BitSet availablePieces = new BitSet(this.pieces.length); + + synchronized (this.pieces) { + for (Piece piece : this.pieces) { + if (piece.available()) { + availablePieces.set(piece.getIndex()); + } + } + } + + return availablePieces; + } + + /** + * Return a copy of the completed pieces bitset. + */ + public BitSet getCompletedPieces() { + if (!this.isInitialized()) { + throw new IllegalStateException("Torrent not yet initialized!"); + } + + synchronized (this.completedPieces) { + return (BitSet)this.completedPieces.clone(); + } + } + + /** + * Return a copy of the requested pieces bitset. + */ + public BitSet getRequestedPieces() { + if (!this.isInitialized()) { + throw new IllegalStateException("Torrent not yet initialized!"); + } + + synchronized (this.requestedPieces) { + return (BitSet)this.requestedPieces.clone(); + } + } + + /** + * Tells whether this torrent has been fully downloaded, or is fully + * available locally. + */ + public synchronized boolean isComplete() { + return this.pieces.length > 0 && + this.completedPieces.cardinality() == this.pieces.length; + } + + /** + * Finalize the download of this torrent. + * + *

+ * This realizes the final, pre-seeding phase actions on this torrent, + * which usually consists in putting the torrent data in their final form + * and at their target location. + *

+ * + * @see TorrentByteStorage#finish + */ + public synchronized void finish() throws IOException { + if (!this.isInitialized()) { + throw new IllegalStateException("Torrent not yet initialized!"); + } + + if (!this.isComplete()) { + throw new IllegalStateException("Torrent download is not complete!"); + } + + this.bucket.finish(); + } + + public synchronized boolean isFinished() { + return this.isComplete() && this.bucket.isFinished(); + } + + /** + * Return the completion percentage of this torrent. + * + *

+ * This is computed from the number of completed pieces divided by the + * number of pieces in this torrent, times 100. + *

+ */ + public float getCompletion() { + return this.isInitialized() + ? (float)this.completedPieces.cardinality() / + (float)this.pieces.length * 100.0f + : 0.0f; + } + + /** + * Mark a piece as completed, decrementing the piece size in bytes from our + * left bytes to download counter. + */ + public synchronized void markCompleted(Piece piece) { + if (this.completedPieces.get(piece.getIndex())) { + return; + } + + // A completed piece means that's that much data left to download for + // this torrent. + this.left -= piece.size(); + this.completedPieces.set(piece.getIndex()); + } + + /** PeerActivityListener handler(s). *************************************/ + + /** + * Peer choked handler. + * + *

+ * When a peer chokes, the requests made to it are canceled and we need to + * mark the eventually piece we requested from it as available again for + * download tentative from another peer. + *

+ * + * @param peer The peer that choked. + */ + @Override + public synchronized void handlePeerChoked(SharingPeer peer) { + Piece piece = peer.getRequestedPiece(); + + if (piece != null) { + this.requestedPieces.set(piece.getIndex(), false); + } + + logger.trace("Peer {} choked, we now have {} outstanding " + + "request(s): {}", + new Object[] { + peer, + this.requestedPieces.cardinality(), + this.requestedPieces + }); + } + + /** + * Peer ready handler. + * + *

+ * When a peer becomes ready to accept piece block requests, select a piece + * to download and go for it. + *

+ * + * @param peer The peer that became ready. + */ + @Override + public synchronized void handlePeerReady(SharingPeer peer) { + BitSet interesting = peer.getAvailablePieces(); + interesting.andNot(this.completedPieces); + interesting.andNot(this.requestedPieces); + + logger.trace("Peer {} is ready and has {} interesting piece(s).", + peer, interesting.cardinality()); + + // If we didn't find interesting pieces, we need to check if we're in + // an end-game situation. If yes, we request an already requested piece + // to try to speed up the end. + if (interesting.cardinality() == 0) { + interesting = peer.getAvailablePieces(); + interesting.andNot(this.completedPieces); + if (interesting.cardinality() == 0) { + logger.trace("No interesting piece from {}!", peer); + return; + } + + if (this.completedPieces.cardinality() < + ENG_GAME_COMPLETION_RATIO * this.pieces.length) { + logger.trace("Not far along enough to warrant end-game mode."); + return; + } + + logger.trace("Possible end-game, we're about to request a piece " + + "that was already requested from another peer."); + } + + Piece chosen = requestStrategy.choosePiece(rarest, interesting, pieces); + this.requestedPieces.set(chosen.getIndex()); + + logger.trace("Requesting {} from {}, we now have {} " + + "outstanding request(s): {}", + new Object[] { + chosen, + peer, + this.requestedPieces.cardinality(), + this.requestedPieces + }); + + peer.downloadPiece(chosen); + } + + /** + * Piece availability handler. + * + *

+ * Handle updates in piece availability from a peer's HAVE message. When + * this happens, we need to mark that piece as available from the peer. + *

+ * + * @param peer The peer we got the update from. + * @param piece The piece that became available. + */ + @Override + public synchronized void handlePieceAvailability(SharingPeer peer, + Piece piece) { + // If we don't have this piece, tell the peer we're interested in + // getting it from him. + if (!this.completedPieces.get(piece.getIndex()) && + !this.requestedPieces.get(piece.getIndex())) { + peer.interesting(); + } + + this.rarest.remove(piece); + piece.seenAt(peer); + this.rarest.add(piece); + + logger.trace("Peer {} contributes {} piece(s) [{}/{}/{}].", + new Object[] { + peer, + peer.getAvailablePieces().cardinality(), + this.completedPieces.cardinality(), + this.getAvailablePieces().cardinality(), + this.pieces.length + }); + + if (!peer.isChoked() && + peer.isInteresting() && + !peer.isDownloading()) { + this.handlePeerReady(peer); + } + } + + /** + * Bit field availability handler. + * + *

+ * Handle updates in piece availability from a peer's BITFIELD message. + * When this happens, we need to mark in all the pieces the peer has that + * they can be reached through this peer, thus augmenting the global + * availability of pieces. + *

+ * + * @param peer The peer we got the update from. + * @param availablePieces The pieces availability bit field of the peer. + */ + @Override + public synchronized void handleBitfieldAvailability(SharingPeer peer, + BitSet availablePieces) { + // Determine if the peer is interesting for us or not, and notify it. + BitSet interesting = (BitSet)availablePieces.clone(); + interesting.andNot(this.completedPieces); + interesting.andNot(this.requestedPieces); + + if (interesting.cardinality() == 0) { + peer.notInteresting(); + } else { + peer.interesting(); + } + + // Record that the peer has all the pieces it told us it had. + for (int i = availablePieces.nextSetBit(0); i >= 0; + i = availablePieces.nextSetBit(i+1)) { + this.rarest.remove(this.pieces[i]); + this.pieces[i].seenAt(peer); + this.rarest.add(this.pieces[i]); + } + + logger.trace("Peer {} contributes {} piece(s) ({} interesting) " + + "[completed={}; available={}/{}].", + new Object[] { + peer, + availablePieces.cardinality(), + interesting.cardinality(), + this.completedPieces.cardinality(), + this.getAvailablePieces().cardinality(), + this.pieces.length + }); + } + + /** + * Piece upload completion handler. + * + *

+ * When a piece has been sent to a peer, we just record that we sent that + * many bytes. If the piece is valid on the peer's side, it will send us a + * HAVE message and we'll record that the piece is available on the peer at + * that moment (see handlePieceAvailability()). + *

+ * + * @param peer The peer we got this piece from. + * @param piece The piece in question. + */ + @Override + public synchronized void handlePieceSent(SharingPeer peer, Piece piece) { + logger.trace("Completed upload of {} to {}.", piece, peer); + this.uploaded += piece.size(); + } + + /** + * Piece download completion handler. + * + *

+ * If the complete piece downloaded is valid, we can record in the torrent + * completedPieces bit field that we know have this piece. + *

+ * + * @param peer The peer we got this piece from. + * @param piece The piece in question. + */ + @Override + public synchronized void handlePieceCompleted(SharingPeer peer, + Piece piece) throws IOException { + // Regardless of validity, record the number of bytes downloaded and + // mark the piece as not requested anymore + this.downloaded += piece.size(); + this.requestedPieces.set(piece.getIndex(), false); + + logger.trace("We now have {} piece(s) and {} outstanding request(s): {}", + new Object[] { + this.completedPieces.cardinality(), + this.requestedPieces.cardinality(), + this.requestedPieces + }); + } + + /** + * Peer disconnection handler. + * + *

+ * When a peer disconnects, we need to mark in all of the pieces it had + * available that they can't be reached through this peer anymore. + *

+ * + * @param peer The peer we got this piece from. + */ + @Override + public synchronized void handlePeerDisconnected(SharingPeer peer) { + BitSet availablePieces = peer.getAvailablePieces(); + + for (int i = availablePieces.nextSetBit(0); i >= 0; + i = availablePieces.nextSetBit(i+1)) { + this.rarest.remove(this.pieces[i]); + this.pieces[i].noLongerAt(peer); + this.rarest.add(this.pieces[i]); + } + + Piece requested = peer.getRequestedPiece(); + if (requested != null) { + this.requestedPieces.set(requested.getIndex(), false); + } + + logger.debug("Peer {} went away with {} piece(s) [completed={}; available={}/{}]", + new Object[] { + peer, + availablePieces.cardinality(), + this.completedPieces.cardinality(), + this.getAvailablePieces().cardinality(), + this.pieces.length + }); + logger.trace("We now have {} piece(s) and {} outstanding request(s): {}", + new Object[] { + this.completedPieces.cardinality(), + this.requestedPieces.cardinality(), + this.requestedPieces + }); + } + + @Override + public synchronized void handleIOException(SharingPeer peer, + IOException ioe) { /* Do nothing */ } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/Announce.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/Announce.java new file mode 100644 index 0000000..66c5918 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/Announce.java @@ -0,0 +1,223 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceRequestMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * BitTorrent announce sub-system. + * + *

+ * A BitTorrent client must check-in to the torrent's tracker(s) to get peers + * and to report certain events. + *

+ * + *

+ * This Announce class implements a periodic announce request thread that will + * notify announce request event listeners for each tracker response. + *

+ * + * @see org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage + */ +public class Announce implements Runnable { + + protected static final Logger logger = + LoggerFactory.getLogger(Announce.class); + + private final Peer peer; + + private final TrackerClient client; + + /** Announce thread and control. */ + private Thread thread; + private boolean stop; + private boolean forceStop; + + /** Announce interval. */ + private int interval; + + private int currentTier; + private int currentClient; + + /** + * Initialize the base announce class members for the announcer. + * + * @param torrent The torrent we're announcing about. + * @param peer Our peer specification. + */ + public Announce(SharedTorrent torrent, Peer peer) { + this.peer = peer; + this.client = new TrackerClient(torrent, peer); + this.thread = null; + this.currentTier = 0; + this.currentClient = 0; + + logger.info("Initialized announce sub-system with 1 trackers on {}.", new Object[] { torrent }); + } + + /** + * Register a new announce response listener. + * + * @param listener The listener to register on this announcer events. + */ + public void register(AnnounceResponseListener listener) { + client.register(listener); + } + + /** + * Start the announce request thread. + */ + public void start() { + this.stop = false; + this.forceStop = false; + + if (this.thread == null || !this.thread.isAlive()) { + this.thread = new Thread(this); + this.thread.setName("bt-announce(" + + this.peer.getShortHexPeerId() + ")"); + this.thread.start(); + } + } + + /** + * Set the announce interval. + */ + public void setInterval(int interval) { + if (interval <= 0) { + this.stop(true); + return; + } + + if (this.interval == interval) { + return; + } + + logger.info("Setting announce interval to {}s per tracker request.", + interval); + this.interval = interval; + } + + /** + * Stop the announce thread. + * + *

+ * One last 'stopped' announce event might be sent to the tracker to + * announce we're going away, depending on the implementation. + *

+ */ + public void stop() { + this.stop = true; + + if (this.thread != null && this.thread.isAlive()) { + this.thread.interrupt(); + this.client.close(); + + try { + this.thread.join(); + } catch (InterruptedException ie) { + // Ignore + } + } + + this.thread = null; + } + + /** + * Main announce loop. + * + *

+ * The announce thread starts by making the initial 'started' announce + * request to register on the tracker and get the announce interval value. + * Subsequent announce requests are ordinary, event-less, periodic requests + * for peers. + *

+ * + *

+ * Unless forcefully stopped, the announce thread will terminate by sending + * a 'stopped' announce request before stopping. + *

+ */ + @Override + public void run() { + logger.info("Starting announce loop..."); + + // Set an initial announce interval to 5 seconds. This will be updated + // in real-time by the tracker's responses to our announce requests. + this.interval = 5; + + AnnounceRequestMessage.RequestEvent event = + AnnounceRequestMessage.RequestEvent.STARTED; + + while (!this.stop) { + try { + this.getCurrentTrackerClient().announce(event, false); + event = AnnounceRequestMessage.RequestEvent.NONE; + } catch (AnnounceException ae) { + logger.info(ae.getMessage()); + logger.info("will re-announce with message " + event.getEventName()); + } + + try { + Thread.sleep(this.interval * 1000); + } catch (InterruptedException ie) { + // Ignore + } + } + + logger.info("Exited announce loop."); + + if (!this.forceStop) { + // Send the final 'stopped' event to the tracker after a little + // while. + event = AnnounceRequestMessage.RequestEvent.STOPPED; + try { + Thread.sleep(500); + } catch (InterruptedException ie) { + // Ignore + } + + try { + this.getCurrentTrackerClient().announce(event, true); + } catch (AnnounceException ae) { + logger.warn(ae.getMessage()); + } + } + } + + /** + * Returns the current tracker client used for announces. + * @throws AnnounceException When the current announce tier isn't defined + * in the torrent. + */ + public TrackerClient getCurrentTrackerClient() throws AnnounceException { + return this.client; + } + + /** + * Stop the announce thread. + * + * @param hard Whether to force stop the announce thread or not, i.e. not + * send the final 'stopped' announce request or not. + */ + private void stop(boolean hard) { + this.forceStop = hard; + this.stop(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceException.java new file mode 100644 index 0000000..7e28029 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceException.java @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + + +/** + * Exception thrown when an announce request failed. + * + */ +public class AnnounceException extends Exception { + + private static final long serialVersionUID = -1; + + public AnnounceException(String message) { + super(message); + } + + public AnnounceException(Throwable cause) { + super(cause); + } + + public AnnounceException(String message, Throwable cause) { + super(message, cause); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceResponseListener.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceResponseListener.java new file mode 100644 index 0000000..5bfcd7c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceResponseListener.java @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.hadoop.yarn.server.broadcast.common.Peer; + +import java.util.EventListener; +import java.util.List; + + +/** + * EventListener interface for objects that want to receive tracker responses. + * + */ +public interface AnnounceResponseListener extends EventListener { + + /** + * Handle an announce response event. + * + * @param interval The announce interval requested by the tracker. + * @param complete The number of seeders on this torrent. + * @param incomplete The number of leechers on this torrent. + */ + public void handleAnnounceResponse(int interval, int complete, + int incomplete); + + /** + * Handle the discovery of new peers. + * + * @param peers The list of peers discovered (from the announce response or + * any other means like DHT/PEX, etc.). + */ + public void handleDiscoveredPeers(List peers); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClient.java new file mode 100644 index 0000000..69c5914 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClient.java @@ -0,0 +1,341 @@ +/** + * Copyright (C) 2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.hadoop.util.MachineList.InetAddressFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceRequestMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceResponseMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.ErrorMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.ErrorMessage.FailureReason; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.MessageValidationException; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPAnnounceRequestMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPTrackerMessage; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.hadoop.yarn.server.broadcast.service.Topology; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.http.HttpVersion; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.entity.mime.MultipartEntity; +import org.apache.http.entity.mime.content.ByteArrayBody; +import org.apache.http.entity.mime.content.ContentBody; +import org.apache.http.entity.mime.content.FileBody; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.params.BasicHttpParams; +import org.apache.http.params.CoreProtocolPNames; +import org.apache.http.params.DefaultedHttpParams; +import org.apache.http.params.HttpParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.*; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Announcer for trackers. + * + * @see BitTorrent tracker request specification + */ +public class TrackerClient { + /** The set of listeners to announce request answers. */ + private final Set listeners; + protected final SharedTorrent torrent; + protected final Peer peer; + private final TrackerClientTransport transport; + private final Topology topology; + + protected static final Logger logger = + LoggerFactory.getLogger(TrackerClient.class); + + /** + * Create a new HTTP announcer for the given torrent. + * + * @param torrent The torrent we're announcing about. + * @param peer Our own peer specification. + */ + protected TrackerClient(SharedTorrent torrent, Peer peer) { + this(torrent, peer, new TrackerClientTransport(), new Topology()); + } + + protected TrackerClient(SharedTorrent torrent, Peer peer, TrackerClientTransport transport, Topology topology) { + this.listeners = new HashSet(); + this.torrent = torrent; + this.peer = peer; + this.transport = transport; + this.topology = topology; + } + + /** + * Register a new announce response listener. + * + * @param listener The listener to register on this announcer events. + */ + public void register(AnnounceResponseListener listener) { + this.listeners.add(listener); + } + + /** + * Close any opened announce connection. + * + *

+ * This method is called by {@link Announce#stop()} to make sure all connections + * are correctly closed when the announce thread is asked to stop. + *

+ */ + protected void close() { + // Do nothing by default, but can be overloaded. + } + + /** + * Formats an announce event into a usable string. + */ + protected String formatAnnounceEvent( + AnnounceRequestMessage.RequestEvent event) { + return AnnounceRequestMessage.RequestEvent.NONE.equals(event) + ? "" + : String.format(" %s", event.name()); + } + + /** + * Handle the announce response from the tracker. + * + *

+ * Analyzes the response from the tracker and acts on it. If the response + * is an error, it is logged. Otherwise, the announce response is used + * to fire the corresponding announce and peer events to all announce + * listeners. + *

+ * + * @param message The incoming {@link TrackerMessage}. + * @param inhibitEvents Whether or not to prevent events from being fired. + */ + protected void handleTrackerAnnounceResponse(TrackerMessage message, + boolean inhibitEvents, String trackerAddr) throws AnnounceException { + logger.info("get tracker response, message: " + message.getType() + new String(message.getData().array())); + if (message instanceof ErrorMessage) { + ErrorMessage error = (ErrorMessage)message; + // if the error is due to unknown torrents, we post torrent to tracker + if (error.getReason().equals(FailureReason.UNKNOWN_TORRENT.getMessage())) { + logger.info("Tracker don't know this torrent. Post torrent to tracker"); + transport.postTorrent(trackerAddr, this.torrent.getEncoded()); + logger.info("finish post"); + } + /* this throw is must even message is UNKNOWN_TORRENT because it keep the announce loop from moving from START_MSG */ + throw new AnnounceException(error.getReason()); + } + + if (! (message instanceof AnnounceResponseMessage)) { + throw new AnnounceException("Unexpected tracker message type " + + message.getType().name() + "!"); + } + + if (inhibitEvents) { + return; + } + + AnnounceResponseMessage response = + (AnnounceResponseMessage)message; + this.fireAnnounceResponseEvent( + response.getComplete(), + response.getIncomplete(), + response.getInterval()); + this.fireDiscoveredPeersEvent(response.getPeers()); + } + + /** + * Fire the announce response event to all listeners. + * + * @param complete The number of seeders on this torrent. + * @param incomplete The number of leechers on this torrent. + * @param interval The announce interval requested by the tracker. + */ + protected void fireAnnounceResponseEvent(int complete, int incomplete, + int interval) { + for (AnnounceResponseListener listener : this.listeners) { + listener.handleAnnounceResponse(interval, complete, incomplete); + } + } + + /** + * Fire the new peer discovery event to all listeners. + * + * @param peers The list of peers discovered. + */ + protected void fireDiscoveredPeersEvent(List peers) { + for (AnnounceResponseListener listener : this.listeners) { + listener.handleDiscoveredPeers(peers); + } + } + + /** + * Build, send and process a tracker announce request. + * + *

+ * This function first builds an announce request for the specified event + * with all the required parameters. Then, the request is made to the + * tracker and the response analyzed. + *

+ * + *

+ * All registered {@link AnnounceResponseListener} objects are then fired + * with the decoded payload. + *

+ * + * @param event The announce event type (can be AnnounceEvent.NONE for + * periodic updates). + * @param inhibitEvents Prevent event listeners from being notified. + */ + public void announce(AnnounceRequestMessage.RequestEvent event, + boolean inhibitEvents) throws AnnounceException { + logger.info(this + "announce: Announcing{} to tracker with {}U/{}D/{}L bytes...", + new Object[] { + this.formatAnnounceEvent(event), + this.torrent.getUploaded(), + this.torrent.getDownloaded(), + this.torrent.getLeft() + }); + + HTTPAnnounceRequestMessage request; + + try { + request = this.buildAnnounceRequest(event); + } catch (IOException ioe) { + throw new AnnounceException("Error building announce request (" + + ioe.getMessage() + ")", ioe); + } catch (MessageValidationException mve) { + throw new AnnounceException("Announce request creation violated " + + "expected protocol (" + mve.getMessage() + ")", mve); + } + + String rackMaster = null; + do { + rackMaster = rackMaster == null ? topology.getRackMaster(this.torrent.getHexInfoHash()) : topology.getNextRackMaster(rackMaster); + try { + logger.info(this + "try announcing to rackMaster " + rackMaster); + announceTo(request, rackMaster, inhibitEvents); + } catch (AnnounceException ae) { + throw ae; + } catch (IOException e) { + logger.error("fail to announce to rackMaster " + rackMaster + ", try next"); + continue; + } + + // If we are rackMaster , we need to announce to globalMaster too + if (rackMaster.equals(topology.getLocalAddr())) { + String globalMaster = null; + do { + globalMaster = globalMaster == null ? topology.getGlobalMaster(this.torrent.getHexInfoHash()) : topology.getNextRackMaster(globalMaster); + // If we are also globalMaster, we cannot announce because otherwise we violate the interval + if (globalMaster.equals(rackMaster)) + break; + try { + logger.info("try announcing to globalMaster " + globalMaster); + announceTo(request, globalMaster, inhibitEvents); + } catch (AnnounceException e) { + throw e; + } catch (IOException e) { + logger.error("fail to announce to globalMaster " + globalMaster + ", try next"); + continue; + } + break; + } while (true); + } + break; + } while (true); + } + + /** + * Send and process a tracker announce request. + * + *

+ * This function send the request to the + * tracker and then the response is analyzed. + *

+ * + *

+ * All registered {@link AnnounceResponseListener} objects are then fired + * with the decoded payload. + *

+ * + * @param request the announce request + * @param trackerAddr the address of the tracker + */ + public void announceTo(HTTPAnnounceRequestMessage request, String trackerAddr, boolean inhibitEvents) throws IOException, AnnounceException { + logger.info(request.getPort() + "Announcing {} to tracker {} with {}U/{}D/{}L bytes..., type" + request.getEvent().getEventName(), + new Object[] { + request.getHexInfoHash(), + trackerAddr, + this.torrent.getUploaded(), + this.torrent.getDownloaded(), + this.torrent.getLeft() + }); + + try { + URL target = request.buildAnnounceURL(new URL("http://" + trackerAddr + ":6969/announce")); + HTTPTrackerMessage message = transport.announce(target, request); + this.handleTrackerAnnounceResponse(message, inhibitEvents, trackerAddr); + } catch (IOException ioe) { + logger.error(ioe.getClass().getCanonicalName()); + logger.error(ioe.getMessage()); + throw ioe; + } catch (MessageValidationException mve) { + logger.error(mve.getClass().getCanonicalName()); + logger.error(mve.getMessage()); + throw new AnnounceException("Tracker message violates expected " + + "protocol (" + mve.getMessage() + ")", mve); + } catch (AnnounceException ae) { + throw ae; + } + } + + /** + * Build the announce request tracker message. + * + * @param event The announce event (can be NONE or null) + * @return Returns an instance of a {@link HTTPAnnounceRequestMessage} + * that can be used to generate the fully qualified announce URL, with + * parameters, to make the announce request. + * @throws UnsupportedEncodingException + * @throws IOException + * @throws MessageValidationException + */ + private HTTPAnnounceRequestMessage buildAnnounceRequest( + AnnounceRequestMessage.RequestEvent event) + throws UnsupportedEncodingException, IOException, + MessageValidationException { + // Build announce request message + return HTTPAnnounceRequestMessage.craft( + this.torrent.getInfoHash(), + this.peer.getPeerId().array(), + this.peer.getPort(), + this.torrent.getUploaded(), + this.torrent.getDownloaded(), + this.torrent.getLeft(), + true, false, event, + this.peer.getIp(), + AnnounceRequestMessage.DEFAULT_NUM_WANT); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClientTransport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClientTransport.java new file mode 100644 index 0000000..809beab --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClientTransport.java @@ -0,0 +1,84 @@ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.MessageValidationException; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPAnnounceRequestMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPTrackerMessage; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.client.DefaultHttpClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; + +public class TrackerClientTransport { + protected static final Logger logger = LoggerFactory.getLogger(TrackerClientTransport.class); + + public HTTPTrackerMessage announce(URL target, HTTPAnnounceRequestMessage request) throws IOException, MessageValidationException { + InputStream in = null; + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) target.openConnection(); + in = conn.getInputStream(); + logger.info("YZY in connection opened"); + } catch (IOException e) { + in = conn.getErrorStream(); + logger.info("YZY err connection opened"); + if (in == null) + throw new IOException(); + } + + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(in); + HTTPTrackerMessage message = HTTPTrackerMessage.parse(ByteBuffer.wrap(baos.toByteArray())); + return message; + } catch (IOException ioe) { + throw ioe; + } catch (MessageValidationException mve) { + throw mve; + } finally { + // Make sure we close everything down at the end to avoid resource + // leaks. + try { + in.close(); + } catch (IOException ioe) { + logger.warn("Problem ensuring error stream closed!", ioe); + } + + // This means trying to close the error stream as well. + InputStream err = conn.getErrorStream(); + if (err != null) { + try { + err.close(); + } catch (IOException ioe) { + logger.warn("Problem ensuring error stream closed!", ioe); + } + } + + conn.disconnect(); + } + } + + public void postTorrent(String trackerAddr, byte[] torrentEncodedData) { + DefaultHttpClient cli = new DefaultHttpClient(); + HttpPost method = new HttpPost("http://" + trackerAddr + ":6969/preannounce"); + InputStreamEntity reqEntity = new InputStreamEntity(new ByteArrayInputStream(torrentEncodedData), torrentEncodedData.length); + reqEntity.setContentType("application/octet-stream"); + reqEntity.setChunked(false); + method.setEntity(reqEntity); + + try { + cli.execute(method); + } catch (IOException e) { + e.printStackTrace(); + } + cli.getConnectionManager().shutdown(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/MessageListener.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/MessageListener.java new file mode 100644 index 0000000..ba3a5ed --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/MessageListener.java @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage; + +import java.util.EventListener; + + +/** + * EventListener interface for objects that want to receive incoming messages + * from peers. + * + */ +public interface MessageListener extends EventListener { + + public void handleMessage(PeerMessage msg); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerActivityListener.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerActivityListener.java new file mode 100644 index 0000000..1f99c22 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerActivityListener.java @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import org.apache.hadoop.yarn.server.broadcast.client.Piece; + +import java.io.IOException; + +import java.util.BitSet; +import java.util.EventListener; + + +/** + * EventListener interface for objects that want to handle peer activity + * events like piece availability, or piece completion events, and more. + * + */ +public interface PeerActivityListener extends EventListener { + + /** + * Peer choked handler. + * + *

+ * This handler is fired when a peer choked and now refuses to send data to + * us. This means we should not try to request or expect anything from it + * until it becomes ready again. + *

+ * + * @param peer The peer that choked. + */ + public void handlePeerChoked(SharingPeer peer); + + /** + * Peer ready handler. + * + *

+ * This handler is fired when a peer notified that it is no longer choked. + * This means we can send piece block requests to it and start downloading. + *

+ * + * @param peer The peer that became ready. + */ + public void handlePeerReady(SharingPeer peer); + + /** + * Piece availability handler. + * + *

+ * This handler is fired when an update in piece availability is received + * from a peer's HAVE message. + *

+ * + * @param peer The peer we got the update from. + * @param piece The piece that became available from this peer. + */ + public void handlePieceAvailability(SharingPeer peer, Piece piece); + + /** + * Bit field availability handler. + * + *

+ * This handler is fired when an update in piece availability is received + * from a peer's BITFIELD message. + *

+ * + * @param peer The peer we got the update from. + * @param availablePieces The pieces availability bit field of the peer. + */ + public void handleBitfieldAvailability(SharingPeer peer, + BitSet availablePieces); + + /** + * Piece upload completion handler. + * + *

+ * This handler is fired when a piece has been uploaded entirely to a peer. + *

+ * + * @param peer The peer the piece was sent to. + * @param piece The piece in question. + */ + public void handlePieceSent(SharingPeer peer, Piece piece); + + /** + * Piece download completion handler. + * + *

+ * This handler is fired when a piece has been downloaded entirely and the + * piece data has been revalidated. + *

+ * + *

+ * Note: the piece may not be valid after it has been + * downloaded, in which case appropriate action should be taken to + * redownload the piece. + *

+ * + * @param peer The peer we got this piece from. + * @param piece The piece in question. + */ + public void handlePieceCompleted(SharingPeer peer, Piece piece) + throws IOException; + + /** + * Peer disconnection handler. + * + *

+ * This handler is fired when a peer disconnects, or is disconnected due to + * protocol violation. + *

+ * + * @param peer The peer we got this piece from. + */ + public void handlePeerDisconnected(SharingPeer peer); + + /** + * Handler for IOException during peer operation. + * + * @param peer The peer whose activity trigger the exception. + * @param ioe The IOException object, for reporting. + */ + public void handleIOException(SharingPeer peer, IOException ioe); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerExchange.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerExchange.java new file mode 100644 index 0000000..1e0c46c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerExchange.java @@ -0,0 +1,428 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage.Type; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.text.ParseException; +import java.util.BitSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + + +/** + * Incoming and outgoing peer communication system. + * + *

+ * The peer exchange is a wrapper around peer communication. It provides both + * incoming and outgoing communication channels to a connected peer after a + * successful handshake. + *

+ * + *

+ * When a socket is bound to a sharing peer, a PeerExchange is automatically + * created to wrap this socket into a more usable system for communication with + * the remote peer. + *

+ * + *

+ * For incoming messages, the peer exchange provides message parsing and calls + * the handleMessage() method of the peer for each successfully + * parsed message. + *

+ * + *

+ * For outgoing message, the peer exchange offers a send() message + * that queues messages, and takes care of automatically sending a keep-alive + * message to the remote peer every two minutes when other message have been + * sent in that period of time, as recommended by the BitTorrent protocol + * specification. + *

+ * + */ +class PeerExchange { + + private static final Logger logger = + LoggerFactory.getLogger(PeerExchange.class); + + private SharingPeer peer; + private SharedTorrent torrent; + private SocketChannel channel; + + private Set listeners; + + private IncomingThread in; + private OutgoingThread out; + private BlockingQueue sendQueue; + private volatile boolean stop; + + /** + * Initialize and start a new peer exchange. + * + * @param peer The remote peer to communicate with. + * @param torrent The torrent we're exchanging on with the peer. + * @param channel A channel on the connected socket to the peer. + */ + public PeerExchange(SharingPeer peer, SharedTorrent torrent, + SocketChannel channel) throws SocketException { + this.peer = peer; + this.torrent = torrent; + this.channel = channel; + + this.listeners = new HashSet(); + this.sendQueue = new LinkedBlockingQueue(); + + if (!this.peer.hasPeerId()) { + throw new IllegalStateException("Peer does not have a " + + "peer ID. Was the handshake made properly?"); + } + + this.in = new IncomingThread(); + this.in.setName("bt-peer(" + + this.peer.getShortHexPeerId() + ")-recv"); + + this.out = new OutgoingThread(); + this.out.setName("bt-peer(" + + this.peer.getShortHexPeerId() + ")-send"); + this.out.setDaemon(true); + + // Automatically start the exchange activity loops + this.stop = false; + this.in.start(); + this.out.start(); + + logger.debug("Started peer exchange with {} for {}.", + this.peer, this.torrent); + + // If we have pieces, start by sending a BITFIELD message to the peer. + BitSet pieces = this.torrent.getCompletedPieces(); + if (pieces.cardinality() > 0) { + this.send(PeerMessage.BitfieldMessage.craft(pieces)); + } + } + + /** + * Register a new message listener to receive messages. + * + * @param listener The message listener object. + */ + public void register(MessageListener listener) { + this.listeners.add(listener); + } + + /** + * Tells if the peer exchange is active. + */ + public boolean isConnected() { + return this.channel.isConnected(); + } + + /** + * Send a message to the connected peer. + * + *

+ * The message is queued in the outgoing message queue and will be + * processed as soon as possible. + *

+ * + * @param message The message object to send. + */ + public void send(PeerMessage message) { + try { + this.sendQueue.put(message); + } catch (InterruptedException ie) { + // Ignore, our send queue will only block if it contains + // MAX_INTEGER messages, in which case we're already in big + // trouble, and we'd have to be interrupted, too. + } + } + + /** + * Close and stop the peer exchange. + * + *

+ * Closes the socket channel and stops both incoming and outgoing threads. + *

+ */ + public void close() { + this.stop = true; + + if (this.channel.isConnected()) { + IOUtils.closeQuietly(this.channel); + } + + logger.debug("Peer exchange with {} closed.", this.peer); + } + + /** + * Abstract Thread subclass that allows conditional rate limiting + * for PIECE messages. + * + *

+ * To impose rate limits, we only want to throttle when processing PIECE + * messages. All other peer messages should be exchanged as quickly as + * possible. + *

+ * + * @author ptgoetz + */ + private abstract class RateLimitThread extends Thread { + + protected final Rate rate = new Rate(); + protected long sleep = 1000; + + /** + * Dynamically determines an amount of time to sleep, based on the + * average read/write throughput. + * + *

+ * The algorithm is functional, but could certainly be improved upon. + * One obvious drawback is that with large changes in + * maxRate, it will take a while for the sleep time to + * adjust and the throttled rate to "smooth out." + *

+ * + *

+ * Ideally, it would calculate the optimal sleep time necessary to hit + * a desired throughput rather than continuously adjust toward a goal. + *

+ * + * @param maxRate the target rate in kB/second. + * @param messageSize the size, in bytes, of the last message read/written. + * @param message the last PeerMessage read/written. + */ + protected void rateLimit(double maxRate, long messageSize, PeerMessage message) { + if (message.getType() != Type.PIECE || maxRate <= 0) { + return; + } + + try { + this.rate.add(messageSize); + + // Continuously adjust the sleep time to try to hit our target + // rate limit. + if (rate.get() > (maxRate * 1024)) { + Thread.sleep(this.sleep); + this.sleep += 50; + } else { + this.sleep = this.sleep > 50 + ? this.sleep - 50 + : 0; + } + } catch (InterruptedException e) { + // Not critical, eat it. + } + } + } + + /** + * Incoming messages thread. + * + *

+ * The incoming messages thread reads from the socket's input stream and + * waits for incoming messages. When a message is fully retrieve, it is + * parsed and passed to the peer's handleMessage() method that + * will act based on the message type. + *

+ * + * @author mpetazzoni + */ + private class IncomingThread extends RateLimitThread { + + /** + * Read data from the incoming channel of the socket using a {@link + * Selector}. + * + * @param selector The socket selector into which the peer socket has + * been inserted. + * @param buffer A {@link ByteBuffer} to put the read data into. + * @return The number of bytes read. + */ + private long read(Selector selector, ByteBuffer buffer) throws IOException { + if (selector.select(5000) == 0 || !buffer.hasRemaining()) { + return 0; + } + + long size = 0; + Iterator it = selector.selectedKeys().iterator(); + while (it.hasNext()) { + SelectionKey key = (SelectionKey) it.next(); + if (key.isReadable()) { + int read = ((SocketChannel) key.channel()).read(buffer); + if (read < 0) { + throw new IOException("Unexpected end-of-stream while reading"); + } + size += read; + } + it.remove(); + } + + return size; + } + + private void handleIOE(IOException ioe) { + logger.debug("Could not read message from {}: {}", + peer, + ioe.getMessage() != null + ? ioe.getMessage() + : ioe.getClass().getName()); + peer.unbind(true); + } + + @Override + public void run() { + ByteBuffer buffer = ByteBuffer.allocateDirect(1024 * 1024); + Selector selector = null; + + try { + selector = Selector.open(); + channel.register(selector, SelectionKey.OP_READ); + + while (!stop) { + buffer.rewind(); + buffer.limit(PeerMessage.MESSAGE_LENGTH_FIELD_SIZE); + + // Keep reading bytes until the length field has been read + // entirely. + while (!stop && buffer.hasRemaining()) { + this.read(selector, buffer); + } + + // Reset the buffer limit to the expected message size. + int pstrlen = buffer.getInt(0); + buffer.limit(PeerMessage.MESSAGE_LENGTH_FIELD_SIZE + pstrlen); + + long size = 0; + while (!stop && buffer.hasRemaining()) { + size += this.read(selector, buffer); + } + + buffer.rewind(); + + try { + PeerMessage message = PeerMessage.parse(buffer, torrent); + logger.trace("Received {} from {}", message, peer); + + // Wait if needed to reach configured download rate. + this.rateLimit( + PeerExchange.this.torrent.getMaxDownloadRate(), + size, message); + + for (MessageListener listener : listeners) + listener.handleMessage(message); + } catch (ParseException pe) { + logger.warn("{}", pe.getMessage()); + } + } + } catch (IOException ioe) { + this.handleIOE(ioe); + } finally { + try { + if (selector != null) { + selector.close(); + } + } catch (IOException ioe) { + this.handleIOE(ioe); + } + } + } + } + + /** + * Outgoing messages thread. + * + *

+ * The outgoing messages thread waits for messages to appear in the send + * queue and processes them, in order, as soon as they arrive. + *

+ * + *

+ * If no message is available for KEEP_ALIVE_IDLE_MINUTES minutes, it will + * automatically send a keep-alive message to the remote peer to keep the + * connection active. + *

+ * + * @author mpetazzoni + */ + private class OutgoingThread extends RateLimitThread { + + @Override + public void run() { + try { + // Loop until told to stop. When stop was requested, loop until + // the queue is served. + while (!stop || (stop && sendQueue.size() > 0)) { + try { + // Wait for two minutes for a message to send + PeerMessage message = sendQueue.poll( + 5, + TimeUnit.SECONDS); + + if (message == null) { + if (stop) { + return; + } + + message = PeerMessage.KeepAliveMessage.craft(); + } + + logger.trace("Sending {} to {}", message, peer); + + ByteBuffer data = message.getData(); + long size = 0; + while (!stop && data.hasRemaining()) { + int written = channel.write(data); + size += written; + if (written < 0) { + throw new EOFException( + "Reached end of stream while writing"); + } + } + + // Wait if needed to reach configured upload rate. + this.rateLimit(PeerExchange.this.torrent.getMaxUploadRate(), + size, message); + } catch (InterruptedException ie) { + // Ignore and potentially terminate + } + } + } catch (IOException ioe) { + logger.debug("Could not send message to {}: {}", + peer, + ioe.getMessage() != null + ? ioe.getMessage() + : ioe.getClass().getName()); + peer.unbind(true); + } + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/Rate.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/Rate.java new file mode 100644 index 0000000..594268c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/Rate.java @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import java.io.Serializable; +import java.util.Comparator; + + +/** + * A data exchange rate representation. + * + *

+ * This is a utility class to keep track, and compare, of the data exchange + * rate (either download or upload) with a peer. + *

+ * + */ +public class Rate implements Comparable { + + public static final Comparator RATE_COMPARATOR = + new RateComparator(); + + private long bytes = 0; + private long reset = 0; + private long last = 0; + + /** + * Add a byte count to the current measurement. + * + * @param count The number of bytes exchanged since the last reset. + */ + public synchronized void add(long count) { + this.bytes += count; + if (this.reset == 0) { + this.reset = System.currentTimeMillis(); + } + this.last = System.currentTimeMillis(); + } + + /** + * Get the current rate. + * + *

+ * The exchange rate is the number of bytes exchanged since the last + * reset and the last input. + *

+ */ + public synchronized float get() { + if (this.last - this.reset == 0) { + return 0; + } + + return this.bytes / ((this.last - this.reset) / 1000.0f); + } + + /** + * Reset the measurement. + */ + public synchronized void reset() { + this.bytes = 0; + this.reset = System.currentTimeMillis(); + this.last = this.reset; + } + + @Override + public int compareTo(Rate other) { + return RATE_COMPARATOR.compare(this, other); + } + + /** + * A rate comparator. + * + *

+ * This class provides a comparator to sort peers by an exchange rate, + * comparing two rates and returning an ascending ordering. + *

+ * + *

+ * Note: we need to make sure here that we don't return 0, which + * would provide an ordering that is inconsistent with + * equals()'s behavior, and result in unpredictable behavior + * for sorted collections using this comparator. + *

+ * + * @author mpetazzoni + */ + private static class RateComparator + implements Comparator, Serializable { + + private static final long serialVersionUID = 72460233003600L; + + /** + * Compare two rates together. + * + *

+ * This method compares float, but we don't care too much about + * rounding errors. It's just to order peers so super-strict rate based + * order is not required. + *

+ * + * @param a + * @param b + */ + @Override + public int compare(Rate a, Rate b) { + if (a.get() > b.get()) { + return 1; + } + + return -1; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/SharingPeer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/SharingPeer.java new file mode 100644 index 0000000..4991cc7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/SharingPeer.java @@ -0,0 +1,796 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage; +import org.apache.hadoop.yarn.server.broadcast.client.Piece; +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; + +import java.io.IOException; +import java.io.Serializable; +import java.net.SocketException; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.util.BitSet; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * A peer exchanging on a torrent with the BitTorrent client. + * + *

+ * A SharingPeer extends the base Peer class with all the data and logic needed + * by the BitTorrent client to interact with a peer exchanging on the same + * torrent. + *

+ * + *

+ * Peers are defined by their peer ID, IP address and httpPort number, just like + * base peers. Peers we exchange with also contain four crucial attributes: + *

+ * + *
    + *
  • choking, which means we are choking this peer and we're + * not willing to send him anything for now;
  • + *
  • interesting, which means we are interested in a piece + * this peer has;
  • + *
  • choked, if this peer is choking and won't send us + * anything right now;
  • + *
  • interested, if this peer is interested in something we + * have.
  • + *
+ * + *

+ * Peers start choked and uninterested. + *

+ * + */ +public class SharingPeer extends Peer implements MessageListener { + + private static final Logger logger = + LoggerFactory.getLogger(SharingPeer.class); + + private static final int MAX_PIPELINED_REQUESTS = 5; + + private boolean choking; + private boolean interesting; + + private boolean choked; + private boolean interested; + + private SharedTorrent torrent; + private BitSet availablePieces; + + private Piece requestedPiece; + private int lastRequestedOffset; + + private BlockingQueue requests; + private volatile boolean downloading; + + private PeerExchange exchange; + private Rate download; + private Rate upload; + + private Set listeners; + + private Object requestsLock, exchangeLock; + + /** + * Create a new sharing peer on a given torrent. + * + * @param ip The peer's IP address. + * @param port The peer's httpPort. + * @param peerId The byte-encoded peer ID. + * @param torrent The torrent this peer exchanges with us on. + */ + public SharingPeer(String ip, int port, ByteBuffer peerId, + SharedTorrent torrent) { + super(ip, port, peerId); + + this.torrent = torrent; + this.listeners = new HashSet(); + this.availablePieces = new BitSet(this.torrent.getPieceCount()); + + this.requestsLock = new Object(); + this.exchangeLock = new Object(); + + this.reset(); + this.requestedPiece = null; + } + + /** + * Register a new peer activity listener. + * + * @param listener The activity listener that wants to receive events from + * this peer's activity. + */ + public void register(PeerActivityListener listener) { + this.listeners.add(listener); + } + + public Rate getDLRate() { + return this.download; + } + + public Rate getULRate() { + return this.upload; + } + + /** + * Reset the peer state. + * + *

+ * Initially, peers are considered choked, choking, and neither interested + * nor interesting. + *

+ */ + public synchronized void reset() { + this.choking = true; + this.interesting = false; + this.choked = true; + this.interested = false; + + this.exchange = null; + + this.requests = null; + this.lastRequestedOffset = 0; + this.downloading = false; + } + + /** + * Choke this peer. + * + *

+ * We don't want to upload to this peer anymore, so mark that we're choking + * from this peer. + *

+ */ + public void choke() { + if (!this.choking) { + logger.trace("Choking {}", this); + this.send(PeerMessage.ChokeMessage.craft()); + this.choking = true; + } + } + + /** + * Unchoke this peer. + * + *

+ * Mark that we are no longer choking from this peer and can resume + * uploading to it. + *

+ */ + public void unchoke() { + if (this.choking) { + logger.trace("Unchoking {}", this); + this.send(PeerMessage.UnchokeMessage.craft()); + this.choking = false; + } + } + + public boolean isChoking() { + return this.choking; + } + + + public void interesting() { + if (!this.interesting) { + logger.trace("Telling {} we're interested.", this); + this.send(PeerMessage.InterestedMessage.craft()); + this.interesting = true; + } + } + + public void notInteresting() { + if (this.interesting) { + logger.trace("Telling {} we're no longer interested.", this); + this.send(PeerMessage.NotInterestedMessage.craft()); + this.interesting = false; + } + } + + public boolean isInteresting() { + return this.interesting; + } + + + public boolean isChoked() { + return this.choked; + } + + public boolean isInterested() { + return this.interested; + } + + /** + * Returns the available pieces from this peer. + * + * @return A clone of the available pieces bit field from this peer. + */ + public BitSet getAvailablePieces() { + synchronized (this.availablePieces) { + return (BitSet)this.availablePieces.clone(); + } + } + + /** + * Returns the currently requested piece, if any. + */ + public Piece getRequestedPiece() { + return this.requestedPiece; + } + + /** + * Tells whether this peer is a seed. + * + * @return Returns true if the peer has all of the torrent's pieces + * available. + */ + public synchronized boolean isSeed() { + return this.torrent.getPieceCount() > 0 && + this.getAvailablePieces().cardinality() == + this.torrent.getPieceCount(); + } + + /** + * Bind a connected socket to this peer. + * + *

+ * This will create a new peer exchange with this peer using the given + * socket, and register the peer as a message listener. + *

+ * + * @param channel The connected socket channel for this peer. + */ + public synchronized void bind(SocketChannel channel) throws SocketException { + this.unbind(true); + + this.exchange = new PeerExchange(this, this.torrent, channel); + this.exchange.register(this); + + this.download = new Rate(); + this.download.reset(); + + this.upload = new Rate(); + this.upload.reset(); + } + + /** + * Tells whether this peer as an active connection through a peer exchange. + */ + public boolean isConnected() { + synchronized (this.exchangeLock) { + return this.exchange != null && this.exchange.isConnected(); + } + } + + /** + * Unbind and disconnect this peer. + * + *

+ * This terminates the eventually present and/or connected peer exchange + * with the peer and fires the peer disconnected event to any peer activity + * listeners registered on this peer. + *

+ * + * @param force Force unbind without sending cancel requests. + */ + public void unbind(boolean force) { + if (!force) { + // Cancel all outgoing requests, and send a NOT_INTERESTED message to + // the peer. + this.cancelPendingRequests(); + this.send(PeerMessage.NotInterestedMessage.craft()); + } + + synchronized (this.exchangeLock) { + if (this.exchange != null) { + this.exchange.close(); + this.exchange = null; + } + } + + this.firePeerDisconnected(); + this.requestedPiece = null; + } + + /** + * Send a message to the peer. + * + *

+ * Delivery of the message can only happen if the peer is connected. + *

+ * + * @param message The message to send to the remote peer through our peer + * exchange. + */ + public void send(PeerMessage message) throws IllegalStateException { + if (this.isConnected()) { + this.exchange.send(message); + } else { + logger.warn("Attempting to send a message to non-connected peer {}!", this); + } + } + + /** + * Download the given piece from this peer. + * + *

+ * Starts a block request queue and pre-fill it with MAX_PIPELINED_REQUESTS + * block requests. + *

+ * + *

+ * Further requests will be added, one by one, every time a block is + * returned. + *

+ * + * @param piece The piece chosen to be downloaded from this peer. + */ + public synchronized void downloadPiece(Piece piece) + throws IllegalStateException { + if (this.isDownloading()) { + IllegalStateException up = new IllegalStateException( + "Trying to download a piece while previous " + + "download not completed!"); + logger.warn("What's going on? {}", up.getMessage(), up); + throw up; // ah ah. + } + + this.requests = new LinkedBlockingQueue( + SharingPeer.MAX_PIPELINED_REQUESTS); + this.requestedPiece = piece; + this.lastRequestedOffset = 0; + this.requestNextBlocks(); + } + + public boolean isDownloading() { + return this.downloading; + } + + /** + * Request some more blocks from this peer. + * + *

+ * Re-fill the pipeline to get download the next blocks from the peer. + *

+ */ + private void requestNextBlocks() { + synchronized (this.requestsLock) { + if (this.requests == null || this.requestedPiece == null) { + // If we've been taken out of a piece download context it means our + // outgoing requests have been cancelled. Don't enqueue new + // requests until a proper piece download context is + // re-established. + return; + } + + while (this.requests.remainingCapacity() > 0 && + this.lastRequestedOffset < this.requestedPiece.size()) { + PeerMessage.RequestMessage request = PeerMessage.RequestMessage + .craft( + this.requestedPiece.getIndex(), + this.lastRequestedOffset, + Math.min( + (int)(this.requestedPiece.size() - + this.lastRequestedOffset), + PeerMessage.RequestMessage.DEFAULT_REQUEST_SIZE)); + this.requests.add(request); + this.send(request); + this.lastRequestedOffset += request.getLength(); + } + + this.downloading = this.requests.size() > 0; + } + } + + /** + * Remove the REQUEST message from the request pipeline matching this + * PIECE message. + * + *

+ * Upon reception of a piece block with a PIECE message, remove the + * corresponding request from the pipeline to make room for the next block + * requests. + *

+ * + * @param message The PIECE message received. + */ + private void removeBlockRequest(PeerMessage.PieceMessage message) { + synchronized (this.requestsLock) { + if (this.requests == null) { + return; + } + + for (PeerMessage.RequestMessage request : this.requests) { + if (request.getPiece() == message.getPiece() && + request.getOffset() == message.getOffset()) { + this.requests.remove(request); + break; + } + } + + this.downloading = this.requests.size() > 0; + } + } + + /** + * Cancel all pending requests. + * + *

+ * This queues CANCEL messages for all the requests in the queue, and + * returns the list of requests that were in the queue. + *

+ * + *

+ * If no request queue existed, or if it was empty, an empty set of request + * messages is returned. + *

+ */ + public Set cancelPendingRequests() { + synchronized (this.requestsLock) { + Set requests = + new HashSet(); + + if (this.requests != null) { + for (PeerMessage.RequestMessage request : this.requests) { + this.send(PeerMessage.CancelMessage.craft(request.getPiece(), + request.getOffset(), request.getLength())); + requests.add(request); + } + } + + this.requests = null; + this.downloading = false; + return requests; + } + } + + /** + * Handle an incoming message from this peer. + * + * @param msg The incoming, parsed message. + */ + @Override + public synchronized void handleMessage(PeerMessage msg) { + switch (msg.getType()) { + case KEEP_ALIVE: + // Nothing to do, we're keeping the connection open anyways. + break; + case CHOKE: + this.choked = true; + this.firePeerChoked(); + this.cancelPendingRequests(); + break; + case UNCHOKE: + this.choked = false; + logger.trace("Peer {} is now accepting requests.", this); + this.firePeerReady(); + break; + case INTERESTED: + this.interested = true; + break; + case NOT_INTERESTED: + this.interested = false; + break; + case HAVE: + // Record this peer has the given piece + PeerMessage.HaveMessage have = (PeerMessage.HaveMessage)msg; + Piece havePiece = this.torrent.getPiece(have.getPieceIndex()); + + synchronized (this.availablePieces) { + this.availablePieces.set(havePiece.getIndex()); + logger.trace("Peer {} now has {} [{}/{}].", + new Object[] { + this, + havePiece, + this.availablePieces.cardinality(), + this.torrent.getPieceCount() + }); + } + + this.firePieceAvailabity(havePiece); + break; + case BITFIELD: + // Augment the hasPiece bit field from this BITFIELD message + PeerMessage.BitfieldMessage bitfield = + (PeerMessage.BitfieldMessage)msg; + + synchronized (this.availablePieces) { + this.availablePieces.or(bitfield.getBitfield()); + logger.trace("Recorded bitfield from {} with {} " + + "pieces(s) [{}/{}].", + new Object[] { + this, + bitfield.getBitfield().cardinality(), + this.availablePieces.cardinality(), + this.torrent.getPieceCount() + }); + } + + this.fireBitfieldAvailabity(); + break; + case REQUEST: + PeerMessage.RequestMessage request = + (PeerMessage.RequestMessage)msg; + Piece rp = this.torrent.getPiece(request.getPiece()); + + // If we are choking from this peer and it still sends us + // requests, it is a violation of the BitTorrent protocol. + // Similarly, if the peer requests a piece we don't have, it + // is a violation of the BitTorrent protocol. In these + // situation, terminate the connection. + if (this.isChoking() || !rp.isValid()) { + logger.warn("Peer {} violated protocol, " + + "terminating exchange.", this); + this.unbind(true); + break; + } + + if (request.getLength() > + PeerMessage.RequestMessage.MAX_REQUEST_SIZE) { + logger.warn("Peer {} requested a block too big, " + + "terminating exchange.", this); + this.unbind(true); + break; + } + + // At this point we agree to send the requested piece block to + // the remote peer, so let's queue a message with that block + try { + ByteBuffer block = rp.read(request.getOffset(), + request.getLength()); + this.send(PeerMessage.PieceMessage.craft(request.getPiece(), + request.getOffset(), block)); + this.upload.add(block.capacity()); + + if (request.getOffset() + request.getLength() == rp.size()) { + this.firePieceSent(rp); + } + } catch (IOException ioe) { + this.fireIOException(new IOException( + "Error while sending piece block request!", ioe)); + } + + break; + case PIECE: + // Record the incoming piece block. + + // Should we keep track of the requested pieces and act when we + // get a piece we didn't ask for, or should we just stay + // greedy? + PeerMessage.PieceMessage piece = (PeerMessage.PieceMessage)msg; + Piece p = this.torrent.getPiece(piece.getPiece()); + + // Remove the corresponding request from the request queue to + // make room for next block requests. + this.removeBlockRequest(piece); + this.download.add(piece.getBlock().capacity()); + + try { + synchronized (p) { + if (p.isValid()) { + this.requestedPiece = null; + this.cancelPendingRequests(); + this.firePeerReady(); + logger.debug("Discarding block for already completed " + p); + break; + } + + p.record(piece.getBlock(), piece.getOffset()); + + // If the block offset equals the piece size and the block + // length is 0, it means the piece has been entirely + // downloaded. In this case, we have nothing to save, but + // we should validate the piece. + if (piece.getOffset() + piece.getBlock().capacity() + == p.size()) { + p.validate(); + this.firePieceCompleted(p); + this.requestedPiece = null; + this.firePeerReady(); + } else { + this.requestNextBlocks(); + } + } + } catch (IOException ioe) { + this.fireIOException(new IOException( + "Error while storing received piece block!", ioe)); + break; + } + break; + case CANCEL: + // No need to support + break; + } + } + + /** + * Fire the peer choked event to all registered listeners. + * + *

+ * The event contains the peer that chocked. + *

+ */ + private void firePeerChoked() { + for (PeerActivityListener listener : this.listeners) { + listener.handlePeerChoked(this); + } + } + + /** + * Fire the peer ready event to all registered listeners. + * + *

+ * The event contains the peer that unchoked or became ready. + *

+ */ + private void firePeerReady() { + for (PeerActivityListener listener : this.listeners) { + listener.handlePeerReady(this); + } + } + + /** + * Fire the piece availability event to all registered listeners. + * + *

+ * The event contains the peer (this), and the piece that became available. + *

+ */ + private void firePieceAvailabity(Piece piece) { + for (PeerActivityListener listener : this.listeners) { + listener.handlePieceAvailability(this, piece); + } + } + + /** + * Fire the bit field availability event to all registered listeners. + * + * The event contains the peer (this), and the bit field of available pieces + * from this peer. + */ + private void fireBitfieldAvailabity() { + for (PeerActivityListener listener : this.listeners) { + listener.handleBitfieldAvailability(this, + this.getAvailablePieces()); + } + } + + /** + * Fire the piece sent event to all registered listeners. + * + *

+ * The event contains the peer (this), and the piece number that was + * sent to the peer. + *

+ * + * @param piece The completed piece. + */ + private void firePieceSent(Piece piece) { + for (PeerActivityListener listener : this.listeners) { + listener.handlePieceSent(this, piece); + } + } + + /** + * Fire the piece completion event to all registered listeners. + * + *

+ * The event contains the peer (this), and the piece number that was + * completed. + *

+ * + * @param piece The completed piece. + */ + private void firePieceCompleted(Piece piece) throws IOException { + for (PeerActivityListener listener : this.listeners) { + listener.handlePieceCompleted(this, piece); + } + } + + /** + * Fire the peer disconnected event to all registered listeners. + * + *

+ * The event contains the peer that disconnected (this). + *

+ */ + private void firePeerDisconnected() { + for (PeerActivityListener listener : this.listeners) { + listener.handlePeerDisconnected(this); + } + } + + /** + * Fire the IOException event to all registered listeners. + * + *

+ * The event contains the peer that triggered the problem, and the + * exception object. + *

+ */ + private void fireIOException(IOException ioe) { + for (PeerActivityListener listener : this.listeners) { + listener.handleIOException(this, ioe); + } + } + + /** + * Download rate comparator. + * + *

+ * Compares sharing peers based on their current download rate. + *

+ * + * @author mpetazzoni + * @see Rate.RateComparator + */ + public static class DLRateComparator + implements Comparator, Serializable { + + private static final long serialVersionUID = 96307229964730L; + + @Override + public int compare(SharingPeer a, SharingPeer b) { + return Rate.RATE_COMPARATOR.compare(a.getDLRate(), b.getDLRate()); + } + } + + /** + * Upload rate comparator. + * + *

+ * Compares sharing peers based on their current upload rate. + *

+ * + * @author mpetazzoni + * @see Rate.RateComparator + */ + public static class ULRateComparator + implements Comparator, Serializable { + + private static final long serialVersionUID = 38794949747717L; + + @Override + public int compare(SharingPeer a, SharingPeer b) { + return Rate.RATE_COMPARATOR.compare(a.getULRate(), b.getULRate()); + } + } + + public String toString() { + return new StringBuilder(super.toString()) + .append(" [") + .append((this.choked ? "C" : "c")) + .append((this.interested ? "I" : "i")) + .append("|") + .append((this.choking ? "C" : "c")) + .append((this.interesting ? "I" : "i")) + .append("|") + .append(this.availablePieces.cardinality()) + .append("]") + .toString(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/FileCollectionStorage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/FileCollectionStorage.java new file mode 100644 index 0000000..ccc58fe --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/FileCollectionStorage.java @@ -0,0 +1,207 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.storage; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.LinkedList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Multi-file torrent byte storage. + * + *

+ * This implementation of the torrent byte storage provides support for + * multi-file torrents and completely abstracts the read/write operations from + * the notion of different files. The byte storage is represented as one + * continuous byte storage, directly accessible by offset regardless of which + * file this offset lands. + *

+ * + */ +public class FileCollectionStorage implements TorrentByteStorage { + + private static final Logger logger = + LoggerFactory.getLogger(FileCollectionStorage.class); + + private final List files; + private final long size; + + /** + * Initialize a new multi-file torrent byte storage. + * + * @param files The list of individual {@link FileStorage} + * objects making up the torrent. + * @param size The total size of the torrent data, in bytes. + */ + public FileCollectionStorage(List files, + long size) { + this.files = files; + this.size = size; + + logger.info("Initialized torrent byte storage on {} file(s) " + + "({} total byte(s)).", files.size(), size); + } + + @Override + public long size() { + return this.size; + } + + @Override + public int read(ByteBuffer buffer, long offset) throws IOException { + int requested = buffer.remaining(); + int bytes = 0; + + for (FileOffset fo : this.select(offset, requested)) { + // TODO: remove cast to int when large ByteBuffer support is + // implemented in Java. + buffer.limit((int)(bytes + fo.length)); + bytes += fo.file.read(buffer, fo.offset); + } + + if (bytes < requested) { + throw new IOException("Storage collection read underrun!"); + } + + return bytes; + } + + @Override + public int write(ByteBuffer buffer, long offset) throws IOException { + int requested = buffer.remaining(); + + int bytes = 0; + + for (FileOffset fo : this.select(offset, requested)) { + buffer.limit(bytes + (int)fo.length); + bytes += fo.file.write(buffer, fo.offset); + } + + if (bytes < requested) { + throw new IOException("Storage collection write underrun!"); + } + + return bytes; + } + + @Override + public void close() throws IOException { + for (FileStorage file : this.files) { + file.close(); + } + } + + @Override + public void finish() throws IOException { + for (FileStorage file : this.files) { + file.finish(); + } + } + + @Override + public boolean isFinished() { + for (FileStorage file : this.files) { + if (!file.isFinished()) { + return false; + } + } + + return true; + } + + /** + * File operation details holder. + * + *

+ * This simple inner class holds the details for a read or write operation + * on one of the underlying {@link FileStorage}s. + *

+ * + * @author dgiffin + * @author mpetazzoni + */ + private static class FileOffset { + + public final FileStorage file; + public final long offset; + public final long length; + + FileOffset(FileStorage file, long offset, long length) { + this.file = file; + this.offset = offset; + this.length = length; + } + }; + + /** + * Select the group of files impacted by an operation. + * + *

+ * This function selects which files are impacted by a read or write + * operation, with their respective relative offset and chunk length. + *

+ * + * @param offset The offset of the operation, in bytes, relative to the + * complete byte storage. + * @param length The number of bytes to read or write. + * @return A list of {@link FileOffset} objects representing the {@link + * FileStorage}s impacted by the operation, bundled with their + * respective relative offset and number of bytes to read or write. + * @throws IllegalArgumentException If the offset and length go over the + * byte storage size. + * @throws IllegalStateException If the files registered with this byte + * storage can't accommodate the request (should not happen, really). + */ + private List select(long offset, long length) { + if (offset + length > this.size) { + throw new IllegalArgumentException("Buffer overrun (" + + offset + " + " + length + " > " + this.size + ") !"); + } + + List selected = new LinkedList(); + long bytes = 0; + + for (FileStorage file : this.files) { + if (file.offset() >= offset + length) { + break; + } + + if (file.offset() + file.size() < offset) { + continue; + } + + long position = offset - file.offset(); + position = position > 0 ? position : 0; + long size = Math.min( + file.size() - position, + length - bytes); + selected.add(new FileOffset(file, position, size)); + bytes += size; + } + + if (selected.size() == 0 || bytes < length) { + throw new IllegalStateException("Buffer underrun (only got " + + bytes + " out of " + length + " byte(s) requested)!"); + } + + return selected; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/FileStorage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/FileStorage.java new file mode 100644 index 0000000..0abc332 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/FileStorage.java @@ -0,0 +1,180 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.storage; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Single-file torrent byte data storage. + * + *

+ * This implementation of TorrentByteStorageFile provides a torrent byte data + * storage relying on a single underlying file and uses a RandomAccessFile + * FileChannel to expose thread-safe read/write methods. + *

+ * + */ +public class FileStorage implements TorrentByteStorage { + + private static final Logger logger = + LoggerFactory.getLogger(FileStorage.class); + + private final File target; + private final File partial; + private final long offset; + private final long size; + + private RandomAccessFile randomAccessFile; + private FileChannel channel; + private File current; + + public FileStorage(File file, long size) throws IOException { + this(file, 0, size); + } + + public FileStorage(File file, long offset, long size) + throws IOException { + this.target = file; + this.offset = offset; + this.size = size; + + this.partial = new File(this.target.getAbsolutePath() + + TorrentByteStorage.PARTIAL_FILE_NAME_SUFFIX); + + if (this.partial.exists()) { + logger.debug("Partial download found at {}. Continuing...", + this.partial.getAbsolutePath()); + this.current = this.partial; + this.randomAccessFile = new RandomAccessFile(this.current, "rw"); + // Set the file length to the appropriate size, eventually truncating + // or extending the file if it already exists with a different size. + this.randomAccessFile.setLength(this.size); + } else if (!this.target.exists()) { + logger.debug("Downloading new file to {}...", + this.partial.getAbsolutePath()); + this.current = this.partial; + this.randomAccessFile = new RandomAccessFile(this.current, "rw"); + // Set the file length to the appropriate size, eventually truncating + // or extending the file if it already exists with a different size. + this.randomAccessFile.setLength(this.size); + } else { + logger.debug("Using existing file {}.", + this.target.getAbsolutePath()); + this.current = this.target; + this.randomAccessFile = new RandomAccessFile(this.current, "r"); + } + + this.channel = randomAccessFile.getChannel(); + logger.info("Initialized byte storage file at {} " + + "({}+{} byte(s)).", + new Object[] { + this.current.getAbsolutePath(), + this.offset, + this.size, + }); + } + + protected long offset() { + return this.offset; + } + + @Override + public long size() { + return this.size; + } + + @Override + public int read(ByteBuffer buffer, long offset) throws IOException { + int requested = buffer.remaining(); + + if (offset + requested > this.size) { + throw new IllegalArgumentException("Invalid storage read request!"); + } + + int bytes = this.channel.read(buffer, offset); + if (bytes < requested) { + throw new IOException("Storage underrun!"); + } + + return bytes; + } + + @Override + public int write(ByteBuffer buffer, long offset) throws IOException { + int requested = buffer.remaining(); + + if (offset + requested > this.size) { + throw new IllegalArgumentException("Invalid storage write request!"); + } + + return this.channel.write(buffer, offset); + } + + @Override + public synchronized void close() throws IOException { + logger.debug("Closing file channel to " + this.current.getName() + "..."); + if (this.channel.isOpen()) { + this.channel.force(true); + } + this.randomAccessFile.close(); + } + + /** Move the partial file to its final location. + */ + @Override + public synchronized void finish() throws IOException { + logger.debug("Closing file channel to " + this.current.getName() + + " (download complete)."); + if (this.channel.isOpen()) { + this.channel.force(true); + } + + // Nothing more to do if we're already on the target file. + if (this.isFinished()) { + return; + } + + this.randomAccessFile.setLength(this.size); + this.randomAccessFile.close(); + FileUtils.deleteQuietly(this.target); + FileUtils.moveFile(this.current, this.target); + + logger.debug("Re-opening torrent byte storage at {}.", + this.target.getAbsolutePath()); + this.randomAccessFile = new RandomAccessFile(this.target, "r"); + + this.channel = this.randomAccessFile.getChannel(); + this.current = this.target; + FileUtils.deleteQuietly(this.partial); + logger.info("Moved torrent data from {} to {}.", + this.partial.getName(), + this.target.getName()); + } + + @Override + public boolean isFinished() { + return this.current.equals(this.target); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/TorrentByteStorage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/TorrentByteStorage.java new file mode 100644 index 0000000..ac7f083 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/TorrentByteStorage.java @@ -0,0 +1,104 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.storage; + +import java.io.IOException; +import java.nio.ByteBuffer; + + +/** + * Abstract torrent byte storage. + * + *

+ * This interface defines the methods for accessing an abstracted torrent byte + * storage. A torrent, especially when it contains multiple files, needs to be + * seen as one single continuous stream of bytes. Torrent pieces will most + * likely span accross file boundaries. This abstracted byte storage aims at + * providing a simple interface for read/write access to the torrent data, + * regardless of how it is composed underneath the piece structure. + *

+ * + */ +public interface TorrentByteStorage { + + public static final String PARTIAL_FILE_NAME_SUFFIX = ".part"; + + /** + * Returns the total size of the torrent storage. + */ + public long size(); + + /** + * Read from the byte storage. + * + *

+ * Read {@code length} bytes at offset {@code offset} from the underlying + * byte storage and return them in a {@link ByteBuffer}. + *

+ * + * @param buffer The buffer to read the bytes into. The buffer's limit will + * control how many bytes are read from the storage. + * @param offset The offset, in bytes, to read from. This must be within + * the storage boundary. + * @return The number of bytes read from the storage. + * @throws IOException If an I/O error occurs while reading from the + * byte storage. + */ + public int read(ByteBuffer buffer, long offset) throws IOException; + + /** + * Write bytes to the byte storage. + * + *

+ *

+ * + * @param block A {@link ByteBuffer} containing the bytes to write to the + * storage. The buffer limit is expected to be set correctly: all bytes + * from the buffer will be used. + * @param offset Offset in the underlying byte storage to write the block + * at. + * @return The number of bytes written to the storage. + * @throws IOException If an I/O error occurs while writing to the byte + * storage. + */ + public int write(ByteBuffer block, long offset) throws IOException; + + /** + * Close this byte storage. + * + * @throws IOException If closing the underlying storage (file(s) ?) + * failed. + */ + public void close() throws IOException; + + /** + * Finalize the byte storage when the download is complete. + * + *

+ * This gives the byte storage the opportunity to perform finalization + * operations when the download completes, like moving the files from a + * temporary location to their destination. + *

+ * + * @throws IOException If the finalization failed. + */ + public void finish() throws IOException; + + /** + * Tells whether this byte storage has been finalized. + */ + public boolean isFinished(); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategy.java new file mode 100644 index 0000000..8907e5e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategy.java @@ -0,0 +1,29 @@ +package org.apache.hadoop.yarn.server.broadcast.client.strategy; + +import java.util.BitSet; +import java.util.SortedSet; + +import org.apache.hadoop.yarn.server.broadcast.client.Piece; + +/** + * Interface for a piece request strategy provider. + * + * @author cjmalloy + * + */ +public interface RequestStrategy { + + /** + * Choose a piece from the remaining pieces. + * + * @param rarest + * A set sorted by how rare the piece is + * @param interesting + * A set of the index of all interesting pieces + * @param pieces + * The complete array of pieces + * + * @return The chosen piece, or null if no piece is interesting + */ + Piece choosePiece(SortedSet rarest, BitSet interesting, Piece[] pieces); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategyImplRarest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategyImplRarest.java new file mode 100644 index 0000000..1291d50 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategyImplRarest.java @@ -0,0 +1,52 @@ +package org.apache.hadoop.yarn.server.broadcast.client.strategy; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Random; +import java.util.SortedSet; + +import org.apache.hadoop.yarn.server.broadcast.client.Piece; + +/** + * The default request strategy implementation- rarest first. + * + * @author cjmalloy + * + */ +public class RequestStrategyImplRarest implements RequestStrategy { + + /** Randomly select the next piece to download from a peer from the + * RAREST_PIECE_JITTER available from it. */ + private static final int RAREST_PIECE_JITTER = 42; + + private Random random; + + public RequestStrategyImplRarest() { + this.random = new Random(System.currentTimeMillis()); + } + + @Override + public Piece choosePiece(SortedSet rarest, BitSet interesting, Piece[] pieces) { + // Extract the RAREST_PIECE_JITTER rarest pieces from the interesting + // pieces of this peer. + ArrayList choice = new ArrayList(RAREST_PIECE_JITTER); + synchronized (rarest) { + for (Piece piece : rarest) { + if (interesting.get(piece.getIndex())) { + choice.add(piece); + if (choice.size() >= RAREST_PIECE_JITTER) { + break; + } + } + } + } + + if (choice.size() == 0) return null; + + Piece chosen = choice.get( + this.random.nextInt( + Math.min(choice.size(), + RAREST_PIECE_JITTER))); + return chosen; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategyImplSequential.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategyImplSequential.java new file mode 100644 index 0000000..39e2eb4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategyImplSequential.java @@ -0,0 +1,24 @@ +package org.apache.hadoop.yarn.server.broadcast.client.strategy; + +import java.util.BitSet; +import java.util.SortedSet; + +import org.apache.hadoop.yarn.server.broadcast.client.Piece; + +/** + * A sequential request strategy implementation. + * + * @author cjmalloy + * + */ +public class RequestStrategyImplSequential implements RequestStrategy { + + @Override + public Piece choosePiece(SortedSet rarest, BitSet interesting, Piece[] pieces) { + + for (Piece p : pieces) { + if (interesting.get(p.getIndex())) return p; + } + return null; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Peer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Peer.java new file mode 100644 index 0000000..5db0338 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Peer.java @@ -0,0 +1,199 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; + + +/** + * A basic BitTorrent peer. + * + *

+ * This class is meant to be a common base for the tracker and client, which + * would presumably subclass it to extend its functionality and fields. + *

+ * + */ +public class Peer { + + private final InetSocketAddress address; + private final String hostId; + + private ByteBuffer peerId; + private String hexPeerId; + + /** + * Instantiate a new peer. + * + * @param address The peer's address, with httpPort. + */ + public Peer(InetSocketAddress address) { + this(address, null); + } + + /** + * Instantiate a new peer. + * + * @param ip The peer's IP address. + * @param port The peer's httpPort. + */ + public Peer(String ip, int port) { + this(new InetSocketAddress(ip, port), null); + } + + /** + * Instantiate a new peer. + * + * @param ip The peer's IP address. + * @param port The peer's httpPort. + * @param peerId The byte-encoded peer ID. + */ + public Peer(String ip, int port, ByteBuffer peerId) { + this(new InetSocketAddress(ip, port), peerId); + } + + /** + * Instantiate a new peer. + * + * @param address The peer's address, with httpPort. + * @param peerId The byte-encoded peer ID. + */ + public Peer(InetSocketAddress address, ByteBuffer peerId) { + this.address = address; + this.hostId = String.format("%s:%d", + this.address.getAddress(), + this.address.getPort()); + + this.setPeerId(peerId); + } + + /** + * Tells whether this peer has a known peer ID yet or not. + */ + public boolean hasPeerId() { + return this.peerId != null; + } + + /** + * Returns the raw peer ID as a {@link ByteBuffer}. + */ + public ByteBuffer getPeerId() { + return this.peerId; + } + + /** + * Set a peer ID for this peer (usually during handshake). + * + * @param peerId The new peer ID for this peer. + */ + public void setPeerId(ByteBuffer peerId) { + if (peerId != null) { + this.peerId = peerId; + this.hexPeerId = Torrent.byteArrayToHexString(peerId.array()); + } else { + this.peerId = null; + this.hexPeerId = null; + } + } + + /** + * Get the hexadecimal-encoded string representation of this peer's ID. + */ + public String getHexPeerId() { + return this.hexPeerId; + } + + /** + * Get the shortened hexadecimal-encoded peer ID. + */ + public String getShortHexPeerId() { + return String.format("..%s", + this.hexPeerId.substring(this.hexPeerId.length()-6).toUpperCase()); + } + + /** + * Returns this peer's IP address. + */ + public String getIp() { + return this.address.getAddress().getHostAddress(); + } + + /** + * Returns this peer's InetAddress. + */ + public InetAddress getAddress() { + return this.address.getAddress(); + } + + /** + * Returns this peer's httpPort number. + */ + public int getPort() { + return this.address.getPort(); + } + + /** + * Returns this peer's host identifier ("host:httpPort"). + */ + public String getHostIdentifier() { + return this.hostId; + } + + /** + * Returns a binary representation of the peer's IP. + */ + public byte[] getRawIp() { + return this.address.getAddress().getAddress(); + } + + /** + * Returns a human-readable representation of this peer. + */ + public String toString() { + StringBuilder s = new StringBuilder("peer://") + .append(this.getIp()).append(":").append(this.getPort()) + .append("/"); + + if (this.hasPeerId()) { + s.append(this.hexPeerId.substring(this.hexPeerId.length()-6)); + } else { + s.append("?"); + } + + if (this.getPort() < 10000) { + s.append(" "); + } + + return s.toString(); + } + + /** + * Tells if two peers seem to look alike (i.e. they have the same IP, httpPort + * and peer ID if they have one). + */ + public boolean looksLike(Peer other) { + if (other == null) { + return false; + } + + return this.hostId.equals(other.hostId) && + (this.hasPeerId() + ? this.hexPeerId.equals(other.hexPeerId) + : true); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Torrent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Torrent.java new file mode 100644 index 0000000..9bb7a25 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Torrent.java @@ -0,0 +1,823 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BDecoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEncoder; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A torrent file tracked by the controller's BitTorrent tracker. + * + *

+ * This class represents an active torrent on the tracker. The torrent + * information is kept in-memory, and is created from the byte blob one would + * usually find in a .torrent file. + *

+ * + *

+ * Each torrent also keeps a repository of the peers seeding and leeching this + * torrent from the tracker. + *

+ * + * @see Torrent meta-info file structure specification + */ +public class Torrent { + + private static final Logger logger = + LoggerFactory.getLogger(Torrent.class); + + public static final int DEFAULT_PIECE_LENGTH = 512 * 1024; + + public static final int PIECE_HASH_SIZE = 20; + + /** The query parameters encoding when parsing byte strings. */ + public static final String BYTE_ENCODING = "ISO-8859-1"; + + /** + * + * @author dgiffin + * @author mpetazzoni + */ + public static class TorrentFile { + + public final File file; + public final long size; + + public TorrentFile(File file, long size) { + this.file = file; + this.size = size; + } + } + + + protected final byte[] encoded; + protected final byte[] encoded_info; + protected final Map decoded; + protected final Map decoded_info; + + private final byte[] info_hash; + private final String hex_info_hash; + + private final List> trackers; + private final Set allTrackers; + private final Date creationDate; + private final String comment; + private final String createdBy; + private final String name; + private final long size; + private final int pieceLength; + + protected final List files; + + private final boolean seeder; + + /** + * Create a new torrent from meta-info binary data. + * + * Parses the meta-info data (which should be B-encoded as described in the + * BitTorrent specification) and create a Torrent object from it. + * + * @param torrent The meta-info byte data. + * @param seeder Whether we'll be seeding for this torrent or not. + * @throws IOException When the info dictionary can't be read or + * encoded and hashed back to create the torrent's SHA-1 hash. + */ + public Torrent(byte[] torrent, boolean seeder) throws IOException { + this.encoded = torrent; + this.seeder = seeder; + + this.decoded = BDecoder.bdecode( + new ByteArrayInputStream(this.encoded)).getMap(); + + this.decoded_info = this.decoded.get("info").getMap(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BEncoder.bencode(this.decoded_info, baos); + this.encoded_info = baos.toByteArray(); + this.info_hash = Torrent.hash(this.encoded_info); + this.hex_info_hash = Torrent.byteArrayToHexString(this.info_hash); + + /** + * Parses the announce information from the decoded meta-info + * structure. + * + *

+ * If the torrent doesn't define an announce-list, use the mandatory + * announce field value as the single tracker in a single announce + * tier. Otherwise, the announce-list must be parsed and the trackers + * from each tier extracted. + *

+ * + * @see BitTorrent BEP#0012 "Multitracker Metadata Extension" + */ + try { + this.trackers = new ArrayList>(); + this.allTrackers = new HashSet(); + + if (this.decoded.containsKey("announce-list")) { + List tiers = this.decoded.get("announce-list").getList(); + for (BEValue tv : tiers) { + List trackers = tv.getList(); + if (trackers.isEmpty()) { + continue; + } + + List tier = new ArrayList(); + for (BEValue tracker : trackers) { + URI uri = new URI(tracker.getString()); + + // Make sure we're not adding duplicate trackers. + if (!this.allTrackers.contains(uri)) { + tier.add(uri); + this.allTrackers.add(uri); + } + } + + // Only add the tier if it's not empty. + if (!tier.isEmpty()) { + this.trackers.add(tier); + } + } + } else if (this.decoded.containsKey("announce")) { + URI tracker = new URI(this.decoded.get("announce").getString()); + this.allTrackers.add(tracker); + + // Build a single-tier announce list. + List tier = new ArrayList(); + tier.add(tracker); + this.trackers.add(tier); + } + } catch (URISyntaxException use) { + throw new IOException(use); + } + + this.creationDate = this.decoded.containsKey("creation date") + ? new Date(this.decoded.get("creation date").getLong() * 1000) + : null; + this.comment = this.decoded.containsKey("comment") + ? this.decoded.get("comment").getString() + : null; + this.createdBy = this.decoded.containsKey("created by") + ? this.decoded.get("created by").getString() + : null; + this.name = this.decoded_info.get("name").getString(); + this.pieceLength = this.decoded_info.get("piece length").getInt(); + + this.files = new LinkedList(); + + // Parse multi-file torrent file information structure. + if (this.decoded_info.containsKey("files")) { + for (BEValue file : this.decoded_info.get("files").getList()) { + Map fileInfo = file.getMap(); + StringBuilder path = new StringBuilder(); + for (BEValue pathElement : fileInfo.get("path").getList()) { + path.append(File.separator) + .append(pathElement.getString()); + } + this.files.add(new TorrentFile( + new File(this.name, path.toString()), + fileInfo.get("length").getLong())); + } + } else { + // For single-file torrents, the name of the torrent is + // directly the name of the file. + this.files.add(new TorrentFile( + new File(this.name), + this.decoded_info.get("length").getLong())); + } + + // Calculate the total size of this torrent from its files' sizes. + long size = 0; + for (TorrentFile file : this.files) { + size += file.size; + } + this.size = size; + + logger.info("{}-file torrent information:", + this.isMultifile() ? "Multi" : "Single"); + logger.info(" Torrent name: {}", this.name); + logger.info(" Announced at:" + (this.trackers.size() == 0 ? " Seems to be trackerless" : "")); + for (int i=0; i < this.trackers.size(); i++) { + List tier = this.trackers.get(i); + for (int j=0; j < tier.size(); j++) { + logger.info(" {}{}", + (j == 0 ? String.format("%2d. ", i+1) : " "), + tier.get(j)); + } + } + + if (this.creationDate != null) { + logger.info(" Created on..: {}", this.creationDate); + } + if (this.comment != null) { + logger.info(" Comment.....: {}", this.comment); + } + if (this.createdBy != null) { + logger.info(" Created by..: {}", this.createdBy); + } + + if (this.isMultifile()) { + logger.info(" Found {} file(s) in multi-file torrent structure.", + this.files.size()); + int i = 0; + for (TorrentFile file : this.files) { + logger.debug(" {}. {} ({} byte(s))", + new Object[] { + String.format("%2d", ++i), + file.file.getPath(), + String.format("%,d", file.size) + }); + } + } + + logger.info(" Pieces......: {} piece(s) ({} byte(s)/piece)", + (this.size / this.decoded_info.get("piece length").getInt()) + 1, + this.decoded_info.get("piece length").getInt()); + logger.info(" Total size..: {} byte(s)", + String.format("%,d", this.size)); + } + + /** + * Get this torrent's name. + * + *

+ * For a single-file torrent, this is usually the name of the file. For a + * multi-file torrent, this is usually the name of a top-level directory + * containing those files. + *

+ */ + public String getName() { + return this.name; + } + + /** + * Get this torrent's comment string. + */ + public String getComment() { + return this.comment; + } + + /** + * Get this torrent's creator (user, software, whatever...). + */ + public String getCreatedBy() { + return this.createdBy; + } + + /** + * Get the total size of this torrent. + */ + public long getSize() { + return this.size; + } + + /** + * Get the file names from this torrent. + * + * @return The list of relative filenames of all the files described in + * this torrent. + */ + public List getFilenames() { + List filenames = new LinkedList(); + for (TorrentFile file : this.files) { + filenames.add(file.file.getPath()); + } + return filenames; + } + + /** + * Tells whether this torrent is multi-file or not. + */ + public boolean isMultifile() { + return this.files.size() > 1; + } + + /** + * Return the hash of the B-encoded meta-info structure of this torrent. + */ + public byte[] getInfoHash() { + return this.info_hash; + } + + /** + * Get this torrent's info hash (as an hexadecimal-coded string). + */ + public String getHexInfoHash() { + return this.hex_info_hash; + } + + /** + * Return a human-readable representation of this torrent object. + * + *

+ * The torrent's name is used. + *

+ */ + public String toString() { + return this.getName(); + } + + /** + * Return the B-encoded meta-info of this torrent. + */ + public byte[] getEncoded() { + return this.encoded; + } + + /** + * Return the trackers for this torrent. + */ + public List> getAnnounceList() { + return this.trackers; + } + + /** + * Returns the number of trackers for this torrent. + */ + public int getTrackerCount() { + return this.allTrackers.size(); + } + + /** + * Tells whether we were an initial seeder for this torrent. + */ + public boolean isSeeder() { + return this.seeder; + } + + /** + * Save this torrent meta-info structure into a .torrent file. + * + * @param output The stream to write to. + * @throws IOException If an I/O error occurs while writing the file. + */ + public void save(OutputStream output) throws IOException { + output.write(this.getEncoded()); + } + + public static byte[] hash(byte[] data) { + return DigestUtils.sha1(data); + } + + /** + * Convert a byte string to a string containing an hexadecimal + * representation of the original data. + * + * @param bytes The byte array to convert. + */ + public static String byteArrayToHexString(byte[] bytes) { + return new String(Hex.encodeHex(bytes, false)); + } + + /** + * Return an hexadecimal representation of the bytes contained in the + * given string, following the default, expected byte encoding. + * + * @param input The input string. + */ + public static String toHexString(String input) { + try { + byte[] bytes = input.getBytes(Torrent.BYTE_ENCODING); + return Torrent.byteArrayToHexString(bytes); + } catch (UnsupportedEncodingException uee) { + return null; + } + } + + /** + * Determine how many threads to use for the piece hashing. + * + *

+ * If the environment variable TTORRENT_HASHING_THREADS is set to an + * integer value greater than 0, its value will be used. Otherwise, it + * defaults to the number of processors detected by the Java Runtime. + *

+ * + * @return How many threads to use for concurrent piece hashing. + */ + protected static int getHashingThreadsCount() { + String threads = System.getenv("TTORRENT_HASHING_THREADS"); + + if (threads != null) { + try { + int count = Integer.parseInt(threads); + if (count > 0) { + return count; + } + } catch (NumberFormatException nfe) { + // Pass + } + } + + return Runtime.getRuntime().availableProcessors(); + } + + /** Torrent loading ---------------------------------------------------- */ + + /** + * Load a torrent from the given torrent file. + * + *

+ * This method assumes we are not a seeder and that local data needs to be + * validated. + *

+ * + * @param torrent The abstract {@link File} object representing the + * .torrent file to load. + * @throws IOException When the torrent file cannot be read. + */ + public static Torrent load(File torrent) throws IOException { + return Torrent.load(torrent, false); + } + + /** + * Load a torrent from the given torrent file. + * + * @param torrent The abstract {@link File} object representing the + * .torrent file to load. + * @param seeder Whether we are a seeder for this torrent or not (disables + * local data validation). + * @throws IOException When the torrent file cannot be read. + */ + public static Torrent load(File torrent, boolean seeder) + throws IOException { + byte[] data = FileUtils.readFileToByteArray(torrent); + return new Torrent(data, seeder); + } + + /** Torrent creation --------------------------------------------------- */ + + /** + * Create a {@link Torrent} object for a file. + * + *

+ * Hash the given file to create the {@link Torrent} object representing + * the Torrent metainfo about this file, needed for announcing and/or + * sharing said file. + *

+ * + * @param source The file to use in the torrent. + * @param announce The announce URI that will be used for this torrent. + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + public static Torrent create(File source, URI announce, String createdBy) + throws InterruptedException, IOException { + return Torrent.create(source, null, DEFAULT_PIECE_LENGTH, + announce, null, createdBy); + } + + /** + * Create a {@link Torrent} object for a set of files. + * + *

+ * Hash the given files to create the multi-file {@link Torrent} object + * representing the Torrent meta-info about them, needed for announcing + * and/or sharing these files. Since we created the torrent, we're + * considering we'll be a full initial seeder for it. + *

+ * + * @param parent The parent directory or location of the torrent files, + * also used as the torrent's name. + * @param files The files to add into this torrent. + * @param announce The announce URI that will be used for this torrent. + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + public static Torrent create(File parent, List files, URI announce, + String createdBy) throws InterruptedException, IOException { + return Torrent.create(parent, files, DEFAULT_PIECE_LENGTH, + announce, null, createdBy); + } + + /** + * Create a {@link Torrent} object for a file. + * + *

+ * Hash the given file to create the {@link Torrent} object representing + * the Torrent metainfo about this file, needed for announcing and/or + * sharing said file. + *

+ * + * @param source The file to use in the torrent. + * @param announceList The announce URIs organized as tiers that will + * be used for this torrent + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + public static Torrent create(File source, int pieceLength, List> announceList, + String createdBy) throws InterruptedException, IOException { + return Torrent.create(source, null, pieceLength, + null, announceList, createdBy); + } + + /** + * Create a {@link Torrent} object for a set of files. + * + *

+ * Hash the given files to create the multi-file {@link Torrent} object + * representing the Torrent meta-info about them, needed for announcing + * and/or sharing these files. Since we created the torrent, we're + * considering we'll be a full initial seeder for it. + *

+ * + * @param source The parent directory or location of the torrent files, + * also used as the torrent's name. + * @param files The files to add into this torrent. + * @param announceList The announce URIs organized as tiers that will + * be used for this torrent + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + public static Torrent create(File source, List files, int pieceLength, + List> announceList, String createdBy) + throws InterruptedException, IOException { + return Torrent.create(source, files, pieceLength, + null, announceList, createdBy); + } + + /** + * Helper method to create a {@link Torrent} object for a set of files. + * + *

+ * Hash the given files to create the multi-file {@link Torrent} object + * representing the Torrent meta-info about them, needed for announcing + * and/or sharing these files. Since we created the torrent, we're + * considering we'll be a full initial seeder for it. + *

+ * + * @param parent The parent directory or location of the torrent files, + * also used as the torrent's name. + * @param files The files to add into this torrent. + * @param announce The announce URI that will be used for this torrent. + * @param announceList The announce URIs organized as tiers that will + * be used for this torrent + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + private static Torrent create(File parent, List files, int pieceLength, + URI announce, List> announceList, String createdBy) + throws InterruptedException, IOException { + if (files == null || files.isEmpty()) { + logger.info("Creating single-file torrent for {}...", + parent.getName()); + } else { + logger.info("Creating {}-file torrent {}...", + files.size(), parent.getName()); + } + + Map torrent = new HashMap(); + + if (announce != null) { + torrent.put("announce", new BEValue(announce.toString())); + } + if (announceList != null) { + List tiers = new LinkedList(); + for (List trackers : announceList) { + List tierInfo = new LinkedList(); + for (URI trackerURI : trackers) { + tierInfo.add(new BEValue(trackerURI.toString())); + } + tiers.add(new BEValue(tierInfo)); + } + torrent.put("announce-list", new BEValue(tiers)); + } + + torrent.put("creation date", new BEValue(new Date().getTime() / 1000)); + torrent.put("created by", new BEValue(createdBy)); + + Map info = new TreeMap(); + info.put("name", new BEValue(parent.getName())); + info.put("piece length", new BEValue(pieceLength)); + + if (files == null || files.isEmpty()) { + info.put("length", new BEValue(parent.length())); + info.put("pieces", new BEValue(Torrent.hashFile(parent, pieceLength), + Torrent.BYTE_ENCODING)); + } else { + List fileInfo = new LinkedList(); + for (File file : files) { + Map fileMap = new HashMap(); + fileMap.put("length", new BEValue(file.length())); + + LinkedList filePath = new LinkedList(); + while (file != null) { + if (file.equals(parent)) { + break; + } + + filePath.addFirst(new BEValue(file.getName())); + file = file.getParentFile(); + } + + fileMap.put("path", new BEValue(filePath)); + fileInfo.add(new BEValue(fileMap)); + } + info.put("files", new BEValue(fileInfo)); + info.put("pieces", new BEValue(Torrent.hashFiles(files, pieceLength), + Torrent.BYTE_ENCODING)); + } + torrent.put("info", new BEValue(info)); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BEncoder.bencode(new BEValue(torrent), baos); + return new Torrent(baos.toByteArray(), true); + } + + /** + * A {@link Callable} to hash a data chunk. + * + * @author mpetazzoni + */ + private static class CallableChunkHasher implements Callable { + + private final MessageDigest md; + private final ByteBuffer data; + + CallableChunkHasher(ByteBuffer buffer) { + this.md = DigestUtils.getSha1Digest(); + + this.data = ByteBuffer.allocate(buffer.remaining()); + buffer.mark(); + this.data.put(buffer); + this.data.clear(); + buffer.reset(); + } + + @Override + public String call() throws UnsupportedEncodingException { + this.md.reset(); + this.md.update(this.data.array()); + return new String(md.digest(), Torrent.BYTE_ENCODING); + } + } + + /** + * Return the concatenation of the SHA-1 hashes of a file's pieces. + * + *

+ * Hashes the given file piece by piece using the default Torrent piece + * length (see {@link #PIECE_LENGTH}) and returns the concatenation of + * these hashes, as a string. + *

+ * + *

+ * This is used for creating Torrent meta-info structures from a file. + *

+ * + * @param file The file to hash. + */ + private static String hashFile(File file, int pieceLenght) + throws InterruptedException, IOException { + return Torrent.hashFiles(Arrays.asList(new File[] { file }), pieceLenght); + } + + private static String hashFiles(List files, int pieceLenght) + throws InterruptedException, IOException { + int threads = getHashingThreadsCount(); + ExecutorService executor = Executors.newFixedThreadPool(threads); + ByteBuffer buffer = ByteBuffer.allocate(pieceLenght); + List> results = new LinkedList>(); + StringBuilder hashes = new StringBuilder(); + + long length = 0L; + int pieces = 0; + + long start = System.nanoTime(); + for (File file : files) { + logger.info("Hashing data from {} with {} threads ({} pieces)...", + new Object[] { + file.getName(), + threads, + (int) (Math.ceil( + (double)file.length() / pieceLenght)) + }); + + length += file.length(); + + FileInputStream fis = new FileInputStream(file); + FileChannel channel = fis.getChannel(); + int step = 10; + + try { + while (channel.read(buffer) > 0) { + if (buffer.remaining() == 0) { + buffer.clear(); + results.add(executor.submit(new CallableChunkHasher(buffer))); + } + + if (results.size() >= threads) { + pieces += accumulateHashes(hashes, results); + } + + if (channel.position() / (double)channel.size() * 100f > step) { + logger.info(" ... {}% complete", step); + step += 10; + } + } + } finally { + channel.close(); + fis.close(); + } + } + + // Hash the last bit, if any + if (buffer.position() > 0) { + buffer.limit(buffer.position()); + buffer.position(0); + results.add(executor.submit(new CallableChunkHasher(buffer))); + } + + pieces += accumulateHashes(hashes, results); + + // Request orderly executor shutdown and wait for hashing tasks to + // complete. + executor.shutdown(); + while (!executor.isTerminated()) { + Thread.sleep(10); + } + long elapsed = System.nanoTime() - start; + + int expectedPieces = (int) (Math.ceil( + (double)length / pieceLenght)); + logger.info("Hashed {} file(s) ({} bytes) in {} pieces ({} expected) in {}ms.", + new Object[] { + files.size(), + length, + pieces, + expectedPieces, + String.format("%.1f", elapsed/1e6), + }); + + return hashes.toString(); + } + + /** + * Accumulate the piece hashes into a given {@link StringBuilder}. + * + * @param hashes The {@link StringBuilder} to append hashes to. + * @param results The list of {@link Future}s that will yield the piece + * hashes. + */ + private static int accumulateHashes(StringBuilder hashes, + List> results) throws InterruptedException, IOException { + try { + int pieces = results.size(); + for (Future chunk : results) { + hashes.append(chunk.get()); + } + results.clear(); + return pieces; + } catch (ExecutionException ee) { + throw new IOException("Error while hashing the torrent data!", ee); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/PeerMessage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/PeerMessage.java new file mode 100644 index 0000000..6d547d7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/PeerMessage.java @@ -0,0 +1,670 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common.protocol; + +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; + +import java.nio.ByteBuffer; +import java.text.ParseException; +import java.util.BitSet; + +/** + * BitTorrent peer protocol messages representations. + * + *

+ * This class and its *Messages subclasses provide POJO + * representations of the peer protocol messages, along with easy parsing from + * an input ByteBuffer to quickly get a usable representation of an incoming + * message. + *

+ * + * @author mpetazzoni + * @see BitTorrent peer wire protocol + */ +public abstract class PeerMessage { + + /** The size, in bytes, of the length field in a message (one 32-bit + * integer). */ + public static final int MESSAGE_LENGTH_FIELD_SIZE = 4; + + /** + * Message type. + * + *

+ * Note that the keep-alive messages don't actually have an type ID defined + * in the protocol as they are of length 0. + *

+ */ + public enum Type { + KEEP_ALIVE(-1), + CHOKE(0), + UNCHOKE(1), + INTERESTED(2), + NOT_INTERESTED(3), + HAVE(4), + BITFIELD(5), + REQUEST(6), + PIECE(7), + CANCEL(8); + + private byte id; + Type(int id) { + this.id = (byte)id; + } + + public boolean equals(byte c) { + return this.id == c; + } + + public byte getTypeByte() { + return this.id; + } + + public static Type get(byte c) { + for (Type t : Type.values()) { + if (t.equals(c)) { + return t; + } + } + return null; + } + }; + + private final Type type; + private final ByteBuffer data; + + private PeerMessage(Type type, ByteBuffer data) { + this.type = type; + this.data = data; + this.data.rewind(); + } + + public Type getType() { + return this.type; + } + + /** + * Returns a {@link ByteBuffer} backed by the same data as this message. + * + *

+ * This method returns a duplicate of the buffer stored in this {@link + * PeerMessage} object to allow for multiple consumers to read from the + * same message without conflicting access to the buffer's position, mark + * and limit. + *

+ */ + public ByteBuffer getData() { + return this.data.duplicate(); + } + + /** + * Validate that this message makes sense for the torrent it's related to. + * + *

+ * This method is meant to be overloaded by distinct message types, where + * it makes sense. Otherwise, it defaults to true. + *

+ * + * @param torrent The torrent this message is about. + */ + public PeerMessage validate(SharedTorrent torrent) + throws MessageValidationException { + return this; + } + + public String toString() { + return this.getType().name(); + } + + /** + * Parse the given buffer into a peer protocol message. + * + *

+ * Parses the provided byte array and builds the corresponding PeerMessage + * subclass object. + *

+ * + * @param buffer The byte buffer containing the message data. + * @param torrent The torrent this message is about. + * @return A PeerMessage subclass instance. + * @throws ParseException When the message is invalid, can't be parsed or + * does not match the protocol requirements. + */ + public static PeerMessage parse(ByteBuffer buffer, SharedTorrent torrent) + throws ParseException { + int length = buffer.getInt(); + if (length == 0) { + return KeepAliveMessage.parse(buffer, torrent); + } else if (length != buffer.remaining()) { + throw new ParseException("Message size did not match announced " + + "size!", 0); + } + + Type type = Type.get(buffer.get()); + if (type == null) { + throw new ParseException("Unknown message ID!", + buffer.position()-1); + } + + switch (type) { + case CHOKE: + return ChokeMessage.parse(buffer.slice(), torrent); + case UNCHOKE: + return UnchokeMessage.parse(buffer.slice(), torrent); + case INTERESTED: + return InterestedMessage.parse(buffer.slice(), torrent); + case NOT_INTERESTED: + return NotInterestedMessage.parse(buffer.slice(), torrent); + case HAVE: + return HaveMessage.parse(buffer.slice(), torrent); + case BITFIELD: + return BitfieldMessage.parse(buffer.slice(), torrent); + case REQUEST: + return RequestMessage.parse(buffer.slice(), torrent); + case PIECE: + return PieceMessage.parse(buffer.slice(), torrent); + case CANCEL: + return CancelMessage.parse(buffer.slice(), torrent); + default: + throw new IllegalStateException("Message type should have " + + "been properly defined by now."); + } + } + + public static class MessageValidationException extends ParseException { + + static final long serialVersionUID = -1; + + public MessageValidationException(PeerMessage m) { + super("Message " + m + " is not valid!", 0); + } + + } + + + /** + * Keep alive message. + * + * + */ + public static class KeepAliveMessage extends PeerMessage { + + private static final int BASE_SIZE = 0; + + private KeepAliveMessage(ByteBuffer buffer) { + super(Type.KEEP_ALIVE, buffer); + } + + public static KeepAliveMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + return (KeepAliveMessage)new KeepAliveMessage(buffer) + .validate(torrent); + } + + public static KeepAliveMessage craft() { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + KeepAliveMessage.BASE_SIZE); + buffer.putInt(KeepAliveMessage.BASE_SIZE); + return new KeepAliveMessage(buffer); + } + } + + /** + * Choke message. + * + * + */ + public static class ChokeMessage extends PeerMessage { + + private static final int BASE_SIZE = 1; + + private ChokeMessage(ByteBuffer buffer) { + super(Type.CHOKE, buffer); + } + + public static ChokeMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + return (ChokeMessage)new ChokeMessage(buffer) + .validate(torrent); + } + + public static ChokeMessage craft() { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + ChokeMessage.BASE_SIZE); + buffer.putInt(ChokeMessage.BASE_SIZE); + buffer.put(PeerMessage.Type.CHOKE.getTypeByte()); + return new ChokeMessage(buffer); + } + } + + /** + * Unchoke message. + * + * + */ + public static class UnchokeMessage extends PeerMessage { + + private static final int BASE_SIZE = 1; + + private UnchokeMessage(ByteBuffer buffer) { + super(Type.UNCHOKE, buffer); + } + + public static UnchokeMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + return (UnchokeMessage)new UnchokeMessage(buffer) + .validate(torrent); + } + + public static UnchokeMessage craft() { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + UnchokeMessage.BASE_SIZE); + buffer.putInt(UnchokeMessage.BASE_SIZE); + buffer.put(PeerMessage.Type.UNCHOKE.getTypeByte()); + return new UnchokeMessage(buffer); + } + } + + /** + * Interested message. + * + * + */ + public static class InterestedMessage extends PeerMessage { + + private static final int BASE_SIZE = 1; + + private InterestedMessage(ByteBuffer buffer) { + super(Type.INTERESTED, buffer); + } + + public static InterestedMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + return (InterestedMessage)new InterestedMessage(buffer) + .validate(torrent); + } + + public static InterestedMessage craft() { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + InterestedMessage.BASE_SIZE); + buffer.putInt(InterestedMessage.BASE_SIZE); + buffer.put(PeerMessage.Type.INTERESTED.getTypeByte()); + return new InterestedMessage(buffer); + } + } + + /** + * Not interested message. + * + * + */ + public static class NotInterestedMessage extends PeerMessage { + + private static final int BASE_SIZE = 1; + + private NotInterestedMessage(ByteBuffer buffer) { + super(Type.NOT_INTERESTED, buffer); + } + + public static NotInterestedMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + return (NotInterestedMessage)new NotInterestedMessage(buffer) + .validate(torrent); + } + + public static NotInterestedMessage craft() { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + NotInterestedMessage.BASE_SIZE); + buffer.putInt(NotInterestedMessage.BASE_SIZE); + buffer.put(PeerMessage.Type.NOT_INTERESTED.getTypeByte()); + return new NotInterestedMessage(buffer); + } + } + + /** + * Have message. + * + * + */ + public static class HaveMessage extends PeerMessage { + + private static final int BASE_SIZE = 5; + + private int piece; + + private HaveMessage(ByteBuffer buffer, int piece) { + super(Type.HAVE, buffer); + this.piece = piece; + } + + public int getPieceIndex() { + return this.piece; + } + + @Override + public HaveMessage validate(SharedTorrent torrent) + throws MessageValidationException { + if (this.piece >= 0 && this.piece < torrent.getPieceCount()) { + return this; + } + + throw new MessageValidationException(this); + } + + public static HaveMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + return new HaveMessage(buffer, buffer.getInt()) + .validate(torrent); + } + + public static HaveMessage craft(int piece) { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + HaveMessage.BASE_SIZE); + buffer.putInt(HaveMessage.BASE_SIZE); + buffer.put(PeerMessage.Type.HAVE.getTypeByte()); + buffer.putInt(piece); + return new HaveMessage(buffer, piece); + } + + public String toString() { + return super.toString() + " #" + this.getPieceIndex(); + } + } + + /** + * Bitfield message. + * + * + */ + public static class BitfieldMessage extends PeerMessage { + + private static final int BASE_SIZE = 1; + + private BitSet bitfield; + + private BitfieldMessage(ByteBuffer buffer, BitSet bitfield) { + super(Type.BITFIELD, buffer); + this.bitfield = bitfield; + } + + public BitSet getBitfield() { + return this.bitfield; + } + + @Override + public BitfieldMessage validate(SharedTorrent torrent) + throws MessageValidationException { + if (this.bitfield.length() <= torrent.getPieceCount()) { + return this; + } + + throw new MessageValidationException(this); + } + + public static BitfieldMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + BitSet bitfield = new BitSet(buffer.remaining()*8); + for (int i=0; i < buffer.remaining()*8; i++) { + if ((buffer.get(i/8) & (1 << (7 -(i % 8)))) > 0) { + bitfield.set(i); + } + } + + return new BitfieldMessage(buffer, bitfield) + .validate(torrent); + } + + public static BitfieldMessage craft(BitSet availablePieces) { + byte[] bitfield = new byte[ + (int) Math.ceil((double)availablePieces.length()/8)]; + for (int i=availablePieces.nextSetBit(0); i >= 0; + i=availablePieces.nextSetBit(i+1)) { + bitfield[i/8] |= 1 << (7 -(i % 8)); + } + + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + BitfieldMessage.BASE_SIZE + bitfield.length); + buffer.putInt(BitfieldMessage.BASE_SIZE + bitfield.length); + buffer.put(PeerMessage.Type.BITFIELD.getTypeByte()); + buffer.put(ByteBuffer.wrap(bitfield)); + return new BitfieldMessage(buffer, availablePieces); + } + + public String toString() { + return super.toString() + " " + this.getBitfield().cardinality(); + } + } + + /** + * Request message. + * + * + */ + public static class RequestMessage extends PeerMessage { + + private static final int BASE_SIZE = 13; + + /** Default block size is 2^14 bytes, or 16kB. */ + public static final int DEFAULT_REQUEST_SIZE = 16384; + + /** Max block request size is 2^17 bytes, or 131kB. */ + public static final int MAX_REQUEST_SIZE = 131072; + + private int piece; + private int offset; + private int length; + + private RequestMessage(ByteBuffer buffer, int piece, + int offset, int length) { + super(Type.REQUEST, buffer); + this.piece = piece; + this.offset = offset; + this.length = length; + } + + public int getPiece() { + return this.piece; + } + + public int getOffset() { + return this.offset; + } + + public int getLength() { + return this.length; + } + + @Override + public RequestMessage validate(SharedTorrent torrent) + throws MessageValidationException { + if (this.piece >= 0 && this.piece < torrent.getPieceCount() && + this.offset + this.length <= + torrent.getPiece(this.piece).size()) { + return this; + } + + throw new MessageValidationException(this); + } + + public static RequestMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + int piece = buffer.getInt(); + int offset = buffer.getInt(); + int length = buffer.getInt(); + return new RequestMessage(buffer, piece, + offset, length).validate(torrent); + } + + public static RequestMessage craft(int piece, int offset, int length) { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + RequestMessage.BASE_SIZE); + buffer.putInt(RequestMessage.BASE_SIZE); + buffer.put(PeerMessage.Type.REQUEST.getTypeByte()); + buffer.putInt(piece); + buffer.putInt(offset); + buffer.putInt(length); + return new RequestMessage(buffer, piece, offset, length); + } + + public String toString() { + return super.toString() + " #" + this.getPiece() + + " (" + this.getLength() + "@" + this.getOffset() + ")"; + } + } + + /** + * Piece message. + * + * + */ + public static class PieceMessage extends PeerMessage { + + private static final int BASE_SIZE = 9; + + private int piece; + private int offset; + private ByteBuffer block; + + private PieceMessage(ByteBuffer buffer, int piece, + int offset, ByteBuffer block) { + super(Type.PIECE, buffer); + this.piece = piece; + this.offset = offset; + this.block = block; + } + + public int getPiece() { + return this.piece; + } + + public int getOffset() { + return this.offset; + } + + public ByteBuffer getBlock() { + return this.block; + } + + @Override + public PieceMessage validate(SharedTorrent torrent) + throws MessageValidationException { + if (this.piece >= 0 && this.piece < torrent.getPieceCount() && + this.offset + this.block.limit() <= + torrent.getPiece(this.piece).size()) { + return this; + } + + throw new MessageValidationException(this); + } + + public static PieceMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + int piece = buffer.getInt(); + int offset = buffer.getInt(); + ByteBuffer block = buffer.slice(); + return new PieceMessage(buffer, piece, offset, block) + .validate(torrent); + } + + public static PieceMessage craft(int piece, int offset, + ByteBuffer block) { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + PieceMessage.BASE_SIZE + block.capacity()); + buffer.putInt(PieceMessage.BASE_SIZE + block.capacity()); + buffer.put(PeerMessage.Type.PIECE.getTypeByte()); + buffer.putInt(piece); + buffer.putInt(offset); + buffer.put(block); + return new PieceMessage(buffer, piece, offset, block); + } + + public String toString() { + return super.toString() + " #" + this.getPiece() + + " (" + this.getBlock().capacity() + "@" + this.getOffset() + ")"; + } + } + + /** + * Cancel message. + * + * + */ + public static class CancelMessage extends PeerMessage { + + private static final int BASE_SIZE = 13; + + private int piece; + private int offset; + private int length; + + private CancelMessage(ByteBuffer buffer, int piece, + int offset, int length) { + super(Type.CANCEL, buffer); + this.piece = piece; + this.offset = offset; + this.length = length; + } + + public int getPiece() { + return this.piece; + } + + public int getOffset() { + return this.offset; + } + + public int getLength() { + return this.length; + } + + @Override + public CancelMessage validate(SharedTorrent torrent) + throws MessageValidationException { + if (this.piece >= 0 && this.piece < torrent.getPieceCount() && + this.offset + this.length <= + torrent.getPiece(this.piece).size()) { + return this; + } + + throw new MessageValidationException(this); + } + + public static CancelMessage parse(ByteBuffer buffer, + SharedTorrent torrent) throws MessageValidationException { + int piece = buffer.getInt(); + int offset = buffer.getInt(); + int length = buffer.getInt(); + return new CancelMessage(buffer, piece, + offset, length).validate(torrent); + } + + public static CancelMessage craft(int piece, int offset, int length) { + ByteBuffer buffer = ByteBuffer.allocateDirect( + MESSAGE_LENGTH_FIELD_SIZE + CancelMessage.BASE_SIZE); + buffer.putInt(CancelMessage.BASE_SIZE); + buffer.put(PeerMessage.Type.CANCEL.getTypeByte()); + buffer.putInt(piece); + buffer.putInt(offset); + buffer.putInt(length); + return new CancelMessage(buffer, piece, offset, length); + } + + public String toString() { + return super.toString() + " #" + this.getPiece() + + " (" + this.getLength() + "@" + this.getOffset() + ")"; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/TrackerMessage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/TrackerMessage.java new file mode 100644 index 0000000..6692837 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/TrackerMessage.java @@ -0,0 +1,290 @@ +/** + * Copyright (C) 2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common.protocol; + +import org.apache.hadoop.yarn.server.broadcast.common.Peer; + +import java.nio.ByteBuffer; +import java.util.List; + + +/** + * BitTorrent tracker protocol messages representations. + * + *

+ * This class and its *TrackerMessage subclasses provide POJO + * representations of the tracker protocol messages, for at least HTTP and UDP + * trackers' protocols, along with easy parsing from an input ByteBuffer to + * quickly get a usable representation of an incoming message. + *

+ * + * @author mpetazzoni + */ +public abstract class TrackerMessage { + + /** + * Message type. + */ + public enum Type { + UNKNOWN(-1), + CONNECT_REQUEST(0), + CONNECT_RESPONSE(0), + ANNOUNCE_REQUEST(1), + ANNOUNCE_RESPONSE(1), + SCRAPE_REQUEST(2), + SCRAPE_RESPONSE(2), + ERROR(3); + + private final int id; + + Type(int id) { + this.id = id; + } + + public int getId() { + return this.id; + } + }; + + private final Type type; + private final ByteBuffer data; + + /** + * Constructor for the base tracker message type. + * + * @param type The message type. + * @param data A byte buffer containing the binary data of the message (a + * B-encoded map, a UDP packet data, etc.). + */ + protected TrackerMessage(Type type, ByteBuffer data) { + this.type = type; + this.data = data; + if (this.data != null) { + this.data.rewind(); + } + } + + /** + * Returns the type of this tracker message. + */ + public Type getType() { + return this.type; + } + + /** + * Returns the encoded binary data for this message. + */ + public ByteBuffer getData() { + return this.data; + } + + /** + * Generic exception for message format and message validation exceptions. + */ + public static class MessageValidationException extends Exception { + + static final long serialVersionUID = -1; + + public MessageValidationException(String s) { + super(s); + } + + public MessageValidationException(String s, Throwable cause) { + super(s, cause); + } + + } + + + /** + * Base interface for connection request messages. + * + *

+ * This interface must be implemented by all subtypes of connection request + * messages for the various tracker protocols. + *

+ * + * @author mpetazzoni + */ + public interface ConnectionRequestMessage { + + }; + + + /** + * Base interface for connection response messages. + * + *

+ * This interface must be implemented by all subtypes of connection + * response messages for the various tracker protocols. + *

+ * + * @author mpetazzoni + */ + public interface ConnectionResponseMessage { + + }; + + + /** + * Base interface for announce request messages. + * + *

+ * This interface must be implemented by all subtypes of announce request + * messages for the various tracker protocols. + *

+ * + * @author mpetazzoni + */ + public interface AnnounceRequestMessage { + + public static final int DEFAULT_NUM_WANT = 50; + + /** + * Announce request event types. + * + *

+ * When the client starts exchanging on a torrent, it must contact the + * torrent's tracker with a 'started' announce request, which notifies the + * tracker this client now exchanges on this torrent (and thus allows the + * tracker to report the existence of this peer to other clients). + *

+ * + *

+ * When the client stops exchanging, or when its download completes, it must + * also send a specific announce request. Otherwise, the client must send an + * eventless (NONE), periodic announce request to the tracker at an + * interval specified by the tracker itself, allowing the tracker to + * refresh this peer's status and acknowledge that it is still there. + *

+ */ + public enum RequestEvent { + NONE(0), + COMPLETED(1), + STARTED(2), + STOPPED(3); + + private final int id; + RequestEvent(int id) { + this.id = id; + } + + public String getEventName() { + return this.name().toLowerCase(); + } + + public int getId() { + return this.id; + } + + public static RequestEvent getByName(String name) { + for (RequestEvent type : RequestEvent.values()) { + if (type.name().equalsIgnoreCase(name)) { + return type; + } + } + return null; + } + + public static RequestEvent getById(int id) { + for (RequestEvent type : RequestEvent.values()) { + if (type.getId() == id) { + return type; + } + } + return null; + } + }; + + public byte[] getInfoHash(); + public String getHexInfoHash(); + public byte[] getPeerId(); + public String getHexPeerId(); + public int getPort(); + public long getUploaded(); + public long getDownloaded(); + public long getLeft(); + public boolean getCompact(); + public boolean getNoPeerIds(); + public RequestEvent getEvent(); + + public String getIp(); + public int getNumWant(); + }; + + + /** + * Base interface for announce response messages. + * + *

+ * This interface must be implemented by all subtypes of announce response + * messages for the various tracker protocols. + *

+ * + * @author mpetazzoni + */ + public interface AnnounceResponseMessage { + + public int getInterval(); + public int getComplete(); + public int getIncomplete(); + public List getPeers(); + }; + + + /** + * Base interface for tracker error messages. + * + *

+ * This interface must be implemented by all subtypes of tracker error + * messages for the various tracker protocols. + *

+ * + * @author mpetazzoni + */ + public interface ErrorMessage { + + /** + * The various tracker error states. + * + *

+ * These errors are reported by the tracker to a client when expected + * parameters or conditions are not present while processing an + * announce request from a BitTorrent client. + *

+ */ + public enum FailureReason { + UNKNOWN_TORRENT("The requested torrent does not exist on this tracker"), + MISSING_HASH("Missing info hash"), + MISSING_PEER_ID("Missing peer ID"), + MISSING_PORT("Missing httpPort"), + INVALID_EVENT("Unexpected event for peer state"), + NOT_IMPLEMENTED("Feature not implemented"); + + private String message; + + FailureReason(String message) { + this.message = message; + } + + public String getMessage() { + return this.message; + } + }; + + public String getReason(); + }; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceRequestMessage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceRequestMessage.java new file mode 100644 index 0000000..26f82b2 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceRequestMessage.java @@ -0,0 +1,300 @@ +/** + * Copyright (C) 2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common.protocol.http; + +import org.apache.hadoop.yarn.server.broadcast.bcodec.BDecoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEncoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.InvalidBEncodingException; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceRequestMessage; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + + +/** + * The announce request message for the HTTP tracker protocol. + * + *

+ * This class represents the announce request message in the HTTP tracker + * protocol. It doesn't add any specific fields compared to the generic + * announce request message, but it provides the means to parse such + * messages and craft them. + *

+ * + * @author mpetazzoni + */ +public class HTTPAnnounceRequestMessage extends HTTPTrackerMessage + implements AnnounceRequestMessage { + + private final byte[] infoHash; + private final Peer peer; + private final long uploaded; + private final long downloaded; + private final long left; + private final boolean compact; + private final boolean noPeerId; + private final RequestEvent event; + private final int numWant; + + private HTTPAnnounceRequestMessage(ByteBuffer data, + byte[] infoHash, Peer peer, long uploaded, long downloaded, + long left, boolean compact, boolean noPeerId, RequestEvent event, + int numWant) { + super(Type.ANNOUNCE_REQUEST, data); + this.infoHash = infoHash; + this.peer = peer; + this.downloaded = downloaded; + this.uploaded = uploaded; + this.left = left; + this.compact = compact; + this.noPeerId = noPeerId; + this.event = event; + this.numWant = numWant; + } + + @Override + public byte[] getInfoHash() { + return this.infoHash; + } + + @Override + public String getHexInfoHash() { + return Torrent.byteArrayToHexString(this.infoHash); + } + + @Override + public byte[] getPeerId() { + return this.peer.getPeerId().array(); + } + + @Override + public String getHexPeerId() { + return this.peer.getHexPeerId(); + } + + @Override + public int getPort() { + return this.peer.getPort(); + } + + @Override + public long getUploaded() { + return this.uploaded; + } + + @Override + public long getDownloaded() { + return this.downloaded; + } + + @Override + public long getLeft() { + return this.left; + } + + @Override + public boolean getCompact() { + return this.compact; + } + + @Override + public boolean getNoPeerIds() { + return this.noPeerId; + } + + @Override + public RequestEvent getEvent() { + return this.event; + } + + @Override + public String getIp() { + return this.peer.getIp(); + } + + @Override + public int getNumWant() { + return this.numWant; + } + + /** + * Build the announce request URL for the given tracker announce URL. + * + * @param trackerAnnounceURL The tracker's announce URL. + * @return The URL object representing the announce request URL. + */ + public URL buildAnnounceURL(URL trackerAnnounceURL) + throws UnsupportedEncodingException, MalformedURLException { + String base = trackerAnnounceURL.toString(); + StringBuilder url = new StringBuilder(base); + url.append(base.contains("?") ? "&" : "?") + .append("info_hash=") + .append(URLEncoder.encode( + new String(this.getInfoHash(), Torrent.BYTE_ENCODING), + Torrent.BYTE_ENCODING)) + .append("&peer_id=") + .append(URLEncoder.encode( + new String(this.getPeerId(), Torrent.BYTE_ENCODING), + Torrent.BYTE_ENCODING)) + .append("&httpPort=").append(this.getPort()) + .append("&uploaded=").append(this.getUploaded()) + .append("&downloaded=").append(this.getDownloaded()) + .append("&left=").append(this.getLeft()) + .append("&compact=").append(this.getCompact() ? 1 : 0) + .append("&no_peer_id=").append(this.getNoPeerIds() ? 1 : 0); + + if (this.getEvent() != null && + !RequestEvent.NONE.equals(this.getEvent())) { + url.append("&event=").append(this.getEvent().getEventName()); + } + + if (this.getIp() != null) { + url.append("&ip=").append(this.getIp()); + } + + return new URL(url.toString()); + } + + public static HTTPAnnounceRequestMessage parse(ByteBuffer data) + throws IOException, MessageValidationException { + BEValue decoded = BDecoder.bdecode(data); + if (decoded == null) { + throw new MessageValidationException( + "Could not decode tracker message (not B-encoded?)!"); + } + + Map params = decoded.getMap(); + + if (!params.containsKey("info_hash")) { + throw new MessageValidationException( + ErrorMessage.FailureReason.MISSING_HASH.getMessage()); + } + + if (!params.containsKey("peer_id")) { + throw new MessageValidationException( + ErrorMessage.FailureReason.MISSING_PEER_ID.getMessage()); + } + + if (!params.containsKey("httpPort")) { + throw new MessageValidationException( + ErrorMessage.FailureReason.MISSING_PORT.getMessage()); + } + + try { + byte[] infoHash = params.get("info_hash").getBytes(); + byte[] peerId = params.get("peer_id").getBytes(); + int port = params.get("httpPort").getInt(); + + // Default 'uploaded' and 'downloaded' to 0 if the client does + // not provide it (although it should, according to the spec). + long uploaded = 0; + if (params.containsKey("uploaded")) { + uploaded = params.get("uploaded").getLong(); + } + + long downloaded = 0; + if (params.containsKey("downloaded")) { + downloaded = params.get("downloaded").getLong(); + } + + // Default 'left' to -1 to avoid peers entering the COMPLETED + // state when they don't provide the 'left' parameter. + long left = -1; + if (params.containsKey("left")) { + left = params.get("left").getLong(); + } + + boolean compact = false; + if (params.containsKey("compact")) { + compact = params.get("compact").getInt() == 1; + } + + boolean noPeerId = false; + if (params.containsKey("no_peer_id")) { + noPeerId = params.get("no_peer_id").getInt() == 1; + } + + int numWant = AnnounceRequestMessage.DEFAULT_NUM_WANT; + if (params.containsKey("numwant")) { + numWant = params.get("numwant").getInt(); + } + + String ip = null; + if (params.containsKey("ip")) { + ip = params.get("ip").getString(Torrent.BYTE_ENCODING); + } + + RequestEvent event = RequestEvent.NONE; + if (params.containsKey("event")) { + event = RequestEvent.getByName(params.get("event") + .getString(Torrent.BYTE_ENCODING)); + } + + return new HTTPAnnounceRequestMessage(data, infoHash, + new Peer(ip, port, ByteBuffer.wrap(peerId)), + uploaded, downloaded, left, compact, noPeerId, + event, numWant); + } catch (InvalidBEncodingException ibee) { + throw new MessageValidationException( + "Invalid HTTP tracker request!", ibee); + } + } + + public static HTTPAnnounceRequestMessage craft(byte[] infoHash, + byte[] peerId, int port, long uploaded, long downloaded, long left, + boolean compact, boolean noPeerId, RequestEvent event, + String ip, int numWant) + throws IOException, MessageValidationException, + UnsupportedEncodingException { + Map params = new HashMap(); + params.put("info_hash", new BEValue(infoHash)); + params.put("peer_id", new BEValue(peerId)); + params.put("httpPort", new BEValue(port)); + params.put("uploaded", new BEValue(uploaded)); + params.put("downloaded", new BEValue(downloaded)); + params.put("left", new BEValue(left)); + params.put("compact", new BEValue(compact ? 1 : 0)); + params.put("no_peer_id", new BEValue(noPeerId ? 1 : 0)); + + if (event != null) { + params.put("event", + new BEValue(event.getEventName(), Torrent.BYTE_ENCODING)); + } + + if (ip != null) { + params.put("ip", + new BEValue(ip, Torrent.BYTE_ENCODING)); + } + + if (numWant != AnnounceRequestMessage.DEFAULT_NUM_WANT) { + params.put("numwant", new BEValue(numWant)); + } + + return new HTTPAnnounceRequestMessage( + BEncoder.bencode(params), + infoHash, new Peer(ip, port, ByteBuffer.wrap(peerId)), + uploaded, downloaded, left, compact, noPeerId, event, numWant); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceResponseMessage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceResponseMessage.java new file mode 100644 index 0000000..b4cc918 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceResponseMessage.java @@ -0,0 +1,210 @@ +/** + * Copyright (C) 2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common.protocol.http; + +import org.apache.hadoop.yarn.server.broadcast.bcodec.BDecoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEncoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.InvalidBEncodingException; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceResponseMessage; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + + +/** + * The announce response message from an HTTP tracker. + * + * @author mpetazzoni + */ +public class HTTPAnnounceResponseMessage extends HTTPTrackerMessage + implements AnnounceResponseMessage { + + private final int interval; + private final int complete; + private final int incomplete; + private final List peers; + + private HTTPAnnounceResponseMessage(ByteBuffer data, + int interval, int complete, int incomplete, List peers) { + super(Type.ANNOUNCE_RESPONSE, data); + this.interval = interval; + this.complete = complete; + this.incomplete = incomplete; + this.peers = peers; + } + + @Override + public int getInterval() { + return this.interval; + } + + @Override + public int getComplete() { + return this.complete; + } + + @Override + public int getIncomplete() { + return this.incomplete; + } + + @Override + public List getPeers() { + return this.peers; + } + + public static HTTPAnnounceResponseMessage parse(ByteBuffer data) + throws IOException, MessageValidationException { + BEValue decoded = BDecoder.bdecode(data); + if (decoded == null) { + throw new MessageValidationException( + "Could not decode tracker message (not B-encoded?)!"); + } + + Map params = decoded.getMap(); + + if (params.get("interval") == null) { + throw new MessageValidationException( + "Tracker message missing mandatory field 'interval'!"); + } + + try { + List peers; + + try { + // First attempt to decode a compact response, since we asked + // for it. + peers = toPeerList(params.get("peers").getBytes()); + } catch (InvalidBEncodingException ibee) { + // Fall back to peer list, non-compact response, in case the + // tracker did not support compact responses. + peers = toPeerList(params.get("peers").getList()); + } + + return new HTTPAnnounceResponseMessage(data, + params.get("interval").getInt(), + params.get("complete") != null ? params.get("complete").getInt() : 0, + params.get("incomplete") != null ? params.get("incomplete").getInt() : 0, + peers); + } catch (InvalidBEncodingException ibee) { + throw new MessageValidationException("Invalid response " + + "from tracker!", ibee); + } catch (UnknownHostException uhe) { + throw new MessageValidationException("Invalid peer " + + "in tracker response!", uhe); + } + } + + /** + * Build a peer list as a list of {@link Peer}s from the + * announce response's peer list (in non-compact mode). + * + * @param peers The list of {@link BEValue}s dictionaries describing the + * peers from the announce response. + * @return A {@link List} of {@link Peer}s representing the + * peers' addresses. Peer IDs are lost, but they are not crucial. + */ + private static List toPeerList(List peers) + throws InvalidBEncodingException { + List result = new LinkedList(); + + for (BEValue peer : peers) { + Map peerInfo = peer.getMap(); + result.add(new Peer( + peerInfo.get("ip").getString(Torrent.BYTE_ENCODING), + peerInfo.get("httpPort").getInt())); + } + + return result; + } + + /** + * Build a peer list as a list of {@link Peer}s from the + * announce response's binary compact peer list. + * + * @param data The bytes representing the compact peer list from the + * announce response. + * @return A {@link List} of {@link Peer}s representing the + * peers' addresses. Peer IDs are lost, but they are not crucial. + */ + private static List toPeerList(byte[] data) + throws InvalidBEncodingException, UnknownHostException { + if (data.length % 6 != 0) { + throw new InvalidBEncodingException("Invalid peers " + + "binary information string!"); + } + + List result = new LinkedList(); + ByteBuffer peers = ByteBuffer.wrap(data); + + for (int i=0; i < data.length / 6 ; i++) { + byte[] ipBytes = new byte[4]; + peers.get(ipBytes); + InetAddress ip = InetAddress.getByAddress(ipBytes); + int port = + (0xFF & (int)peers.get()) << 8 | + (0xFF & (int)peers.get()); + result.add(new Peer(new InetSocketAddress(ip, port))); + } + + return result; + } + + /** + * Craft a compact announce response message. + * + * @param interval + * @param minInterval + * @param trackerId + * @param complete + * @param incomplete + * @param peers + */ + public static HTTPAnnounceResponseMessage craft(int interval, + int minInterval, String trackerId, int complete, int incomplete, + List peers) throws IOException, UnsupportedEncodingException { + Map response = new HashMap(); + response.put("interval", new BEValue(interval)); + response.put("complete", new BEValue(complete)); + response.put("incomplete", new BEValue(incomplete)); + + ByteBuffer data = ByteBuffer.allocate(peers.size() * 6); + for (Peer peer : peers) { + byte[] ip = peer.getRawIp(); + if (ip == null || ip.length != 4) { + continue; + } + data.put(ip); + data.putShort((short)peer.getPort()); + } + response.put("peers", new BEValue(data.array())); + + return new HTTPAnnounceResponseMessage( + BEncoder.bencode(response), + interval, complete, incomplete, peers); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPTrackerErrorMessage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPTrackerErrorMessage.java new file mode 100644 index 0000000..81015df --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPTrackerErrorMessage.java @@ -0,0 +1,87 @@ +/** + * Copyright (C) 2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common.protocol.http; + +import org.apache.hadoop.yarn.server.broadcast.bcodec.BDecoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEncoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.InvalidBEncodingException; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.ErrorMessage; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + + +/** + * An error message from an HTTP tracker. + * + * @author mpetazzoni + */ +public class HTTPTrackerErrorMessage extends HTTPTrackerMessage + implements ErrorMessage { + + private final String reason; + + private HTTPTrackerErrorMessage(ByteBuffer data, String reason) { + super(Type.ERROR, data); + this.reason = reason; + } + + @Override + public String getReason() { + return this.reason; + } + + public static HTTPTrackerErrorMessage parse(ByteBuffer data) + throws IOException, MessageValidationException { + BEValue decoded = BDecoder.bdecode(data); + if (decoded == null) { + throw new MessageValidationException( + "Could not decode tracker message (not B-encoded?)!"); + } + + Map params = decoded.getMap(); + + try { + return new HTTPTrackerErrorMessage( + data, + params.get("failure reason") + .getString(Torrent.BYTE_ENCODING)); + } catch (InvalidBEncodingException ibee) { + throw new MessageValidationException("Invalid tracker error " + + "message!", ibee); + } + } + + public static HTTPTrackerErrorMessage craft( + ErrorMessage.FailureReason reason) throws IOException, + MessageValidationException { + return HTTPTrackerErrorMessage.craft(reason.getMessage()); + } + + public static HTTPTrackerErrorMessage craft(String reason) + throws IOException, MessageValidationException { + Map params = new HashMap(); + params.put("failure reason", + new BEValue(reason, Torrent.BYTE_ENCODING)); + return new HTTPTrackerErrorMessage( + BEncoder.bencode(params), + reason); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPTrackerMessage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPTrackerMessage.java new file mode 100644 index 0000000..32f80a5 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPTrackerMessage.java @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common.protocol.http; + +import org.apache.hadoop.yarn.server.broadcast.bcodec.BDecoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; + + +/** + * Base class for HTTP tracker messages. + * + * @author mpetazzoni + */ +public abstract class HTTPTrackerMessage extends TrackerMessage { + + protected HTTPTrackerMessage(Type type, ByteBuffer data) { + super(type, data); + } + + public static HTTPTrackerMessage parse(ByteBuffer data) + throws IOException, MessageValidationException { + BEValue decoded = BDecoder.bdecode(data); + if (decoded == null) { + throw new MessageValidationException( + "Could not decode tracker message (not B-encoded?)!"); + } + + Map params = decoded.getMap(); + + if (params.containsKey("info_hash")) { + return HTTPAnnounceRequestMessage.parse(data); + } else if (params.containsKey("peers")) { + return HTTPAnnounceResponseMessage.parse(data); + } else if (params.containsKey("failure reason")) { + return HTTPTrackerErrorMessage.parse(data); + } + + throw new MessageValidationException("Unknown HTTP tracker message!"); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BTDownload.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BTDownload.java new file mode 100644 index 0000000..19bca34 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BTDownload.java @@ -0,0 +1,218 @@ +package org.apache.hadoop.yarn.server.broadcast.service; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.util.ConverterUtils; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.LoadingCache; + +/** + * Download a single URL to the local disk. + * + */ +@LimitedPrivate({"YARN", "MapReduce"}) +public class BTDownload implements Callable { + + private static final Log LOG = LogFactory.getLog(BTDownload.class); + + private FileContext files; + private LocalResource resource; + /** The local FS dir path under which this resource is to be localized to */ + private Path destDirPath; + private String appId; + + static final FsPermission PUBLIC_FILE_PERMS = new FsPermission((short) 0555); + static final FsPermission PRIVATE_FILE_PERMS = new FsPermission( + (short) 0500); + static final FsPermission PUBLIC_DIR_PERMS = new FsPermission((short) 0755); + static final FsPermission PRIVATE_DIR_PERMS = new FsPermission((short) 0700); + + + public BTDownload(FileContext files, UserGroupInformation ugi, Configuration conf, + Path destDirPath, LocalResource resource, String appId) { + this.destDirPath = destDirPath; + this.files = files; + this.resource = resource; + this.appId = appId; + } + + /** + * Returns a boolean to denote whether a cache file is visible to all (public) + * or not + * + * @return true if the path in the current path is visible to all, false + * otherwise + */ + @Private + public static boolean isPublic(FileSystem fs, Path current, FileStatus sStat, + LoadingCache> statCache) throws IOException { + current = fs.makeQualified(current); + //the leaf level file should be readable by others + if (!checkPublicPermsForAll(fs, sStat, FsAction.READ_EXECUTE, FsAction.READ)) { + return false; + } + + if (Shell.WINDOWS && fs instanceof LocalFileSystem) { + // Relax the requirement for public cache on LFS on Windows since default + // permissions are "700" all the way up to the drive letter. In this + // model, the only requirement for a user is to give EVERYONE group + // permission on the file and the file will be considered public. + // This code path is only hit when fs.default.name is file:/// (mainly + // in tests). + return true; + } + return ancestorsHaveExecutePermissions(fs, current.getParent(), statCache); + } + + private static boolean checkPublicPermsForAll(FileSystem fs, + FileStatus status, FsAction dir, FsAction file) + throws IOException { + FsPermission perms = status.getPermission(); + FsAction otherAction = perms.getOtherAction(); + if (status.isDirectory()) { + if (!otherAction.implies(dir)) { + return false; + } + + for (FileStatus child : fs.listStatus(status.getPath())) { + if(!checkPublicPermsForAll(fs, child, dir, file)) { + return false; + } + } + return true; + } + return (otherAction.implies(file)); + } + + /** + * Returns true if all ancestors of the specified path have the 'execute' + * permission set for all users (i.e. that other users can traverse + * the directory hierarchy to the given path) + */ + @VisibleForTesting + static boolean ancestorsHaveExecutePermissions(FileSystem fs, + Path path, LoadingCache> statCache) + throws IOException { + Path current = path; + while (current != null) { + //the subdirs in the path should have execute permissions for others + if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE, statCache)) { + return false; + } + current = current.getParent(); + } + return true; + } + + /** + * Checks for a given path whether the Other permissions on it + * imply the permission in the passed FsAction + * @param fs + * @param path + * @param action + * @return true if the path in the uri is visible to all, false otherwise + * @throws IOException + */ + private static boolean checkPermissionOfOther(FileSystem fs, Path path, + FsAction action, LoadingCache> statCache) + throws IOException { + FileStatus status = getFileStatus(fs, path, statCache); + FsPermission perms = status.getPermission(); + FsAction otherAction = perms.getOtherAction(); + return otherAction.implies(action); + } + + /** + * Obtains the file status, first by checking the stat cache if it is + * available, and then by getting it explicitly from the filesystem. If we got + * the file status from the filesystem, it is added to the stat cache. + * + * The stat cache is expected to be managed by callers who provided it to + * FSDownload. + */ + private static FileStatus getFileStatus(final FileSystem fs, final Path path, + LoadingCache> statCache) throws IOException { + // if the stat cache does not exist, simply query the filesystem + if (statCache == null) { + return fs.getFileStatus(path); + } + + try { + // get or load it from the cache + return statCache.get(path).get(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + // the underlying exception should normally be IOException + if (cause instanceof IOException) { + throw (IOException)cause; + } else { + throw new IOException(cause); + } + } catch (InterruptedException e) { // should not happen + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + + @Override + public Path call() throws Exception { + // .../torrentDataBase64String/filename + // write torrent data into a file + Path sCopy; + try { + sCopy = ConverterUtils.getPathFromYarnURL(resource.getResource()); + } catch (URISyntaxException e) { + throw new IOException("Invalid resource", e); + } + + String torrentStr = (sCopy.getParent()+"").substring(22); + byte[] torrentData = Base64.decodeBase64(torrentStr); + + String torrentPath = destDirPath + "/" + sCopy.getName() + ".torrent"; + + + + FileOutputStream output = new FileOutputStream(new File(torrentPath)); + IOUtils.write(torrentData, output); + IOUtils.closeQuietly(output); + + { + File torrentFile = new File(torrentPath); + byte[] data = null; + try { + data = IOUtils.toByteArray(new FileInputStream(torrentFile)); + } catch (Exception e) { + e.printStackTrace(); + } + } + // issue http request to aux service + BroadcastClient.btDownload(torrentPath, destDirPath+"", appId); + return files.makeQualified(new Path(destDirPath, sCopy.getName())); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BroadcastClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BroadcastClient.java new file mode 100644 index 0000000..2c4d907 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BroadcastClient.java @@ -0,0 +1,125 @@ +package org.apache.hadoop.yarn.server.broadcast.service; + +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.http.HttpResponse; +import org.apache.http.HttpVersion; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.params.CoreProtocolPNames; +import org.jboss.netty.handler.codec.http.HttpRequest; + +import java.io.*; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + + +public class BroadcastClient { + private static Log LOG = LogFactory.getLog(BroadcastClient.class); + + /** + * make torrent (in the same dir) and launch a ttorrent client + * @param filePath full path of data file + * @param appId application id + */ + public static String beginBroadcast(String filePath, String appId, int blocksize) { + String result = null; + try { + // make torrent + result = makeTorrent(filePath, blocksize); + File torrentFile = new File(filePath+".torrent"); + + // begin init seeding + btDownload(torrentFile.getCanonicalPath(), torrentFile.getParentFile().getCanonicalPath(), appId); + } catch (Exception e) { + e.printStackTrace(); + } + return result; + } + + public static String getTorrentStr(String filePath) { + File file = new File(filePath); + File torrentFile = new File(filePath + ".torrent"); + byte[] data = null; + try { + data = IOUtils.toByteArray(new FileInputStream(torrentFile)); + } catch (Exception e) { + e.printStackTrace(); + } + String str = Base64.encodeBase64URLSafeString(data); + StringBuilder sb = new StringBuilder(); + sb.append('/'); + sb.append(str); + sb.append('/'); + sb.append(file.getName()); + + return sb.toString(); + } + + // FIXME if download failed somehow, we should notify caller the failure + public static void btDownload(String torrentPath, String dirPath, String appId) { + try { + String url = "http://localhost:9696/btdownload?torrentPath="+torrentPath+"&dir="+dirPath+"&appId="+appId; + + // request btdownload to launch client and get clientid + HttpClient httpclient = new DefaultHttpClient(); + httpclient.getParams().setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_0); + String clientId = getResponseString(httpclient.execute(new HttpGet(url))); + + // keep querying client status and waiting until download finished + url = "http://localhost:9696/isfinished?clientid="+clientId; + while (true) { + String status = getResponseString(httpclient.execute(new HttpGet(url))); + if (status.equals(Boolean.TRUE.toString())) { + break; + } else if (status.equals(Boolean.FALSE.toString())) { + Thread.sleep(3000); + } else { + // FIXME something strange happens, how to handle this + } + } + + httpclient.getConnectionManager().shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // FIXME handle exception + private static String getResponseString(HttpResponse response) { + // parse clientid + StringWriter writer = new StringWriter(); + try { + IOUtils.copy(response.getEntity().getContent(), writer); + } catch (IOException e) { + e.printStackTrace(); + } + + return writer.toString(); + } + + + private static String makeTorrent(String filepath, int blocksize) throws Exception { + String tpath = filepath+".torrent"; + OutputStream fos = new FileOutputStream(tpath); + File source = new File(filepath); + /* FIXME this should be remove in future because we can infer tracker by infohash */ + List> announceList = new ArrayList>(); + List annouceURIs = new ArrayList(); + annouceURIs.add(new URI("http://localhost:6969/announce")); + announceList.add(annouceURIs); + + Torrent torrent = Torrent.create(source, blocksize, announceList, "" /* createdBy */); + torrent.save(fos); + fos.close(); + + /* FIXME this call should be removed because this function is only used here */ + return getTorrentStr(filepath); + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BroadcastService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BroadcastService.java new file mode 100644 index 0000000..920d0f8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/BroadcastService.java @@ -0,0 +1,628 @@ +package org.apache.hadoop.yarn.server.broadcast.service; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; +import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; +import org.apache.hadoop.yarn.server.api.AuxiliaryService; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEncoder; +import org.apache.hadoop.yarn.server.broadcast.client.Client; +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceRequestMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.ErrorMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.ErrorMessage.FailureReason; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.MessageValidationException; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPAnnounceRequestMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPAnnounceResponseMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPTrackerErrorMessage; +import org.apache.hadoop.yarn.server.broadcast.tracker.TrackedPeer; +import org.apache.hadoop.yarn.server.broadcast.tracker.TrackedTorrent; +import org.jboss.netty.bootstrap.ServerBootstrap; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.ChannelBuffers; +import org.jboss.netty.channel.*; +import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; +import org.jboss.netty.handler.codec.http.*; +import org.jboss.netty.handler.codec.http.multipart.*; +import org.jboss.netty.handler.codec.http.multipart.HttpPostRequestDecoder.EndOfDataDecoderException; +import org.jboss.netty.handler.codec.http.multipart.HttpPostRequestDecoder.ErrorDataDecoderException; +import org.jboss.netty.handler.codec.http.multipart.HttpPostRequestDecoder.NotEnoughDataDecoderException; +import org.jboss.netty.handler.codec.http.multipart.InterfaceHttpData.HttpDataType; +import org.jboss.netty.util.CharsetUtil; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.*; +import static org.jboss.netty.handler.codec.http.HttpResponseStatus.*; +import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; + +/** + * FIXME if two broadcast has same file and get same torrent, we may have problem + * one solution can be add broadcast-specific info into torrent so that we have different hashes + */ +public class BroadcastService extends AuxiliaryService { + public final int trackerPort = 6969; + public final int servicePort = 9696; + private Log LOG = LogFactory.getLog(BroadcastService.class); + private Topology topology = new Topology(); + + // service related data + + /* client-id: client, used for client status query, used by btdownload */ + private ConcurrentHashMap clientidClientMap = new ConcurrentHashMap<>(); + /* appId : clients, used for cleanup purpose, used by btdownload */ + private ConcurrentHashMap> appidClientMap = new ConcurrentHashMap<>(); + private String localDir; + + // tracker related data + /** + * The list of announce request URL fields that need to be interpreted as + * numeric and thus converted as such in the request message parsing. + */ + private final String[] NUMERIC_REQUEST_FIELDS = + new String[] { + "httpPort", "uploaded", "downloaded", "left", + "compact", "no_peer_id", "numwant" + }; + private final String version = "BitTorrent Tracker (ttorrent)"; + private PeerCollectorThread collector; + private final ConcurrentMap torrents = new ConcurrentHashMap<>(); + // torrent hash : client, track whether a client has been launched by this tracker for a specified torrent + // to avoid multiple client for the same torrent + // this map cannot be inside class Handler because the handler is created for every message + private final ConcurrentHashMap infohashClientMap = new ConcurrentHashMap<>(); + + protected BroadcastService() { + super("broadcast_service"); + } + + @Override + protected void serviceInit(Configuration conf) { + localDir = new YarnConfiguration().get("yarn.nodemanager.local-dirs"); + // FIXME now always use first local dir, but should use different dir for different broadcast for fault tolerance + if (localDir.indexOf(',') != -1) + localDir = localDir.substring(0, localDir.indexOf(',')); + + // setup tracker + try { + ThreadFactory bossFactory = new ThreadFactoryBuilder().build(); + ThreadFactory workerFactory = new ThreadFactoryBuilder().build(); + ChannelFactory selector = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory)); + ServerBootstrap bootstrap = new ServerBootstrap(selector); + bootstrap.setOption("reuseAddress", true); + HttpTrackerPipelineFactory pipelineFact = new HttpTrackerPipelineFactory(); + bootstrap.setPipelineFactory(pipelineFact); + bootstrap.bind(new InetSocketAddress(trackerPort)); + } catch (Exception e) { + e.printStackTrace(); + } + + // setup service + try { + ThreadFactory bossFactory = new ThreadFactoryBuilder().build(); + ThreadFactory workerFactory = new ThreadFactoryBuilder().build(); + ChannelFactory selector = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory)); + ServerBootstrap bootstrap = new ServerBootstrap(selector); + bootstrap.setOption("reuseAddress", true); + HttpServicePipelineFactory pipelineFact = new HttpServicePipelineFactory(); + bootstrap.setPipelineFactory(pipelineFact); + bootstrap.bind(new InetSocketAddress(servicePort)); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + protected void serviceStart() throws Exception { + this.collector = new PeerCollectorThread(); + this.collector.setName("peer-collector"); + this.collector.start(); + } + + @Override + protected void serviceStop() throws Exception { + } + + @Override + public void initializeApplication( + ApplicationInitializationContext initAppContext) { + } + + @Override + /** + * clean up application related data, including clients and tracker data + * @param stopAppContext context for the application termination + */ + public void stopApplication(ApplicationTerminationContext stopAppContext) { + String stopAppId = stopAppContext.getApplicationId().toString(); + for (Client c : appidClientMap.get(stopAppId)) { + c.stop(); + clientidClientMap.remove(c.getPeerSpec().getHexPeerId()); + // FIXME not sure whether there is race between this and peer collect + if (torrents.containsKey(c.getTorrent().getHexInfoHash())) + torrents.remove(c.getTorrent().getHexInfoHash()); + } + appidClientMap.remove(stopAppId); + } + + @Override + public ByteBuffer getMetaData() { + return null; + } + + class HttpServicePipelineFactory implements ChannelPipelineFactory { + @Override + public ChannelPipeline getPipeline() throws Exception { + ChannelPipeline pipeline = Channels.pipeline(); + + // upstream handler + pipeline.addLast("decoder", new HttpRequestDecoder()); + // downstream handler + pipeline.addLast("encoder", new HttpResponseEncoder()); + pipeline.addLast("handler", new ServiceHandler()); + + return pipeline; + } + } + + class HttpTrackerPipelineFactory implements ChannelPipelineFactory { + @Override + public ChannelPipeline getPipeline() throws Exception { + ChannelPipeline pipeline = Channels.pipeline(); + + // upstream handler + pipeline.addLast("decoder", new HttpRequestDecoder()); + pipeline.addLast("aggregator", new HttpChunkAggregator(1024*1024*1024)); + pipeline.addLast("ddecoder", new HttpContentDecompressor()); + + //pipeline.addLast("logger", new LoggingHandler()); + // by default http client use chunked transfer encoding, so we need aggregator + // FIXME no idea about how to set max content length, currently 1 MB + //pipeline.addLast("aggregator", new HttpChunkAggregator(1 * 1024 * 1024)); + // downstream handler + pipeline.addLast("encoder", new HttpResponseEncoder()); + pipeline.addLast("handler", new TrackerHandler()); + + + + return pipeline; + } + } + + static { + DiskFileUpload.deleteOnExitTemporaryFile = true; // should delete file + // on exit (in normal + // exit) + DiskFileUpload.baseDirectory = null; // system temp directory + DiskAttribute.deleteOnExitTemporaryFile = true; // should delete file on + // exit (in normal exit) + DiskAttribute.baseDirectory = null; // system temp directory + } + + class TrackerHandler extends SimpleChannelUpstreamHandler { + private Log LOG = LogFactory.getLog(TrackerHandler.class); + + public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt) throws Exception { + HttpRequest request = (HttpRequest) evt.getMessage(); + String path = new QueryStringDecoder(request.getUri()).getPath(); + LOG.info(" YZY message received " + path); + + if (path.equals("/announce")) { + handleAnnounce(request, evt); + } else if (path.equals("/preannounce") && request.getMethod().equals(HttpMethod.POST)) { + handlePreAnnounce(request, evt); + + } else { + evt.getChannel().write(new DefaultHttpResponse(HTTP_1_1, NOT_FOUND)).addListener(ChannelFutureListener.CLOSE); + } + + LOG.info(" YZY message handled" + path); + } + + public void handlePreAnnounce(HttpRequest request, MessageEvent evt) throws Exception { + try { + byte[] data = new byte[(int) HttpHeaders.getContentLength(request)]; + request.getContent().readBytes(data); + + TrackedTorrent torrent = new TrackedTorrent(data); + LOG.info("add torrent "+torrent.getHexInfoHash()); + torrents.putIfAbsent(torrent.getHexInfoHash(), torrent); + + //System.out.println(request.getContent().); + evt.getChannel().write(new DefaultHttpResponse(HTTP_1_1, OK)).addListener(ChannelFutureListener.CLOSE); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Process the announce request. + * + *

+ * This method attemps to read and parse the incoming announce request into + * an announce request message, then creates the appropriate announce + * response message and sends it back to the client. + *

+ * @param request The incoming announce request. + * @param evt + */ + private void handleAnnounce(HttpRequest request, MessageEvent evt) throws IOException { + // Prepare the response headers. + Channel ch = evt.getChannel(); + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.addHeader(CONTENT_TYPE,"text/plain"); + response.addHeader(SERVER, version); + response.addHeader(DATE, System.currentTimeMillis()); + + /** + * Parse the query parameters into an announce request message. + * + * We need to rely on our own query parsing function because + * SimpleHTTP's Query map will contain UTF-8 decoded parameters, which + * doesn't work well for the byte-encoded strings we expect. + */ + HTTPAnnounceRequestMessage announceRequest = null; + try { + announceRequest = this.parseQuery(request, evt); + } catch (Exception e) { + e.printStackTrace(); + } + LOG.info("get announce request from " + ((InetSocketAddress)evt.getRemoteAddress()).getPort() + ",type: " + announceRequest.getEvent().getEventName()); + + // get torrent or notify client torrent is unknown by this tracker + TrackedTorrent torrent = torrents.get(announceRequest.getHexInfoHash()); + if (torrent == null) { + LOG.warn("Requested torrent hash was: " + announceRequest.getHexInfoHash()); + this.serveError(response, ch, BAD_REQUEST, ErrorMessage.FailureReason.UNKNOWN_TORRENT); + return; + } + + AnnounceRequestMessage.RequestEvent event = announceRequest.getEvent(); + String peerId = announceRequest.getHexPeerId(); + + // When no event is specified, it's a periodic update while the client + // is operating. If we don't have a peer for this announce, it means + // the tracker restarted while the client was running. Consider this + // announce request as a 'started' event. + if ((event == null || + AnnounceRequestMessage.RequestEvent.NONE.equals(event)) && + torrent.getPeer(peerId) == null) { + event = AnnounceRequestMessage.RequestEvent.STARTED; + } + + // If an event other than 'started' is specified and we also haven't + // seen the peer on this torrent before, something went wrong. A + // previous 'started' announce request should have been made by the + // client that would have had us register that peer on the torrent this + // request refers to. + if (event != null && torrent.getPeer(peerId) == null && + !AnnounceRequestMessage.RequestEvent.STARTED.equals(event)) { + this.serveError(response, ch, BAD_REQUEST, + ErrorMessage.FailureReason.INVALID_EVENT); + return; + } + + // if node is on the same rack but we didn't launch a client for this torrent, then launch a client + synchronized (infohashClientMap) { + LOG.info("map " + infohashClientMap); + if (topology.onSameRack(InetAddress.getLocalHost().getHostAddress(), ((InetSocketAddress) evt.getRemoteAddress()).getHostString()) && !infohashClientMap.containsKey(torrent.getHexInfoHash())) { + LOG.info("tracker launch a client for torrent " + torrent.getHexInfoHash()); + try { + Client c = new Client(InetAddress.getLocalHost(), torrent.getEncoded(), localDir); + infohashClientMap.put(torrent.getHexInfoHash(), c); + c.share(-1); + // FIXME track client for cleanup, need appid in either torrent or announcement + } catch (Exception e) { + e.printStackTrace(); + } + } else { + LOG.info("tracker don't need to launch a client for torrent " + torrent.getHexInfoHash()); + } + } + + // Update the torrent according to the announce event + TrackedPeer peer = null; + try { + peer = torrent.update(event, + ByteBuffer.wrap(announceRequest.getPeerId()), + announceRequest.getHexPeerId(), + announceRequest.getIp(), + announceRequest.getPort(), + announceRequest.getUploaded(), + announceRequest.getDownloaded(), + announceRequest.getLeft()); + } catch (IllegalArgumentException iae) { + this.serveError(response, ch, BAD_REQUEST, + ErrorMessage.FailureReason.INVALID_EVENT); + return; + } + + // Craft and output the answer + HTTPAnnounceResponseMessage announceResponse = null; + try { + announceResponse = HTTPAnnounceResponseMessage.craft( + torrent.getAnnounceInterval(), + TrackedTorrent.MIN_ANNOUNCE_INTERVAL_SECONDS, + version, + torrent.seeders(), + torrent.leechers(), + torrent.getSomePeers(peer)); + response.setContent(ChannelBuffers.wrappedBuffer(announceResponse.getData())); + } catch (Exception e) { + this.serveError(response, ch, INTERNAL_SERVER_ERROR, + e.getMessage()); + } + + evt.getChannel().write(response).addListener(ChannelFutureListener.CLOSE); + LOG.info("response write finished"); + } + + /** + * Parse the query parameters using our defined BYTE_ENCODING. + * + *

+ * Because we're expecting byte-encoded strings as query parameters, we + * can't rely on SimpleHTTP's QueryParser which uses the wrong encoding for + * the job and returns us unparsable byte data. We thus have to implement + * our own little parsing method that uses BYTE_ENCODING to decode + * parameters from the URI. + *

+ * + *

+ * Note: array parameters are not supported. If a key is present + * multiple times in the URI, the latest value prevails. We don't really + * need to implement this functionality as this never happens in the + * Tracker HTTP protocol. + *

+ * + * @param request The request's full URI, including query parameters. + * @param evt + * @return The {@link AnnounceRequestMessage} representing the client's + * announce request. + */ + private HTTPAnnounceRequestMessage parseQuery(HttpRequest request, MessageEvent evt) + throws IOException, MessageValidationException { + Map params = new HashMap(); + + try { + String uri = request.getUri(); + for (String pair : uri.split("[?]")[1].split("&")) { + String[] keyval = pair.split("[=]", 2); + if (keyval.length == 1) { + this.recordParam(params, keyval[0], null); + } else { + this.recordParam(params, keyval[0], keyval[1]); + } + } + } catch (ArrayIndexOutOfBoundsException e) { + params.clear(); + } + + // Make sure we have the peer IP, fallbacking on the request's source + // address if the peer didn't provide it. + if (params.get("ip") == null) { + params.put("ip", new BEValue( + ((InetSocketAddress)(evt.getRemoteAddress())).getAddress().getHostAddress(), + Torrent.BYTE_ENCODING)); + } + + return HTTPAnnounceRequestMessage.parse(BEncoder.bencode(params)); + } + + private void recordParam(Map params, String key, + String value) { + try { + value = URLDecoder.decode(value, Torrent.BYTE_ENCODING); + + for (String f : NUMERIC_REQUEST_FIELDS) { + if (f.equals(key)) { + params.put(key, new BEValue(Long.valueOf(value))); + return; + } + } + + params.put(key, new BEValue(value, Torrent.BYTE_ENCODING)); + } catch (UnsupportedEncodingException uee) { + // Ignore, act like parameter was not there + return; + } + } + + /** + * Write a {@link HTTPTrackerErrorMessage} to the response with the given + * HTTP status code. + * @param response The HTTP response object. + * @param ch The response output stream to write to. + * @param status The HTTP status code to return. + * @param error The error reported by the tracker. + */ + private void serveError(HttpResponse response, Channel ch, + HttpResponseStatus status, HTTPTrackerErrorMessage error) throws IOException { + LOG.warn("Could not process announce request (" + error.getReason() + ") !"); + response = new DefaultHttpResponse(HTTP_1_1, status); + response.setHeader(CONTENT_LENGTH, error.getData().array().length); + response.setContent(ChannelBuffers.wrappedBuffer(error.getData())); + ch.write(response).addListener(ChannelFutureListener.CLOSE); + LOG.warn("Write serveError response !"); + } + + /** + * Write an error message to the response with the given HTTP status code. + * @param response The HTTP response object. + * @param body The response output stream to write to. + * @param status The HTTP status code to return. + * @param error The error message reported by the tracker. + */ + private void serveError(HttpResponse response, Channel body, + HttpResponseStatus status, String error) throws IOException { + try { + this.serveError(response, body, status, + HTTPTrackerErrorMessage.craft(error)); + } catch (MessageValidationException mve) { + LOG.warn("Could not craft tracker error message!", mve); + } + } + + /** + * Write a tracker failure reason code to the response with the given HTTP + * status code. + * @param response The HTTP response object. + * @param ch The response output stream to write to. + * @param status The HTTP status code to return. + * @param reason The failure reason reported by the tracker. + */ + private void serveError(HttpResponse response, Channel ch, + HttpResponseStatus status, FailureReason reason) throws IOException { + this.serveError(response, ch, status, reason.getMessage()); + } + } + + class ServiceHandler extends SimpleChannelUpstreamHandler { + private Log LOG = LogFactory.getLog(ServiceHandler.class); + public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt) throws Exception { + HttpRequest request = (HttpRequest) evt.getMessage(); + String path = new QueryStringDecoder(request.getUri()).getPath(); + LOG.info(" YZY message received " + path); + + if (path.equals("/btdownload")) { + handleBtdownload(request, evt); + } else if (path.equals("/isfinished")) { + handleIsfinished(request, evt); + } else if (path.equals("/cleanup")) { + handleCleanup(request, evt); + } else { + evt.getChannel().write(new DefaultHttpResponse(HTTP_1_1, NOT_FOUND)).addListener(ChannelFutureListener.CLOSE); + } + + LOG.info(" YZY message handled" + path); + } + + /** + * temporary tool to cleanup all clients launched by tracker + * will be removed in future + * @param request + * @param evt + * @throws Exception + */ + public void handleCleanup(HttpRequest request, MessageEvent evt) throws Exception { + for (Client c : infohashClientMap.values()) { + c.stop(true); + if (c.getTorrent().isFinished()) { + (new File(localDir+"/"+c.getTorrent().getName())).delete(); + } else { + (new File(localDir+"/"+c.getTorrent().getName()+".part")).delete(); + } + if (torrents.containsKey(c.getTorrent().getHexInfoHash())) + torrents.remove(c.getTorrent().getHexInfoHash()); + } + infohashClientMap.clear(); + evt.getChannel().write(new DefaultHttpResponse(HTTP_1_1, OK)).addListener(ChannelFutureListener.CLOSE); + } + + /** + * * handle isfinished request, query client status using client id + * @param request + * @param evt + */ + private void handleIsfinished(HttpRequest request, MessageEvent evt) { + Map> params = new QueryStringDecoder(request.getUri()).getParameters(); + String clientId = params.get("clientid").get(0); + + String status = clientidClientMap.get(clientId).getTorrent().isComplete() ? Boolean.TRUE.toString() : Boolean.FALSE.toString(); + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + HttpHeaders.setContentLength(response, status.length()); + response.setContent(ChannelBuffers.wrappedBuffer(status.getBytes())); + evt.getChannel().write(response).addListener(ChannelFutureListener.CLOSE); + } + + /** + * handle the btdownload request: launch the client and return client id + * @param request http request + * @param evt + */ + private void handleBtdownload(HttpRequest request, MessageEvent evt) { + Map> params = new QueryStringDecoder(request.getUri()).getParameters(); + String torrentPath = params.get("torrentPath").get(0), dirPath = params.get("dir").get(0), appId = params.get("appId").get(0); + + // FIXME THE TORRENT FILE MUST EXIST FOR THE WHOLE LIFECYCLE OF CLIENT + try { + File torrentFile = new File(torrentPath); + Client c = new Client(InetAddress.getLocalHost(), SharedTorrent.fromFile(torrentFile, (new File(dirPath)).getCanonicalFile())); + c.share(-1); + + appidClientMap.putIfAbsent(appId, new HashSet()); + appidClientMap.get(appId).add(c); + + String clientId = c.getPeerSpec().getHexPeerId(); + clientidClientMap.put(clientId, c); + + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + HttpHeaders.setContentLength(response, clientId.length()); + response.setContent(ChannelBuffers.wrappedBuffer(clientId.getBytes())); + evt.getChannel().write(response).addListener(ChannelFutureListener.CLOSE); + } catch (UnknownHostException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * The unfresh peer collector thread. + * + *

+ * Every PEER_COLLECTION_FREQUENCY_SECONDS, this thread will collect + * unfresh peers from all announced torrents. + *

+ */ + private class PeerCollectorThread extends Thread { + + private static final int PEER_COLLECTION_FREQUENCY_SECONDS = 15; + private boolean stop = false; + + @Override + public void run() { + LOG.info("Starting tracker peer collection for tracker at"); + + while (!stop) { + for (TrackedTorrent torrent : torrents.values()) { + torrent.collectUnfreshPeers(); + } + + try { + Thread.sleep(PeerCollectorThread + .PEER_COLLECTION_FREQUENCY_SECONDS * 1000); + } catch (InterruptedException ie) { + // Ignore + } + } + } + + public void setStop() { + this.stop = true; + } + } +} + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/Topology.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/Topology.java new file mode 100644 index 0000000..e99bd82 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/Topology.java @@ -0,0 +1,124 @@ +package org.apache.hadoop.yarn.server.broadcast.service; + + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.broadcast.client.announce.AnnounceException; + +import java.io.*; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; + +/** + * parse topology input and offer consistent hashing functionality + * FIXME if there is no topology input or topology input is not up to date, this won't work well + */ +public class Topology { + private static Log LOG = LogFactory.getLog(Topology.class); + /* hash to name dictionary */ + private Map hashDict = new HashMap<>(); + /* the following there collections are all about hashes */ + // sorted rack hash + private ArrayList racks = new ArrayList<>(); + // sorted node hash in each value list + private Map> rackNodeMap = new HashMap<>(); + private Map nodeRackMap = new HashMap<>(); + private String localAddr; + + // parse the topology input + public Topology(String topologyFilePath) { + try { + localAddr = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + // FIXME no idea how to handle this + } + + try (BufferedReader br = new BufferedReader(new FileReader(topologyFilePath))) { + String line = null; + while ((line = br.readLine()) != null) { + String[] tokens = line.split(" "); + String node = tokens[0], rack = tokens[1]; + String nodeHash = DigestUtils.sha1Hex(node), rackHash = DigestUtils.sha1Hex(rack); + hashDict.put(nodeHash, node); + hashDict.put(rackHash, rack); + if (!rackNodeMap.containsKey(rackHash)) { + rackNodeMap.put(rackHash, new ArrayList()); + racks.add(rackHash); + } + rackNodeMap.get(rackHash).add(nodeHash); + nodeRackMap.put(nodeHash, rackHash); + } + + Collections.sort(racks); + for (ArrayList nodes : rackNodeMap.values()) + Collections.sort(nodes); + } catch (IOException e) { + e.printStackTrace(); + } + + + } + + public Topology() { + this(new YarnConfiguration().get("yarn.nodemanager.broadcast_service.topology_file")); + } + + /* return the index of key or where it should be inserted into */ + private int binarySearch(ArrayList list, String key) { + int idx = Collections.binarySearch(list, key); + return ((idx < 0) ? (-idx-1) : idx)%list.size(); + } + + public String getGlobalMaster(String key) { + // choose rack + int rackIdx = binarySearch(racks, key); + // choose node + ArrayList nodes = rackNodeMap.get(racks.get(rackIdx)); + int nodeIdx = binarySearch(nodes, key); + return hashDict.get(nodes.get(nodeIdx)); + } + + public String getNext(String hostname) { + String node = DigestUtils.sha1Hex(hostname); + // try to find next node in current rack + String rack = nodeRackMap.get(node); + ArrayList nodes = rackNodeMap.get(rack); + int nodeIdx = binarySearch(nodes, node); + if (nodeIdx != nodes.size()-1) { + return hashDict.get(nodes.get(nodeIdx + 1)); + } + // find next node in next rack + int rackIdx = (binarySearch(racks, rack)+1) % racks.size(); + rack = racks.get(rackIdx); + nodes = rackNodeMap.get(rack); + return hashDict.get(nodes.get(0)); + } + + public String getRackMaster(String key) { + String node = DigestUtils.sha1Hex(localAddr); + String rack = nodeRackMap.get(node); + ArrayList nodes = rackNodeMap.get(rack); + int nodeIdx = binarySearch(nodes, key); + return hashDict.get(nodes.get(nodeIdx)); + } + + public String getNextRackMaster(String hostname) { + String node = DigestUtils.sha1Hex(hostname); + String rack = nodeRackMap.get(node); + ArrayList nodes = rackNodeMap.get(rack); + int nodeIdx = binarySearch(nodes, node); + return hashDict.get(nodes.get(nodeIdx == nodes.size()-1 ? 0 : nodeIdx+1)); + } + + /* host1 not equal host2 and they are on the same rack */ + public boolean onSameRack(String host1, String host2) { + return !host1.equals(host2) && nodeRackMap.get(DigestUtils.sha1Hex(host1)).equals(nodeRackMap.get(DigestUtils.sha1Hex(host2))); + } + + public String getLocalAddr() { + return localAddr; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/tracker/TrackedPeer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/tracker/TrackedPeer.java new file mode 100644 index 0000000..6f95e0e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/tracker/TrackedPeer.java @@ -0,0 +1,212 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.tracker; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * A BitTorrent tracker peer. + * + *

+ * Represents a peer exchanging on a given torrent. In this implementation, + * we don't really care about the status of the peers and how much they + * have downloaded / exchanged because we are not a torrent exchange and + * don't need to keep track of what peers are doing while they're + * downloading. We only care about when they start, and when they are done. + *

+ * + *

+ * We also never expire peers automatically. Unless peers send a STOPPED + * announce request, they remain as long as the torrent object they are a + * part of. + *

+ */ +public class TrackedPeer extends Peer { + + private static final Logger logger = + LoggerFactory.getLogger(TrackedPeer.class); + + private static final int FRESH_TIME_SECONDS = 30; + + private long uploaded; + private long downloaded; + private long left; + private Torrent torrent; + + /** + * Represents the state of a peer exchanging on this torrent. + * + *

+ * Peers can be in the STARTED state, meaning they have announced + * themselves to us and are eventually exchanging data with other peers. + * Note that a peer starting with a completed file will also be in the + * started state and will never notify as being in the completed state. + * This information can be inferred from the fact that the peer reports 0 + * bytes left to download. + *

+ * + *

+ * Peers enter the COMPLETED state when they announce they have entirely + * downloaded the file. As stated above, we may also elect them for this + * state if they report 0 bytes left to download. + *

+ * + *

+ * Peers enter the STOPPED state very briefly before being removed. We + * still pass them to the STOPPED state in case someone else kept a + * reference on them. + *

+ */ + public enum PeerState { + UNKNOWN, + STARTED, + COMPLETED, + STOPPED; + }; + + private PeerState state; + private Date lastAnnounce; + + /** + * Instantiate a new tracked peer for the given torrent. + * + * @param torrent The torrent this peer exchanges on. + * @param ip The peer's IP address. + * @param port The peer's port. + * @param peerId The byte-encoded peer ID. + */ + public TrackedPeer(Torrent torrent, String ip, int port, + ByteBuffer peerId) { + super(ip, port, peerId); + this.torrent = torrent; + + // Instantiated peers start in the UNKNOWN state. + this.state = PeerState.UNKNOWN; + this.lastAnnounce = null; + + this.uploaded = 0; + this.downloaded = 0; + this.left = 0; + } + + /** + * Update this peer's state and information. + * + *

+ * Note: if the peer reports 0 bytes left to download, its state will + * be automatically be set to COMPLETED. + *

+ * + * @param state The peer's state. + * @param uploaded Uploaded byte count, as reported by the peer. + * @param downloaded Downloaded byte count, as reported by the peer. + * @param left Left-to-download byte count, as reported by the peer. + */ + public void update(PeerState state, long uploaded, long downloaded, + long left) { + if (PeerState.STARTED.equals(state) && left == 0) { + state = PeerState.COMPLETED; + } + + if (!state.equals(this.state)) { + logger.info("Peer {} {} download of {}.", + new Object[] { + this, + state.name().toLowerCase(), + this.torrent, + }); + } + + this.state = state; + this.lastAnnounce = new Date(); + this.uploaded = uploaded; + this.downloaded = downloaded; + this.left = left; + } + + /** + * Tells whether this peer has completed its download and can thus be + * considered a seeder. + */ + public boolean isCompleted() { + return PeerState.COMPLETED.equals(this.state); + } + + /** + * Returns how many bytes the peer reported it has uploaded so far. + */ + public long getUploaded() { + return this.uploaded; + } + + /** + * Returns how many bytes the peer reported it has downloaded so far. + */ + public long getDownloaded() { + return this.downloaded; + } + + /** + * Returns how many bytes the peer reported it needs to retrieve before + * its download is complete. + */ + public long getLeft() { + return this.left; + } + + /** + * Tells whether this peer has checked in with the tracker recently. + * + *

+ * Non-fresh peers are automatically terminated and collected by the + * Tracker. + *

+ */ + public boolean isFresh() { + return (this.lastAnnounce != null && + (this.lastAnnounce.getTime() + (FRESH_TIME_SECONDS * 1000) > + new Date().getTime())); + } + + /** + * Returns a BEValue representing this peer for inclusion in an + * announce reply from the tracker. + * + * The returned BEValue is a dictionary containing the peer ID (in its + * original byte-encoded form), the peer's IP and the peer's port. + */ + public BEValue toBEValue() throws UnsupportedEncodingException { + Map peer = new HashMap(); + if (this.hasPeerId()) { + peer.put("peer id", new BEValue(this.getPeerId().array())); + } + peer.put("ip", new BEValue(this.getIp(), Torrent.BYTE_ENCODING)); + peer.put("port", new BEValue(this.getPort())); + return new BEValue(peer); + } +} + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/tracker/TrackedTorrent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/tracker/TrackedTorrent.java new file mode 100644 index 0000000..04f39bb --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/tracker/TrackedTorrent.java @@ -0,0 +1,306 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.tracker; + + +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceRequestMessage.RequestEvent; +import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Tracked torrents are torrent for which we don't expect to have data files + * for. + * + *

+ * {@link TrackedTorrent} objects are used by the BitTorrent tracker to + * represent a torrent that is announced by the tracker. As such, it is not + * expected to point to any valid local data like. It also contains some + * additional information used by the tracker to keep track of which peers + * exchange on it, etc. + *

+ * + * @author mpetazzoni + */ +public class TrackedTorrent extends Torrent { + + private static final Logger logger = + LoggerFactory.getLogger(TrackedTorrent.class); + + /** Minimum announce interval requested from peers, in seconds. */ + public static final int MIN_ANNOUNCE_INTERVAL_SECONDS = 5; + + /** Default number of peers included in a tracker response. */ + private static final int DEFAULT_ANSWER_NUM_PEERS = 30; + + /** Default announce interval requested from peers, in seconds. */ + private static final int DEFAULT_ANNOUNCE_INTERVAL_SECONDS = 10; + + private int answerPeers; + private int announceInterval; + + /** Peers currently exchanging on this torrent. */ + private ConcurrentMap peers; + public long createTimeMillis; + + /** + * Create a new tracked torrent from meta-info binary data. + * + * @param torrent The meta-info byte data. + * @throws IOException When the info dictionary can't be + * encoded and hashed back to create the torrent's SHA-1 hash. + */ + public TrackedTorrent(byte[] torrent) throws IOException { + super(torrent, false); + + this.peers = new ConcurrentHashMap(); + this.answerPeers = TrackedTorrent.DEFAULT_ANSWER_NUM_PEERS; + this.announceInterval = TrackedTorrent.DEFAULT_ANNOUNCE_INTERVAL_SECONDS; + this.createTimeMillis = System.currentTimeMillis(); + } + + public TrackedTorrent(Torrent torrent) throws IOException { + this(torrent.getEncoded()); + } + + /** + * Returns the map of all peers currently exchanging on this torrent. + */ + public Map getPeers() { + return this.peers; + } + + /** + * Add a peer exchanging on this torrent. + * + * @param peer The new Peer involved with this torrent. + */ + public void addPeer(TrackedPeer peer) { + logger.info("torrent " + this.getHexInfoHash() + " add new peer: " + peer.getPort()); + this.peers.put(peer.getHexPeerId(), peer); + } + + /** + * Retrieve a peer exchanging on this torrent. + * + * @param peerId The hexadecimal representation of the peer's ID. + */ + public TrackedPeer getPeer(String peerId) { + return this.peers.get(peerId); + } + + /** + * Remove a peer from this torrent's swarm. + * + * @param peerId The hexadecimal representation of the peer's ID. + */ + public TrackedPeer removePeer(String peerId) { + return this.peers.remove(peerId); + } + + /** + * Count the number of seeders (peers in the COMPLETED state) on this + * torrent. + */ + public int seeders() { + int count = 0; + for (TrackedPeer peer : this.peers.values()) { + if (peer.isCompleted()) { + count++; + } + } + return count; + } + + /** + * Count the number of leechers (non-COMPLETED peers) on this torrent. + */ + public int leechers() { + int count = 0; + for (TrackedPeer peer : this.peers.values()) { + if (!peer.isCompleted()) { + count++; + } + } + return count; + } + + /** + * Remove unfresh peers from this torrent. + * + *

+ * Collect and remove all non-fresh peers from this torrent. This is + * usually called by the periodic peer collector of the BitTorrent tracker. + *

+ */ + public void collectUnfreshPeers() { + for (TrackedPeer peer : this.peers.values()) { + if (!peer.isFresh()) { + this.peers.remove(peer.getHexPeerId()); + } + } + } + + /** + * Get the announce interval for this torrent. + */ + public int getAnnounceInterval() { + return this.announceInterval; + } + + /** + * Set the announce interval for this torrent. + * + * @param interval New announce interval, in seconds. + */ + public void setAnnounceInterval(int interval) { + if (interval <= 0) { + throw new IllegalArgumentException("Invalid announce interval"); + } + + this.announceInterval = interval; + } + + /** + * Update this torrent's swarm from an announce event. + * + *

+ * This will automatically create a new peer on a 'started' announce event, + * and remove the peer on a 'stopped' announce event. + *

+ * + * @param event The reported event. If null, means a regular + * interval announce event, as defined in the BitTorrent specification. + * @param peerId The byte-encoded peer ID. + * @param hexPeerId The hexadecimal representation of the peer's ID. + * @param ip The peer's IP address. + * @param port The peer's inbound port. + * @param uploaded The peer's reported uploaded byte count. + * @param downloaded The peer's reported downloaded byte count. + * @param left The peer's reported left to download byte count. + * @return The peer that sent us the announce request. + */ + public TrackedPeer update(RequestEvent event, ByteBuffer peerId, + String hexPeerId, String ip, int port, long uploaded, long downloaded, + long left) throws UnsupportedEncodingException { + TrackedPeer peer; + TrackedPeer.PeerState state = TrackedPeer.PeerState.UNKNOWN; + + if (RequestEvent.STARTED.equals(event)) { + peer = new TrackedPeer(this, ip, port, peerId); + state = TrackedPeer.PeerState.STARTED; + this.addPeer(peer); + } else if (RequestEvent.STOPPED.equals(event)) { + peer = this.removePeer(hexPeerId); + state = TrackedPeer.PeerState.STOPPED; + } else if (RequestEvent.COMPLETED.equals(event)) { + peer = this.getPeer(hexPeerId); + state = TrackedPeer.PeerState.COMPLETED; + } else if (RequestEvent.NONE.equals(event)) { + peer = this.getPeer(hexPeerId); + state = TrackedPeer.PeerState.STARTED; + } else { + throw new IllegalArgumentException("Unexpected announce event type!"); + } + + peer.update(state, uploaded, downloaded, left); + return peer; + } + + /** + * Get a list of peers we can return in an announce response for this + * torrent. + * + * @param peer The peer making the request, so we can exclude it from the + * list of returned peers. + * @return A list of peers we can include in an announce response. + */ + public List getSomePeers(TrackedPeer peer) { + List peers = new LinkedList(); + + + // Extract answerPeers random peers + List candidates = + new LinkedList(this.peers.values()); + Collections.shuffle(candidates); + + logger.info("get peers for " + peer.getHexPeerId()); + logger.info("number of peers: " + candidates.size()); + + int count = 0; + for (TrackedPeer candidate : candidates) { + // Collect unfresh peers, and obviously don't serve them as well. + if (!candidate.isFresh() || candidate.looksLike(peer) && !candidate.equals(peer)) { + logger.info("peer " + candidate.getHexPeerId() + "not fresh, removed"); + + logger.info("Collecting stale peer {}...", candidate); + this.peers.remove(candidate.getHexPeerId()); + continue; + } + + // Don't include the requesting peer in the answer. + if (peer.getHexPeerId().equals(candidate.getHexPeerId())) { + logger.info("peer " + candidate.getHexPeerId() + "is target self, removed"); + continue; + } + + // Collect unfresh peers, and obviously don't serve them as well. + if (!candidate.isFresh()) { + logger.info("Collecting stale peer {}...", + candidate.getHexPeerId()); + this.peers.remove(candidate.getHexPeerId()); + continue; + } + + // Only serve at most ANSWER_NUM_PEERS peers + if (count++ > this.answerPeers) { + break; + } + + peers.add(candidate); + } + + return peers; + } + + /** + * Load a tracked torrent from the given torrent file. + * + * @param torrent The abstract {@link File} object representing the + * .torrent file to load. + * @throws IOException When the torrent file cannot be read. + */ + public static TrackedTorrent load(File torrent) throws IOException { + byte[] data = FileUtils.readFileToByteArray(torrent); + return new TrackedTorrent(data); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/test/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TestTrackerClientFaultTolerance.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/test/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TestTrackerClientFaultTolerance.java new file mode 100644 index 0000000..9847346 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/test/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TestTrackerClientFaultTolerance.java @@ -0,0 +1,221 @@ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceRequestMessage.RequestEvent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.ErrorMessage.FailureReason; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.MessageValidationException; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPAnnounceRequestMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPAnnounceResponseMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPTrackerErrorMessage; +import org.apache.hadoop.yarn.server.broadcast.service.Topology; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.ArrayList; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.notNull; +import static org.mockito.Mockito.*; + +public class TestTrackerClientFaultTolerance { + SharedTorrent torrent; + Peer peer; + + @Before + public void initialize() { + torrent = mock(SharedTorrent.class); + when(torrent.getHexInfoHash()).thenReturn(DigestUtils.sha1Hex("torrent")); + when(torrent.getInfoHash()).thenReturn("infohash".getBytes()); + // FIXME these method may be not useful, may remove them + when(torrent.getUploaded()).thenReturn(0L); + when(torrent.getDownloaded()).thenReturn(0L); + when(torrent.getLeft()).thenReturn(0L); + + peer = new Peer("localhost", 30000); + peer.setPeerId(ByteBuffer.wrap("peerid".getBytes())); + } + + @Test + /** + * if trackerClient's announcement to rackMaster but get UNKNOWN_TORRENT response, it should post torrent to tracker + */ + public void testPostTorrent() { + // construct tracker client + Topology topology = mock(Topology.class); + TrackerClientTransport transport = mock(TrackerClientTransport.class); + TrackerClient trackerClient = new TrackerClient(torrent, peer, transport, topology); + + // setup behavior + + when(topology.getRackMaster((String) notNull())).thenReturn("192.168.3.2"); + when(topology.getLocalAddr()).thenReturn(""); + try { + when(transport.announce((URL) notNull(), (HTTPAnnounceRequestMessage) notNull())) + .thenReturn(HTTPTrackerErrorMessage.craft(FailureReason.UNKNOWN_TORRENT)) + .thenReturn(HTTPAnnounceResponseMessage.craft(10, 1, "1", 1, 1, new ArrayList())); + } catch (IOException e) { + e.printStackTrace(); + } catch (MessageValidationException e) { + e.printStackTrace(); + } + + // let a tracker client announce + RequestEvent event = RequestEvent.STARTED; + boolean inhibitEvents = false; + try { + trackerClient.announce(event, inhibitEvents); + } catch (AnnounceException e) { + // this exception is expected + } + + // examine whether peer announce to next + verify(transport).postTorrent((String)any(), (byte[])any()); + } + + @Test + /** + * if trackerClient's announcement to rackMaster failed, it should announce to the next one in the rack + */ + public void testNodeAnnounceToRackMasterFail() { + String rackMaster = "192.168.3.2", nextRackMaster = "192.168.3.5", localAddr = "192.168.3.3"; + + // construct tracker client + Topology topology = mock(Topology.class); + TrackerClientTransport transport = mock(TrackerClientTransport.class); + TrackerClient trackerClient = new TrackerClient(torrent, peer, transport, topology); + + // setup behavior + when(topology.getRackMaster((String) notNull())).thenReturn(rackMaster); + when(topology.getLocalAddr()).thenReturn(localAddr); + when(topology.getNextRackMaster(rackMaster)).thenReturn(nextRackMaster); + try { + when(transport.announce((URL) notNull(), (HTTPAnnounceRequestMessage) notNull())) + .thenThrow(new IOException()) + .thenReturn(HTTPAnnounceResponseMessage.craft(10, 1, "1", 1, 1, new ArrayList())); + } catch (IOException e) { + e.printStackTrace(); + } catch (MessageValidationException e) { + e.printStackTrace(); + } + + // let a tracker client announce + RequestEvent event = RequestEvent.NONE; + boolean inhibitEvents = false; + try { + trackerClient.announce(event, inhibitEvents); + } catch (AnnounceException e) { + // this exception is expected + } + + // examine whether peer announce to next + verify(topology).getNextRackMaster((String) any()); + try { + ArgumentCaptor announceURLCaptor = ArgumentCaptor.forClass(URL.class); + verify(transport, times(2)).announce(announceURLCaptor.capture(), (HTTPAnnounceRequestMessage) any()); + assertTrue(announceURLCaptor.getAllValues().get(1).getHost().equals(nextRackMaster)); + } catch (IOException | MessageValidationException e) { + // nothing to do, just wrap the exceptions + } + } + + @Test + /** + * if a failed rackMaster recovers, trackerClients should announce to it again + */ + public void testNodeAnnounceToRackMasterRecover() { + String rackMaster = "192.168.3.2", nextRackMaster = "192.168.3.5", localAddr = "192.168.3.3"; + + // construct tracker client + Topology topology = mock(Topology.class); + TrackerClientTransport transport = mock(TrackerClientTransport.class); + TrackerClient trackerClient = new TrackerClient(torrent, peer, transport, topology); + + // setup behavior + when(topology.getRackMaster((String) notNull())).thenReturn(rackMaster); + when(topology.getLocalAddr()).thenReturn(localAddr); + when(topology.getNextRackMaster(rackMaster)).thenReturn(nextRackMaster); + try { + when(transport.announce((URL) notNull(), (HTTPAnnounceRequestMessage) notNull())) + .thenThrow(new IOException()) + .thenReturn(HTTPAnnounceResponseMessage.craft(10, 1, "1", 1, 1, new ArrayList())) + .thenReturn(HTTPAnnounceResponseMessage.craft(10, 1, "1", 1, 1, new ArrayList())); + } catch (IOException e) { + e.printStackTrace(); + } catch (MessageValidationException e) { + e.printStackTrace(); + } + + // let a tracker client announce + RequestEvent event = RequestEvent.NONE; + boolean inhibitEvents = false; + try { + trackerClient.announce(event, inhibitEvents); + trackerClient.announce(event, inhibitEvents); + } catch (AnnounceException e) { + // this exception is expected + } + + // examine whether peer announce to next + try { + ArgumentCaptor announceURLCaptor = ArgumentCaptor.forClass(URL.class); + verify(transport, times(3)).announce(announceURLCaptor.capture(), (HTTPAnnounceRequestMessage) any()); + assertTrue(announceURLCaptor.getAllValues().get(0).getHost().equals(rackMaster)); + assertTrue(announceURLCaptor.getAllValues().get(1).getHost().equals(nextRackMaster)); + assertTrue(announceURLCaptor.getAllValues().get(2).getHost().equals(rackMaster)); + } catch (IOException | MessageValidationException e) { + // nothing to do, just wrap the exceptions + } + } + + @Test + /** + * if trackerClient is on the same host of rackMaster, it should announce to both rackMaster and globalMaster + */ + public void testNodeBecomeRackMaster() { + String rackMaster = "192.168.3.2", globalMaster = "192.168.3.5", localAddr = rackMaster; + + // construct tracker client + Topology topology = mock(Topology.class); + TrackerClientTransport transport = mock(TrackerClientTransport.class); + TrackerClient trackerClient = new TrackerClient(torrent, peer, transport, topology); + + // setup behavior + when(topology.getRackMaster((String) notNull())).thenReturn(rackMaster); + when(topology.getLocalAddr()).thenReturn(localAddr); + when(topology.getGlobalMaster((String) notNull())).thenReturn(globalMaster); + try { + when(transport.announce((URL) notNull(), (HTTPAnnounceRequestMessage) notNull())) + .thenReturn(HTTPAnnounceResponseMessage.craft(10, 1, "1", 1, 1, new ArrayList())); + } catch (IOException e) { + e.printStackTrace(); + } catch (MessageValidationException e) { + e.printStackTrace(); + } + + // let a tracker client announce + RequestEvent event = RequestEvent.NONE; + boolean inhibitEvents = false; + try { + trackerClient.announce(event, inhibitEvents); + } catch (AnnounceException e) { + // this exception is expected + } + + // examine whether peer announce to both rackMaster and globalMaster + try { + ArgumentCaptor announceURLCaptor = ArgumentCaptor.forClass(URL.class); + verify(transport, times(2)).announce(announceURLCaptor.capture(), (HTTPAnnounceRequestMessage) any()); + assertTrue(announceURLCaptor.getAllValues().get(0).getHost().equals(rackMaster)); + assertTrue(announceURLCaptor.getAllValues().get(1).getHost().equals(globalMaster)); + } catch (IOException | MessageValidationException e) { + // nothing to do, just wrap the exceptions + } + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/test/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TestTrackerClientLoadBalance.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/test/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TestTrackerClientLoadBalance.java new file mode 100644 index 0000000..19c7148 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/test/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TestTrackerClientLoadBalance.java @@ -0,0 +1,93 @@ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.yarn.server.broadcast.service.Topology; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import static org.junit.Assert.*; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +public class TestTrackerClientLoadBalance { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private String topologyFileName = "topology"; + private File topologyFile; + + @Before + public void initialize() { + try { + topologyFile = folder.newFile(topologyFileName); + BufferedWriter bw = new BufferedWriter(new FileWriter(topologyFile)); + for (int i = 1; i < 9; i++) + bw.write("192.168.3."+i+" "+(i%3)+"\n"); + bw.write(InetAddress.getLocalHost().getHostAddress()+" "+0+"\n"); + bw.close(); + } catch (IOException e) { + e.printStackTrace(); + assertTrue(false); + } + } + + @After + public void destory() { + topologyFile.delete(); + } + + @Test + /** + * torrent should be distributed on all node in a rack + */ + public void testRackMasterForDifferentTorrent() { + Topology topology = new Topology(topologyFile.getAbsolutePath()); + Map cntHandledTorrent = new HashMap<>(); + Random randomGenerator = new Random(); + + // simulate 100 torrents + for (int i = 0; i < 100; i++) { + String key = DigestUtils.sha1Hex(String.valueOf(randomGenerator.nextDouble())); + String rackMaster = topology.getRackMaster(key); + cntHandledTorrent.put(rackMaster, cntHandledTorrent.containsKey(rackMaster) ? cntHandledTorrent.get(rackMaster) + 1 : 1); + } + + // check distribution + for (String rackMaster : cntHandledTorrent.keySet()) { + if (cntHandledTorrent.get(rackMaster) == 0) + assertTrue(false); + } + } + + @Test + /** + * torrent should be distributed on all rack + */ + public void testGlobalMasterForDifferentTorrent() { + Topology topology = new Topology(topologyFile.getAbsolutePath()); + Map cntHandledTorrent = new HashMap<>(); + Random randomGenerator = new Random(); + + // simulate 100 torrents + for (int i = 0; i < 100; i++) { + String key = DigestUtils.sha1Hex(String.valueOf(randomGenerator.nextDouble())); + String globalMaster = topology.getGlobalMaster(key); + cntHandledTorrent.put(globalMaster, cntHandledTorrent.containsKey(globalMaster) ? cntHandledTorrent.get(globalMaster) + 1 : 1); + } + + // check distribution + for (String globalMaster : cntHandledTorrent.keySet()) { + if (cntHandledTorrent.get(globalMaster) == 0) + assertTrue(false); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 71e4f39..0d0dce3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -41,6 +41,7 @@ hadoop-common provided
+ @@ -48,6 +49,12 @@ hadoop-yarn-common + + org.apache.hadoop + hadoop-yarn-server-broadcast + 2.8.0-SNAPSHOT + + org.apache.hadoop hadoop-yarn-api diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 65fd9d8..b7d911d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -55,6 +55,7 @@ import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -62,6 +63,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.broadcast.service.BTDownload; import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol; import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus; @@ -200,7 +202,17 @@ ExecutorService createDownloadThreadPool() { Callable download(Path path, LocalResource rsrc, UserGroupInformation ugi) throws IOException { DiskChecker.checkDir(new File(path.toUri().getRawPath())); - return new FSDownload(lfs, ugi, conf, path, rsrc); + if (rsrc.getType() == LocalResourceType.BTFILE) { + return new BTDownload(lfs, ugi, conf, path, rsrc, appId); + } else { + String str = null; + try { + str = ConverterUtils.getPathFromYarnURL(rsrc.getResource()).toString(); + } catch (Exception e) { + e.printStackTrace(); + } + return new FSDownload(lfs, ugi, conf, path, rsrc); + } } static long getEstimatedSize(LocalResource rsrc) { @@ -432,3 +444,4 @@ private static void createDir(FileContext lfs, Path dirPath, } } } + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml index 80544bd..a1f031a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -36,6 +36,7 @@ hadoop-yarn-server-common + hadoop-yarn-server-broadcast hadoop-yarn-server-nodemanager hadoop-yarn-server-web-proxy hadoop-yarn-server-resourcemanager