+ * A b-encoded byte stream can represent byte arrays, numbers, lists and maps + * (dictionaries). This class implements a decoder of such streams into + * {@link BEValue}s. + *
+ * + *+ * Inspired by Snark's implementation. + *
+ * + * @see B-encoding specification + */ +public class BDecoder { + + // The InputStream to BDecode. + private final InputStream in; + + // The last indicator read. + // Zero if unknown. + // '0'..'9' indicates a byte[]. + // 'i' indicates an Number. + // 'l' indicates a List. + // 'd' indicates a Map. + // 'e' indicates end of Number, List or Map (only used internally). + // -1 indicates end of stream. + // Call getNextIndicator to get the current value (will never return zero). + private int indicator = 0; + + /** + * Initializes a new BDecoder. + * + *
+ * Nothing is read from the given InputStream yet.
+ *
+ * Automatically instantiates a new BDecoder for the provided input stream + * and decodes its root member. + *
+ * + * @param in The input stream to read from. + */ + public static BEValue bdecode(InputStream in) throws IOException { + return new BDecoder(in).bdecode(); + } + + /** + * Decode a B-encoded byte buffer. + * + *+ * Automatically instantiates a new BDecoder for the provided buffer and + * decodes its root member. + *
+ * + * @param data The {@link ByteBuffer} to read from. + */ + public static BEValue bdecode(ByteBuffer data) throws IOException { + return BDecoder.bdecode(new AutoCloseInputStream( + new ByteArrayInputStream(data.array()))); + } + + /** + * Returns what the next b-encoded object will be on the stream or -1 + * when the end of stream has been reached. + * + *+ * Can return something unexpected (not '0' .. '9', 'i', 'l' or 'd') when + * the stream isn't b-encoded. + *
+ * + * This might or might not read one extra byte from the stream. + */ + private int getNextIndicator() throws IOException { + if (this.indicator == 0) { + this.indicator = in.read(); + } + return this.indicator; + } + + /** + * Gets the next indicator and returns either null when the stream + * has ended or b-decodes the rest of the stream and returns the + * appropriate BEValue encoded object. + */ + public BEValue bdecode() throws IOException { + if (this.getNextIndicator() == -1) + return null; + + if (this.indicator >= '0' && this.indicator <= '9') + return this.bdecodeBytes(); + else if (this.indicator == 'i') + return this.bdecodeNumber(); + else if (this.indicator == 'l') + return this.bdecodeList(); + else if (this.indicator == 'd') + return this.bdecodeMap(); + else + throw new InvalidBEncodingException + ("Unknown indicator '" + this.indicator + "'"); + } + + /** + * Returns the next b-encoded value on the stream and makes sure it is a + * byte array. + * + * @throws InvalidBEncodingException If it is not a b-encoded byte array. + */ + public BEValue bdecodeBytes() throws IOException { + int c = this.getNextIndicator(); + int num = c - '0'; + if (num < 0 || num > 9) + throw new InvalidBEncodingException("Number expected, not '" + + (char)c + "'"); + this.indicator = 0; + + c = this.read(); + int i = c - '0'; + while (i >= 0 && i <= 9) { + // This can overflow! + num = num*10 + i; + c = this.read(); + i = c - '0'; + } + + if (c != ':') { + throw new InvalidBEncodingException("Colon expected, not '" + + (char)c + "'"); + } + + return new BEValue(read(num)); + } + + /** + * Returns the next b-encoded value on the stream and makes sure it is a + * number. + * + * @throws InvalidBEncodingException If it is not a number. + */ + public BEValue bdecodeNumber() throws IOException { + int c = this.getNextIndicator(); + if (c != 'i') { + throw new InvalidBEncodingException("Expected 'i', not '" + + (char)c + "'"); + } + this.indicator = 0; + + c = this.read(); + if (c == '0') { + c = this.read(); + if (c == 'e') + return new BEValue(BigInteger.ZERO); + else + throw new InvalidBEncodingException("'e' expected after zero," + + " not '" + (char)c + "'"); + } + + // We don't support more the 255 char big integers + char[] chars = new char[256]; + int off = 0; + + if (c == '-') { + c = this.read(); + if (c == '0') + throw new InvalidBEncodingException("Negative zero not allowed"); + chars[off] = '-'; + off++; + } + + if (c < '1' || c > '9') + throw new InvalidBEncodingException("Invalid Integer start '" + + (char)c + "'"); + chars[off] = (char)c; + off++; + + c = this.read(); + int i = c - '0'; + while (i >= 0 && i <= 9) { + chars[off] = (char)c; + off++; + c = read(); + i = c - '0'; + } + + if (c != 'e') + throw new InvalidBEncodingException("Integer should end with 'e'"); + + String s = new String(chars, 0, off); + return new BEValue(new BigInteger(s)); + } + + /** + * Returns the next b-encoded value on the stream and makes sure it is a + * list. + * + * @throws InvalidBEncodingException If it is not a list. + */ + public BEValue bdecodeList() throws IOException { + int c = this.getNextIndicator(); + if (c != 'l') { + throw new InvalidBEncodingException("Expected 'l', not '" + + (char)c + "'"); + } + this.indicator = 0; + + List+ * This class provides utility methods to encode objects and + * {@link BEValue}s to B-encoding into a provided output stream. + *
+ * + *+ * Inspired by Snark's implementation. + *
+ * + * @see B-encoding specification + */ +public class BEncoder { + + @SuppressWarnings("unchecked") + public static void bencode(Object o, OutputStream out) + throws IOException, IllegalArgumentException { + if (o instanceof BEValue) { + o = ((BEValue)o).getValue(); + } + + if (o instanceof String) { + bencode((String)o, out); + } else if (o instanceof byte[]) { + bencode((byte[])o, out); + } else if (o instanceof Number) { + bencode((Number)o, out); + } else if (o instanceof List) { + bencode((List+ * A BitTorrent client in its bare essence shares a given torrent. If the + * torrent is not complete locally, it will continue to download it. If or + * after the torrent is complete, the client may eventually continue to seed it + * for other clients. + *
+ * + *+ * This BitTorrent client implementation is made to be simple to embed and + * simple to use. First, initialize a ShareTorrent object from a torrent + * meta-info source (either a file or a byte array, see + * com.turn.ttorrent.SharedTorrent for how to create a SharedTorrent object). + * Then, instantiate your Client object with this SharedTorrent and call one of + * {@link #download} to simply download the torrent, or {@link #share} to + * download and continue seeding for the given amount of time after the + * download completes. + *
+ * + */ +public class Client extends Observable implements Runnable, + AnnounceResponseListener, IncomingConnectionListener, PeerActivityListener { + + private static final Logger logger = + LoggerFactory.getLogger(Client.class); + + /** Peers unchoking frequency, in seconds. Current BitTorrent specification + * recommends 10 seconds to avoid choking fibrilation. */ + private static final int UNCHOKING_FREQUENCY = 3; + + /** Optimistic unchokes are done every 2 loop iterations, i.e. every + * 2*UNCHOKING_FREQUENCY seconds. */ + private static final int OPTIMISTIC_UNCHOKE_ITERATIONS = 3; + + private static final int RATE_COMPUTATION_ITERATIONS = 2; + private static final int MAX_DOWNLOADERS_UNCHOKE = 4; + + public Client(InetAddress address, SharedTorrent sharedTorrent) throws IOException { + this.torrent = sharedTorrent; + this.state = ClientState.WAITING; + + String id = Client.BITTORRENT_ID_PREFIX + UUID.randomUUID() + .toString().split("-")[4]; + + // Initialize the incoming connection handler and register ourselves to + // it. + this.service = new ConnectionHandler(this.torrent, id, address); + this.service.register(this); + + this.self = new Peer( + this.service.getSocketAddress() + .getAddress().getHostAddress(), + (short)this.service.getSocketAddress().getPort(), + ByteBuffer.wrap(id.getBytes(Torrent.BYTE_ENCODING))); + + // Initialize the announce request thread, and register ourselves to it + // as well. + this.announce = new Announce(this.torrent, this.self); + this.announce.register(this); + + logger.info("BitTorrent client [{}] for {} started and " + + "listening at {}:{}...", + new Object[] { + this.self.getShortHexPeerId(), + this.torrent.getName(), + this.self.getIp(), + this.self.getPort() + }); + + this.peers = new ConcurrentHashMap+ * If the state has changed, this client's observers will be notified. + *
+ * + * @param state The new client state. + */ + private synchronized void setState(ClientState state) { + if (this.state != state) { + this.setChanged(); + } + this.state = state; + this.notifyObservers(this.state); + } + + /** + * Return the current state of this BitTorrent client. + */ + public ClientState getState() { + return this.state; + } + + /** + * Download the torrent without seeding after completion. + */ + public void download() { + this.share(0); + } + + /** + * Download and share this client's torrent until interrupted. + */ + public void share() { + this.share(-1); + } + + /** + * Download and share this client's torrent. + * + * @param seed Seed time in seconds after the download is complete. Pass + *0 to immediately stop after downloading.
+ */
+ public synchronized void share(int seed) {
+ this.seed = seed;
+ this.stop = false;
+
+ if (this.thread == null || !this.thread.isAlive()) {
+ this.thread = new Thread(this);
+ this.thread.setName("bt-client(" +
+ this.self.getShortHexPeerId() + ")");
+ this.thread.setDaemon(true);
+ this.thread.start();
+ }
+ }
+
+ /**
+ * Immediately but gracefully stop this client.
+ */
+ public void stop() {
+ this.stop(true);
+ }
+
+ /**
+ * Immediately but gracefully stop this client.
+ *
+ * @param wait Whether to wait for the client execution thread to complete
+ * or not. This allows for the client's state to be settled down in one of
+ * the DONE or ERROR states when this method returns.
+ */
+ public void stop(boolean wait) {
+ this.stop = true;
+
+ if (this.thread != null && this.thread.isAlive()) {
+ this.thread.interrupt();
+ if (wait) {
+ this.waitForCompletion();
+ }
+ }
+
+ this.thread = null;
+ }
+
+ /**
+ * Wait for downloading (and seeding, if requested) to complete.
+ */
+ public void waitForCompletion() {
+ if (this.thread != null && this.thread.isAlive()) {
+ try {
+ this.thread.join();
+ } catch (InterruptedException ie) {
+ logger.error(ie.getMessage(), ie);
+ }
+ }
+ }
+
+ /**
+ * Tells whether we are a seed for the torrent we're sharing.
+ */
+ public boolean isSeed() {
+ return this.torrent.isComplete();
+ }
+
+ /**
+ * Main client loop.
+ *
+ * + * The main client download loop is very simple: it starts the announce + * request thread, the incoming connection handler service, and loops + * unchoking peers every UNCHOKING_FREQUENCY seconds until told to stop. + * Every OPTIMISTIC_UNCHOKE_ITERATIONS, an optimistic unchoke will be + * attempted to try out other peers. + *
+ * + *+ * Once done, it stops the announce and connection services, and returns. + *
+ */ + @Override + public void run() { + // First, analyze the torrent's local data. + try { + this.setState(ClientState.VALIDATING); + this.torrent.init(); + } catch (IOException ioe) { + logger.warn("Error while initializing torrent data: {}!", + ioe.getMessage(), ioe); + } catch (InterruptedException ie) { + logger.warn("Client was interrupted during initialization. " + + "Aborting right away."); + } finally { + if (!this.torrent.isInitialized()) { + try { + this.service.close(); + } catch (IOException ioe) { + logger.warn("Error while releasing bound channel: {}!", + ioe.getMessage(), ioe); + } + + this.setState(ClientState.ERROR); + this.torrent.close(); + return; + } + } + + // Initial completion test + if (this.torrent.isComplete()) { + this.seed(); + } else { + this.setState(ClientState.SHARING); + } + + // Detect early stop + if (this.stop) { + logger.info("Download is complete and no seeding was requested."); + this.finish(); + return; + } + + this.announce.start(); + this.service.start(); + + int optimisticIterations = 0; + int rateComputationIterations = 0; + + while (!this.stop) { + optimisticIterations = + (optimisticIterations == 0 ? + Client.OPTIMISTIC_UNCHOKE_ITERATIONS : + optimisticIterations - 1); + + rateComputationIterations = + (rateComputationIterations == 0 ? + Client.RATE_COMPUTATION_ITERATIONS : + rateComputationIterations - 1); + + try { + this.unchokePeers(optimisticIterations == 0); + this.info(); + if (rateComputationIterations == 0) { + this.resetPeerRates(); + } + } catch (Exception e) { + logger.error("An exception occurred during the BitTorrent " + + "client main loop execution!", e); + } + + try { + Thread.sleep(Client.UNCHOKING_FREQUENCY*1000); + } catch (InterruptedException ie) { + logger.trace("BitTorrent main loop interrupted."); + } + } + + logger.debug("Stopping BitTorrent client connection service " + + "and announce threads..."); + + this.service.stop(); + try { + this.service.close(); + } catch (IOException ioe) { + logger.warn("Error while releasing bound channel: {}!", + ioe.getMessage(), ioe); + } + + this.announce.stop(); + + // Close all peer connections + logger.debug("Closing all remaining peer connections..."); + for (SharingPeer peer : this.connected.values()) { + peer.unbind(false); + } + + this.finish(); + } + + /** + * Close torrent and set final client state before signing off. + */ + private void finish() { + this.torrent.close(); + + // Determine final state + if (this.torrent.isFinished()) { + this.setState(ClientState.DONE); + } else { + this.setState(ClientState.ERROR); + } + + logger.info("BitTorrent client signing off."); + } + + /** + * Display information about the BitTorrent client state. + * + *+ * This emits an information line in the log about this client's state. It + * includes the number of choked peers, number of connected peers, number + * of known peers, information about the torrent availability and + * completion and current transmission rates. + *
+ */ + public synchronized void info() { + float dl = 0; + float ul = 0; + for (SharingPeer peer : this.connected.values()) { + dl += peer.getDLRate().get(); + ul += peer.getULRate().get(); + } + + logger.info("{} {}/{} pieces ({}%) [{}/{}] with {}/{} peers at {}/{} kB/s.", + new Object[] { + this.getState().name(), + this.torrent.getCompletedPieces().cardinality(), + this.torrent.getPieceCount(), + String.format("%.2f", this.torrent.getCompletion()), + this.torrent.getAvailablePieces().cardinality(), + this.torrent.getRequestedPieces().cardinality(), + this.connected.size(), + this.peers.size(), + String.format("%.2f", dl/1024.0), + String.format("%.2f", ul/1024.0), + }); + for (SharingPeer peer : this.connected.values()) { + Piece piece = peer.getRequestedPiece(); + logger.debug(" | {} {}", + peer, + piece != null + ? "(downloading " + piece + ")" + : "" + ); + } + } + + /** + * Reset peers download and upload rates. + * + *+ * This method is called every RATE_COMPUTATION_ITERATIONS to reset the + * download and upload rates of all peers. This contributes to making the + * download and upload rate computations rolling averages every + * UNCHOKING_FREQUENCY * RATE_COMPUTATION_ITERATIONS seconds (usually 20 + * seconds). + *
+ */ + private synchronized void resetPeerRates() { + for (SharingPeer peer : this.connected.values()) { + peer.getDLRate().reset(); + peer.getULRate().reset(); + } + } + + /** + * Retrieve a SharingPeer object from the given peer specification. + * + *+ * This function tries to retrieve an existing peer object based on the + * provided peer specification or otherwise instantiates a new one and adds + * it to our peer repository. + *
+ * + * @param search The {@link Peer} specification. + */ + private SharingPeer getOrCreatePeer(Peer search) { + SharingPeer peer; + + synchronized (this.peers) { + logger.trace("Searching for {}...", search); + if (search.hasPeerId()) { + peer = this.peers.get(search.getHexPeerId()); + if (peer != null) { + logger.trace("Found peer (by peer ID): {}.", peer); + this.peers.put(peer.getHostIdentifier(), peer); + this.peers.put(search.getHostIdentifier(), peer); + return peer; + } + } + + peer = this.peers.get(search.getHostIdentifier()); + if (peer != null) { + if (search.hasPeerId()) { + logger.trace("Recording peer ID {} for {}.", + search.getHexPeerId(), peer); + peer.setPeerId(search.getPeerId()); + this.peers.put(search.getHexPeerId(), peer); + } + + logger.debug("Found peer (by host ID): {}.", peer); + return peer; + } + + peer = new SharingPeer(search.getIp(), search.getPort(), + search.getPeerId(), this.torrent); + logger.trace("Created new peer: {}.", peer); + + this.peers.put(peer.getHostIdentifier(), peer); + if (peer.hasPeerId()) { + this.peers.put(peer.getHexPeerId(), peer); + } + + return peer; + } + } + + /** + * Retrieve a peer comparator. + * + *+ * Returns a peer comparator based on either the download rate or the + * upload rate of each peer depending on our state. While sharing, we rely + * on the download rate we get from each peer. When our download is + * complete and we're only seeding, we use the upload rate instead. + *
+ * + * @return A SharingPeer comparator that can be used to sort peers based on + * the download or upload rate we get from them. + */ + private Comparator+ * This is one of the "clever" places of the BitTorrent client. Every + * OPTIMISTIC_UNCHOKING_FREQUENCY seconds, we decide which peers should be + * unchocked and authorized to grab pieces from us. + *
+ * + *+ * Reciprocation (tit-for-tat) and upload capping is implemented here by + * carefully choosing which peers we unchoke, and which peers we choke. + *
+ * + *+ * The four peers with the best download rate and are interested in us get + * unchoked. This maximizes our download rate as we'll be able to get data + * from there four "best" peers quickly, while allowing these peers to + * download from us and thus reciprocate their generosity. + *
+ * + *+ * Peers that have a better download rate than these four downloaders but + * are not interested get unchoked too, we want to be able to download from + * them to get more data more quickly. If one becomes interested, it takes + * a downloader's place as one of the four top downloaders (i.e. we choke + * the downloader with the worst upload rate). + *
+ * + * @param optimistic Whether to perform an optimistic unchoke as well. + */ + private synchronized void unchokePeers(boolean optimistic) { + // Build a set of all connected peers, we don't care about peers we're + // not connected to. + TreeSet+ * This handler is called once the connection has been successfully + * established and the handshake exchange made. This generally simply means + * binding the peer to the socket, which will put in place the communication + * thread and logic with this peer. + *
+ * + * @param channel The connected socket channel to the remote peer. Note + * that if the peer somehow rejected our handshake reply, this socket might + * very soon get closed, but this is handled down the road. + * @param peerId The byte-encoded peerId extracted from the peer's + * handshake, after validation. + * @see org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer + */ + @Override + public void handleNewPeerConnection(SocketChannel channel, byte[] peerId) { + Peer search = new Peer( + channel.socket().getInetAddress().getHostAddress(), + channel.socket().getPort(), + (peerId != null + ? ByteBuffer.wrap(peerId) + : null)); + + logger.info("Handling new peer connection with {}...", search); + SharingPeer peer = this.getOrCreatePeer(search); + + try { + synchronized (peer) { + if (peer.isConnected()) { + logger.info("Already connected with {}, closing link.", + peer); + channel.close(); + return; + } + + peer.register(this); + peer.bind(channel); + } + + this.connected.put(peer.getHexPeerId(), peer); + peer.register(this.torrent); + logger.debug("New peer connection with {} [{}/{}].", + new Object[] { + peer, + this.connected.size(), + this.peers.size() + }); + } catch (Exception e) { + this.connected.remove(peer.getHexPeerId()); + logger.warn("Could not handle new peer connection " + + "with {}: {}", peer, e.getMessage()); + } + } + + /** + * Handle a failed peer connection. + * + *+ * If an outbound connection failed (could not connect, invalid handshake, + * etc.), remove the peer from our known peers. + *
+ * + * @param peer The peer we were trying to connect with. + * @param cause The exception encountered when connecting with the peer. + */ + @Override + public void handleFailedConnection(SharingPeer peer, Throwable cause) { + logger.warn("Could not connect to {}: {}.", peer, cause.getMessage()); + this.peers.remove(peer.getHostIdentifier()); + if (peer.hasPeerId()) { + this.peers.remove(peer.getHexPeerId()); + } + } + + /** PeerActivityListener handler(s). **************************************/ + + @Override + public void handlePeerChoked(SharingPeer peer) { /* Do nothing */ } + + @Override + public void handlePeerReady(SharingPeer peer) { /* Do nothing */ } + + @Override + public void handlePieceAvailability(SharingPeer peer, + Piece piece) { /* Do nothing */ } + + @Override + public void handleBitfieldAvailability(SharingPeer peer, + BitSet availablePieces) { /* Do nothing */ } + + @Override + public void handlePieceSent(SharingPeer peer, + Piece piece) { /* Do nothing */ } + + /** + * Piece download completion handler. + * + *+ * When a piece is completed, and valid, we announce to all connected peers + * that we now have this piece. + *
+ * + *+ * We use this handler to identify when all of the pieces have been + * downloaded. When that's the case, we can start the seeding period, if + * any. + *
+ * + * @param peer The peer we got the piece from. + * @param piece The piece in question. + */ + @Override + public void handlePieceCompleted(SharingPeer peer, Piece piece) + throws IOException { + synchronized (this.torrent) { + if (piece.isValid()) { + // Make sure the piece is marked as completed in the torrent + // Note: this is required because the order the + // PeerActivityListeners are called is not defined, and we + // might be called before the torrent's piece completion + // handler is. + this.torrent.markCompleted(piece); + logger.debug("Completed download of {} from {}. " + + "We now have {}/{} pieces", + new Object[] { + piece, + peer, + this.torrent.getCompletedPieces().cardinality(), + this.torrent.getPieceCount() + }); + + // Send a HAVE message to all connected peers + PeerMessage have = PeerMessage.HaveMessage.craft(piece.getIndex()); + for (SharingPeer remote : this.connected.values()) { + remote.send(have); + } + + // Force notify after each piece is completed to propagate download + // completion information (or new seeding state) + this.setChanged(); + this.notifyObservers(this.state); + } else { + logger.warn("Downloaded piece#{} from {} was not valid ;-(", + piece.getIndex(), peer); + } + + if (this.torrent.isComplete()) { + logger.info("Last piece validated and completed, finishing download..."); + + // Cancel all remaining outstanding requests + for (SharingPeer remote : this.connected.values()) { + if (remote.isDownloading()) { + int requests = remote.cancelPendingRequests().size(); + logger.info("Cancelled {} remaining pending requests on {}.", + requests, remote); + } + } + + this.torrent.finish(); + + try { + this.announce.getCurrentTrackerClient() + .announce(TrackerMessage + .AnnounceRequestMessage + .RequestEvent.COMPLETED, true); + } catch (AnnounceException ae) { + logger.warn("Error announcing completion event to " + + "tracker: {}", ae.getMessage()); + } + + logger.info("Download is complete and finalized."); + this.seed(); + } + } + } + + @Override + public void handlePeerDisconnected(SharingPeer peer) { + if (this.connected.remove(peer.hasPeerId() + ? peer.getHexPeerId() + : peer.getHostIdentifier()) != null) { + logger.debug("Peer {} disconnected, [{}/{}].", + new Object[] { + peer, + this.connected.size(), + this.peers.size() + }); + } + + peer.reset(); + } + + @Override + public void handleIOException(SharingPeer peer, IOException ioe) { + logger.warn("I/O error while exchanging data with {}, " + + "closing connection with it!", peer, ioe.getMessage()); + peer.unbind(true); + } + + + /** Post download seeding. ************************************************/ + + /** + * Start the seeding period, if any. + * + *+ * This method is called when all the pieces of our torrent have been + * retrieved. This may happen immediately after the client starts if the + * torrent was already fully download or we are the initial seeder client. + *
+ * + *
+ * When the download is complete, the client switches to seeding mode for
+ * as long as requested in the share() call, if seeding was
+ * requested. If not, the {@link ClientShutdown} will execute
+ * immediately to stop the client's main loop.
+ *
+ * This TimerTask will be called by a timer set after the download is + * complete to stop seeding from this client after a certain amount of + * requested seed time (might be 0 for immediate termination). + *
+ * + *
+ * This task simply contains a reference to this client instance and calls
+ * its stop() method to interrupt the client's main loop.
+ *
+ * Every BitTorrent client, BitTorrent being a peer-to-peer protocol, listens + * on a httpPort for incoming connections from other peers sharing the same + * torrent. + *
+ * + *+ * This ConnectionHandler implements this service and starts a listening socket + * in the first available httpPort in the default BitTorrent client httpPort range + * 6881-6889. When a peer connects to it, it expects the BitTorrent handshake + * message, parses it and replies with our own handshake. + *
+ * + *+ * Outgoing connections to other peers are also made through this service, + * which handles the handshake procedure with the remote peer. Regardless of + * the direction of the connection, once this handshake is successful, all + * {@link IncomingConnectionListener}s are notified and passed the connected + * socket and the remote peer ID. + *
+ * + *+ * This class does nothing more. All further peer-to-peer communication happens + * in the {@link org.apache.hadoop.yarn.server.broadcast.client.peer.PeerExchange PeerExchange} + * class. + *
+ * + * @see BitTorrent handshake specification + */ +public class ConnectionHandler implements Runnable { + + private static final Logger logger = + LoggerFactory.getLogger(ConnectionHandler.class); + + public static final int PORT_RANGE_START = 30000; + public static final int PORT_RANGE_END = 40000; + + private static final int OUTBOUND_CONNECTIONS_POOL_SIZE = 20; + private static final int OUTBOUND_CONNECTIONS_THREAD_KEEP_ALIVE_SECS = 10; + + private static final int CLIENT_KEEP_ALIVE_MINUTES = 3; + + private SharedTorrent torrent; + private String id; + private ServerSocketChannel channel; + private InetSocketAddress address; + + private Set+ * This binds to the first available httpPort in the client httpPort range + * PORT_RANGE_START to PORT_RANGE_END. + *
+ * + * @param torrent The torrent shared by this client. + * @param id This client's peer ID. + * @param address The address to bind to. + * @throws IOException When the service can't be started because no httpPort in + * the defined range is available or usable. + */ + ConnectionHandler(SharedTorrent torrent, String id, InetAddress address) + throws IOException { + this.torrent = torrent; + this.id = id; + + // Bind to the first available httpPort in the range + // [PORT_RANGE_START; PORT_RANGE_END]. + for (int port = ConnectionHandler.PORT_RANGE_START; + port <= ConnectionHandler.PORT_RANGE_END; + port++) { + InetSocketAddress tryAddress = + new InetSocketAddress(address, port); + + try { + this.channel = ServerSocketChannel.open(); + this.channel.socket().bind(tryAddress); + this.channel.configureBlocking(false); + this.address = tryAddress; + break; + } catch (IOException ioe) { + // Ignore, try next httpPort + logger.warn("Could not bind to {}, trying next httpPort...", tryAddress); + } + } + + if (this.channel == null || !this.channel.socket().isBound()) { + throw new IOException("No available httpPort for the BitTorrent client!"); + } + + logger.info("Listening for incoming connections on {}.", this.address); + + this.listeners = new HashSet+ * Note: the underlying socket remains open and bound. + *
+ */ + public void stop() { + this.stop = true; + + if (this.thread != null && this.thread.isAlive()) { + try { + this.thread.join(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + } + + if (this.executor != null && !this.executor.isShutdown()) { + this.executor.shutdownNow(); + } + + this.executor = null; + this.thread = null; + } + + /** + * Close this connection handler to release the httpPort it is bound to. + * + * @throws IOException If the channel could not be closed. + */ + public void close() throws IOException { + if (this.channel != null) { + this.channel.close(); + this.channel = null; + } + } + + /** + * The main service loop. + * + *+ * The service waits for new connections for 250ms, then waits 100ms so it + * can be interrupted. + *
+ */ + @Override + public void run() { + while (!this.stop) { + try { + SocketChannel client = this.channel.accept(); + if (client != null) { + this.accept(client); + } + } catch (SocketTimeoutException ste) { + // Ignore and go back to sleep + } catch (IOException ioe) { + logger.warn("Unrecoverable error in connection handler", ioe); + this.stop(); + } + + try { + Thread.sleep(100); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * Return a human-readable representation of a connected socket channel. + * + * @param channel The socket channel to represent. + * @return A textual representation (host:httpPort) of the given + * socket. + */ + private String socketRepr(SocketChannel channel) { + Socket s = channel.socket(); + return String.format("%s:%d%s", + s.getInetAddress().getHostName(), + s.getPort(), + channel.isConnected() ? "+" : "-"); + } + + /** + * Accept the next incoming connection. + * + *+ * When a new peer connects to this service, wait for it to send its + * handshake. We then parse and check that the handshake advertises the + * torrent hash we expect, then reply with our own handshake. + *
+ * + *
+ * If everything goes according to plan, notify the
+ * IncomingConnectionListeners with the connected socket and
+ * the parsed peer ID.
+ *
+ * Submits an asynchronous connection task to the outbound connections + * executor to connect to the given peer. + *
+ * + * @param peer The peer to connect to. + */ + public void connect(SharingPeer peer) { + if (!this.isAlive()) { + throw new IllegalStateException( + "Connection handler is not accepting new peers at this time!"); + } + + this.executor.submit(new ConnectorTask(this, peer)); + } + + /** + * Validate an expected handshake on a connection. + * + *+ * Reads an expected handshake message from the given connected socket, + * parses it and validates that the torrent hash_info corresponds to the + * torrent we're sharing, and that the peerId matches the peer ID we expect + * to see coming from the remote peer. + *
+ * + * @param channel The connected socket channel to the remote peer. + * @param peerId The peer ID we expect in the handshake. If null, + * any peer ID is accepted (this is the case for incoming connections). + * @return The validated handshake message object. + */ + private Handshake validateHandshake(SocketChannel channel, byte[] peerId) + throws IOException, ParseException { + ByteBuffer len = ByteBuffer.allocate(1); + ByteBuffer data; + + // Read the handshake from the wire + logger.trace("Reading handshake size (1 byte) from {}...", this.socketRepr(channel)); + if (channel.read(len) < len.capacity()) { + throw new IOException("Handshake size read underrrun"); + } + + len.rewind(); + int pstrlen = len.get(); + + data = ByteBuffer.allocate(Handshake.BASE_HANDSHAKE_LENGTH + pstrlen); + data.put((byte)pstrlen); + int expected = data.remaining(); + int read = channel.read(data); + if (read < expected) { + throw new IOException("Handshake data read underrun (" + + read + " < " + expected + " bytes)"); + } + + // Parse and check the handshake + data.rewind(); + Handshake hs = Handshake.parse(data); + if (!Arrays.equals(hs.getInfoHash(), this.torrent.getInfoHash())) { + throw new ParseException("Handshake for unknow torrent " + + Torrent.byteArrayToHexString(hs.getInfoHash()) + + " from " + this.socketRepr(channel) + ".", pstrlen + 9); + } + + if (peerId != null && !Arrays.equals(hs.getPeerId(), peerId)) { + throw new ParseException("Announced peer ID " + + Torrent.byteArrayToHexString(hs.getPeerId()) + + " did not match expected peer ID " + + Torrent.byteArrayToHexString(peerId) + ".", pstrlen + 29); + } + + return hs; + } + + /** + * Send our handshake message to the socket. + * + * @param channel The socket channel to the remote peer. + */ + private int sendHandshake(SocketChannel channel) throws IOException { + return channel.write( + Handshake.craft( + this.torrent.getInfoHash(), + this.id.getBytes(Torrent.BYTE_ENCODING)).getData()); + } + + /** + * Trigger the new peer connection event on all registered listeners. + * + * @param channel The socket channel to the newly connected peer. + * @param peerId The peer ID of the connected peer. + */ + private void fireNewPeerConnection(SocketChannel channel, byte[] peerId) { + for (IncomingConnectionListener listener : this.listeners) { + listener.handleNewPeerConnection(channel, peerId); + } + } + + private void fireFailedConnection(SharingPeer peer, Throwable cause) { + for (IncomingConnectionListener listener : this.listeners) { + listener.handleFailedConnection(peer, cause); + } + } + + + /** + * A simple thread factory that returns appropriately named threads for + * outbound connector threads. + * + * @author mpetazzoni + */ + private static class ConnectorThreadFactory implements ThreadFactory { + + private int number = 0; + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setName("bt-connect-" + ++this.number); + return t; + } + } + + + /** + * An outbound connection task. + * + *+ * These tasks are fed to the thread executor in charge of processing + * outbound connection requests. It attempts to connect to the given peer + * and proceeds with the BitTorrent handshake. If the handshake is + * successful, the new peer connection event is fired to all incoming + * connection listeners. Otherwise, the failed connection event is fired. + *
+ * + * @author mpetazzoni + */ + private static class ConnectorTask implements Runnable { + + private final ConnectionHandler handler; + private final SharingPeer peer; + + private ConnectorTask(ConnectionHandler handler, SharingPeer peer) { + this.handler = handler; + this.peer = peer; + } + + @Override + public void run() { + InetSocketAddress address = + new InetSocketAddress(this.peer.getIp(), this.peer.getPort()); + SocketChannel channel = null; + + try { + logger.info("Connecting to {}...", this.peer); + channel = SocketChannel.open(address); + while (!channel.isConnected()) { + Thread.sleep(10); + } + + logger.debug("Connected. Sending handshake to {}...", this.peer); + channel.configureBlocking(true); + int sent = this.handler.sendHandshake(channel); + logger.debug("Sent handshake ({} bytes), waiting for response...", sent); + Handshake hs = this.handler.validateHandshake(channel, + (this.peer.hasPeerId() + ? this.peer.getPeerId().array() + : null)); + logger.info("Handshaked with {}, peer ID is {}.", + this.peer, Torrent.byteArrayToHexString(hs.getPeerId())); + + // Go to non-blocking mode for peer interaction + channel.configureBlocking(false); + this.handler.fireNewPeerConnection(channel, hs.getPeerId()); + } catch (Exception e) { + if (channel != null && channel.isConnected()) { + IOUtils.closeQuietly(channel); + } + this.handler.fireFailedConnection(this.peer, e); + } + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Handshake.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Handshake.java new file mode 100644 index 0000000..018cc5e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Handshake.java @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.text.ParseException; + + +/** + * Peer handshake handler. + * + */ +public class Handshake { + + public static final String BITTORRENT_PROTOCOL_IDENTIFIER = "BitTorrent protocol"; + public static final int BASE_HANDSHAKE_LENGTH = 49; + + ByteBuffer data; + ByteBuffer infoHash; + ByteBuffer peerId; + + private Handshake(ByteBuffer data, ByteBuffer infoHash, + ByteBuffer peerId) { + this.data = data; + this.data.rewind(); + + this.infoHash = infoHash; + this.peerId = peerId; + } + + public ByteBuffer getData() { + return this.data; + } + + public byte[] getInfoHash() { + return this.infoHash.array(); + } + + public byte[] getPeerId() { + return this.peerId.array(); + } + + public static Handshake parse(ByteBuffer buffer) + throws ParseException, UnsupportedEncodingException { + int pstrlen = Byte.valueOf(buffer.get()).intValue(); + if (pstrlen < 0 || + buffer.remaining() != BASE_HANDSHAKE_LENGTH + pstrlen - 1) { + throw new ParseException("Incorrect handshake message length " + + "(pstrlen=" + pstrlen + ") !", 0); + } + + // Check the protocol identification string + byte[] pstr = new byte[pstrlen]; + buffer.get(pstr); + + if (!Handshake.BITTORRENT_PROTOCOL_IDENTIFIER.equals( + new String(pstr, Torrent.BYTE_ENCODING))) { + throw new ParseException("Invalid protocol identifier!", 1); + } + + // Ignore reserved bytes + byte[] reserved = new byte[8]; + buffer.get(reserved); + + byte[] infoHash = new byte[20]; + buffer.get(infoHash); + byte[] peerId = new byte[20]; + buffer.get(peerId); + return new Handshake(buffer, ByteBuffer.wrap(infoHash), + ByteBuffer.wrap(peerId)); + } + + public static Handshake craft(byte[] torrentInfoHash, + byte[] clientPeerId) { + try { + ByteBuffer buffer = ByteBuffer.allocate( + Handshake.BASE_HANDSHAKE_LENGTH + + Handshake.BITTORRENT_PROTOCOL_IDENTIFIER.length()); + + byte[] reserved = new byte[8]; + ByteBuffer infoHash = ByteBuffer.wrap(torrentInfoHash); + ByteBuffer peerId = ByteBuffer.wrap(clientPeerId); + + buffer.put((byte)Handshake + .BITTORRENT_PROTOCOL_IDENTIFIER.length()); + buffer.put(Handshake + .BITTORRENT_PROTOCOL_IDENTIFIER.getBytes(Torrent.BYTE_ENCODING)); + buffer.put(reserved); + buffer.put(infoHash); + buffer.put(peerId); + + return new Handshake(buffer, infoHash, peerId); + } catch (UnsupportedEncodingException uee) { + return null; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/IncomingConnectionListener.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/IncomingConnectionListener.java new file mode 100644 index 0000000..71caa44 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/IncomingConnectionListener.java @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer; + +import java.nio.channels.SocketChannel; +import java.util.EventListener; + +/** + * EventListener interface for objects that want to handle incoming peer + * connections. + * + */ +public interface IncomingConnectionListener extends EventListener { + + public void handleNewPeerConnection(SocketChannel channel, byte[] peerId); + + public void handleFailedConnection(SharingPeer peer, Throwable cause); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Piece.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Piece.java new file mode 100644 index 0000000..c2dec3a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/Piece.java @@ -0,0 +1,312 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client; + +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.client.peer.SharingPeer; +import org.apache.hadoop.yarn.server.broadcast.client.storage.TorrentByteStorage; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.concurrent.Callable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * A torrent piece. + * + *+ * This class represents a torrent piece. Torrents are made of pieces, which + * are in turn made of blocks that are exchanged using the peer protocol. + * The piece length is defined at the torrent level, but the last piece that + * makes the torrent might be smaller. + *
+ * + *+ * If the torrent has multiple files, pieces can spread across file boundaries. + * The TorrentByteStorage abstracts this problem to give Piece objects the + * impression of a contiguous, linear byte storage. + *
+ * + */ +public class Piece implements Comparable+ * All pieces, except the last one, are expected to have the same size. + *
+ */ + public long size() { + return this.length; + } + + /** + * Tells whether this piece is available in the current connected peer swarm. + */ + public boolean available() { + return this.seen > 0; + } + + /** + * Mark this piece as being seen at the given peer. + * + * @param peer The sharing peer this piece has been seen available at. + */ + public void seenAt(SharingPeer peer) { + this.seen++; + } + + /** + * Mark this piece as no longer being available at the given peer. + * + * @param peer The sharing peer from which the piece is no longer available. + */ + public void noLongerAt(SharingPeer peer) { + this.seen--; + } + + /** + * Validates this piece. + * + * @return Returns true if this piece, as stored in the underlying byte + * storage, is valid, i.e. its SHA1 sum matches the one from the torrent + * meta-info. + */ + public synchronized boolean validate() throws IOException { + if (this.seeder) { + logger.trace("Skipping validation of {} (seeder mode).", this); + this.valid = true; + return true; + } + + logger.trace("Validating {}...", this); + this.valid = false; + + ByteBuffer buffer = this._read(0, this.length); + byte[] data = new byte[(int)this.length]; + buffer.get(data); + this.valid = Arrays.equals(Torrent.hash(data), this.hash); + + return this.isValid(); + } + + /** + * Internal piece data read function. + * + *+ * This function will read the piece data without checking if the piece has + * been validated. It is simply meant at factoring-in the common read code + * from the validate and read functions. + *
+ * + * @param offset Offset inside this piece where to start reading. + * @param length Number of bytes to read from the piece. + * @return A byte buffer containing the piece data. + * @throws IllegalArgumentException If offset + length goes over + * the piece boundary. + * @throws IOException If the read can't be completed (I/O error, or EOF + * reached, which can happen if the piece is not complete). + */ + private ByteBuffer _read(long offset, long length) throws IOException { + if (offset + length > this.length) { + throw new IllegalArgumentException("Piece#" + this.index + + " overrun (" + offset + " + " + length + " > " + + this.length + ") !"); + } + + // TODO: remove cast to int when large ByteBuffer support is + // implemented in Java. + ByteBuffer buffer = ByteBuffer.allocate((int)length); + int bytes = this.bucket.read(buffer, this.offset + offset); + buffer.rewind(); + buffer.limit(bytes >= 0 ? bytes : 0); + return buffer; + } + + /** + * Read a piece block from the underlying byte storage. + * + *+ * This is the public method for reading this piece's data, and it will + * only succeed if the piece is complete and valid on disk, thus ensuring + * any data that comes out of this function is valid piece data we can send + * to other peers. + *
+ * + * @param offset Offset inside this piece where to start reading. + * @param length Number of bytes to read from the piece. + * @return A byte buffer containing the piece data. + * @throws IllegalArgumentException If offset + length goes over + * the piece boundary. + * @throws IllegalStateException If the piece is not valid when attempting + * to read it. + * @throws IOException If the read can't be completed (I/O error, or EOF + * reached, which can happen if the piece is not complete). + */ + public ByteBuffer read(long offset, int length) + throws IllegalArgumentException, IllegalStateException, IOException { + if (!this.valid) { + throw new IllegalStateException("Attempting to read an " + + "known-to-be invalid piece!"); + } + + return this._read(offset, length); + } + + /** + * Record the given block at the given offset in this piece. + * + *+ * Note: this has synchronized access to the underlying byte storage. + *
+ * + * @param block The ByteBuffer containing the block data. + * @param offset The block offset in this piece. + */ + public synchronized void record(ByteBuffer block, int offset) + throws IOException { + if (this.data == null || offset == 0) { + // TODO: remove cast to int when large ByteBuffer support is + // implemented in Java. + this.data = ByteBuffer.allocate((int)this.length); + } + + int pos = block.position(); + this.data.position(offset); + this.data.put(block); + block.position(pos); + + if (block.remaining() + offset == this.length) { + this.data.rewind(); + logger.trace("Recording {}...", this); + this.bucket.write(this.data, this.offset); + this.data = null; + } + } + + /** + * Return a human-readable representation of this piece. + */ + public String toString() { + return String.format("piece#%4d%s", + this.index, + this.isValid() ? "+" : "-"); + } + + /** + * Piece comparison function for ordering pieces based on their + * availability. + * + * @param other The piece to compare with, should not be null. + */ + public int compareTo(Piece other) { + if (this.seen != other.seen) { + return this.seen < other.seen ? -1 : 1; + } + return this.index == other.index ? 0 : + (this.index < other.index ? -1 : 1); + } + + /** + * A {@link Callable} to call the piece validation function. + * + *+ * This {@link Callable} implementation allows for the calling of the piece + * validation function in a controlled context like a thread or an + * executor. It returns the piece it was created for. Results of the + * validation can easily be extracted from the {@link Piece} object after + * it is returned. + *
+ * + * @author mpetazzoni + */ + public static class CallableHasher implements Callable+ * The {@link SharedTorrent} class extends the Torrent class with all the data + * and logic required by the BitTorrent client implementation. + *
+ * + *+ * Note: this implementation currently only supports single-file + * torrents. + *
+ * + */ +public class SharedTorrent extends Torrent implements PeerActivityListener { + + private static final Logger logger = + LoggerFactory.getLogger(SharedTorrent.class); + + /** End-game trigger ratio. + * + *+ * Eng-game behavior (requesting already requested pieces from available + * and ready peers to try to speed-up the end of the transfer) will only be + * enabled when the ratio of completed pieces over total pieces in the + * torrent is over this value. + *
+ */ + private static final float ENG_GAME_COMPLETION_RATIO = 0.95f; + + /** Default Request Strategy. + * + * Use the rarest-first strategy by default. + */ + private static final RequestStrategy DEFAULT_REQUEST_STRATEGY = new RequestStrategyImplRarest(); + + private boolean stop; + + private long uploaded; + private long downloaded; + private long left; + + private final TorrentByteStorage bucket; + + private final int pieceLength; + private final ByteBuffer piecesHashes; + + private boolean initialized; + private Piece[] pieces; + private SortedSet+ * This will recreate a SharedTorrent object from the provided Torrent + * object's encoded meta-info data. + *
+ * + * @param torrent The Torrent object. + * @param destDir The destination directory or location of the torrent + * files. + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(Torrent torrent, File destDir) + throws FileNotFoundException, IOException { + this(torrent, destDir, false); + } + + /** + * Create a new shared torrent from a base Torrent object. + * + *+ * This will recreate a SharedTorrent object from the provided Torrent + * object's encoded meta-info data. + *
+ * + * @param torrent The Torrent object. + * @param destDir The destination directory or location of the torrent + * files. + * @param seeder Whether we're a seeder for this torrent or not (disables + * validation). + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(Torrent torrent, File destDir, boolean seeder) + throws FileNotFoundException, IOException { + this(torrent.getEncoded(), destDir, seeder, DEFAULT_REQUEST_STRATEGY); + } + + /** + * Create a new shared torrent from a base Torrent object. + * + *+ * This will recreate a SharedTorrent object from the provided Torrent + * object's encoded meta-info data. + *
+ * + * @param torrent The Torrent object. + * @param destDir The destination directory or location of the torrent + * files. + * @param seeder Whether we're a seeder for this torrent or not (disables + * validation). + * @param requestStrategy The request strategy implementation. + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(Torrent torrent, File destDir, boolean seeder, + RequestStrategy requestStrategy) + throws FileNotFoundException, IOException { + this(torrent.getEncoded(), destDir, seeder, requestStrategy); + } + + /** + * Create a new shared torrent from meta-info binary data. + * + * @param torrent The meta-info byte data. + * @param destDir The destination directory or location of the torrent + * files. + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(byte[] torrent, File destDir) + throws FileNotFoundException, IOException { + this(torrent, destDir, false); + } + + /** + * Create a new shared torrent from meta-info binary data. + * + * @param torrent The meta-info byte data. + * @param parent The parent directory or location the torrent files. + * @param seeder Whether we're a seeder for this torrent or not (disables + * validation). + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(byte[] torrent, File parent, boolean seeder) + throws FileNotFoundException, IOException { + this(torrent, parent, seeder, DEFAULT_REQUEST_STRATEGY); + } + + /** + * Create a new shared torrent from meta-info binary data. + * + * @param torrent The meta-info byte data. + * @param parent The parent directory or location the torrent files. + * @param seeder Whether we're a seeder for this torrent or not (disables + * validation). + * @param requestStrategy The request strategy implementation. + * @throws FileNotFoundException If the torrent file location or + * destination directory does not exist and can't be created. + * @throws IOException If the torrent file cannot be read or decoded. + */ + public SharedTorrent(byte[] torrent, File parent, boolean seeder, + RequestStrategy requestStrategy) + throws FileNotFoundException, IOException { + super(torrent, seeder); + + if (parent == null || !parent.isDirectory()) { + throw new IllegalArgumentException("Invalid parent directory!"); + } + + String parentPath = parent.getCanonicalPath(); + + try { + this.pieceLength = this.decoded_info.get("piece length").getInt(); + this.piecesHashes = ByteBuffer.wrap(this.decoded_info.get("pieces") + .getBytes()); + + if (this.piecesHashes.capacity() / Torrent.PIECE_HASH_SIZE * + (long)this.pieceLength < this.getSize()) { + throw new IllegalArgumentException("Torrent size does not " + + "match the number of pieces and the piece size!"); + } + } catch (InvalidBEncodingException ibee) { + throw new IllegalArgumentException( + "Error reading torrent meta-info fields!"); + } + + List.torrent file to read the torrent
+ * meta-info from.
+ * @param parent The parent directory or location of the torrent files.
+ * @throws IOException When the torrent file cannot be read or decoded.
+ */
+ public static SharedTorrent fromFile(File source, File parent)
+ throws IOException {
+ byte[] data = FileUtils.readFileToByteArray(source);
+ return new SharedTorrent(data, parent);
+ }
+
+ public double getMaxUploadRate() {
+ return this.maxUploadRate;
+ }
+
+ /**
+ * Set the maximum upload rate (in kB/second) for this
+ * torrent. A setting of <= 0.0 disables rate limiting.
+ *
+ * @param rate The maximum upload rate
+ */
+ public void setMaxUploadRate(double rate) {
+ this.maxUploadRate = rate;
+ }
+
+ public double getMaxDownloadRate() {
+ return this.maxDownloadRate;
+ }
+
+ /**
+ * Set the maximum download rate (in kB/second) for this
+ * torrent. A setting of <= 0.0 disables rate limiting.
+ *
+ * @param rate The maximum download rate
+ */
+ public void setMaxDownloadRate(double rate) {
+ this.maxDownloadRate = rate;
+ }
+
+ /**
+ * Get the number of bytes uploaded for this torrent.
+ */
+ public long getUploaded() {
+ return this.uploaded;
+ }
+
+ /**
+ * Get the number of bytes downloaded for this torrent.
+ *
+ * + * Note: this could be more than the torrent's length, and should + * not be used to determine a completion percentage. + *
+ */ + public long getDownloaded() { + return this.downloaded; + } + + /** + * Get the number of bytes left to download for this torrent. + */ + public long getLeft() { + return this.left; + } + + /** + * Tells whether this torrent has been fully initialized yet. + */ + public boolean isInitialized() { + return this.initialized; + } + + /** + * Stop the torrent initialization as soon as possible. + */ + public void stop() { + this.stop = true; + } + + /** + * Build this torrent's pieces array. + * + *+ * Hash and verify any potentially present local data and create this + * torrent's pieces array from their respective hash provided in the + * torrent meta-info. + *
+ * + *+ * This function should be called soon after the constructor to initialize + * the pieces array. + *
+ */ + public synchronized void init() throws InterruptedException, IOException { + if (this.isInitialized()) { + throw new IllegalStateException("Torrent was already initialized!"); + } + + int threads = getHashingThreadsCount(); + int nPieces = (int) (Math.ceil( + (double)this.getSize() / this.pieceLength)); + int step = 10; + + this.pieces = new Piece[nPieces]; + this.completedPieces = new BitSet(nPieces); + this.piecesHashes.clear(); + + ExecutorService executor = Executors.newFixedThreadPool(threads); + List+ * Available pieces are pieces available in the swarm, and it does not + * include our own pieces. + *
+ */ + public BitSet getAvailablePieces() { + if (!this.isInitialized()) { + throw new IllegalStateException("Torrent not yet initialized!"); + } + + BitSet availablePieces = new BitSet(this.pieces.length); + + synchronized (this.pieces) { + for (Piece piece : this.pieces) { + if (piece.available()) { + availablePieces.set(piece.getIndex()); + } + } + } + + return availablePieces; + } + + /** + * Return a copy of the completed pieces bitset. + */ + public BitSet getCompletedPieces() { + if (!this.isInitialized()) { + throw new IllegalStateException("Torrent not yet initialized!"); + } + + synchronized (this.completedPieces) { + return (BitSet)this.completedPieces.clone(); + } + } + + /** + * Return a copy of the requested pieces bitset. + */ + public BitSet getRequestedPieces() { + if (!this.isInitialized()) { + throw new IllegalStateException("Torrent not yet initialized!"); + } + + synchronized (this.requestedPieces) { + return (BitSet)this.requestedPieces.clone(); + } + } + + /** + * Tells whether this torrent has been fully downloaded, or is fully + * available locally. + */ + public synchronized boolean isComplete() { + return this.pieces.length > 0 && + this.completedPieces.cardinality() == this.pieces.length; + } + + /** + * Finalize the download of this torrent. + * + *+ * This realizes the final, pre-seeding phase actions on this torrent, + * which usually consists in putting the torrent data in their final form + * and at their target location. + *
+ * + * @see TorrentByteStorage#finish + */ + public synchronized void finish() throws IOException { + if (!this.isInitialized()) { + throw new IllegalStateException("Torrent not yet initialized!"); + } + + if (!this.isComplete()) { + throw new IllegalStateException("Torrent download is not complete!"); + } + + this.bucket.finish(); + } + + public synchronized boolean isFinished() { + return this.isComplete() && this.bucket.isFinished(); + } + + /** + * Return the completion percentage of this torrent. + * + *+ * This is computed from the number of completed pieces divided by the + * number of pieces in this torrent, times 100. + *
+ */ + public float getCompletion() { + return this.isInitialized() + ? (float)this.completedPieces.cardinality() / + (float)this.pieces.length * 100.0f + : 0.0f; + } + + /** + * Mark a piece as completed, decrementing the piece size in bytes from our + * left bytes to download counter. + */ + public synchronized void markCompleted(Piece piece) { + if (this.completedPieces.get(piece.getIndex())) { + return; + } + + // A completed piece means that's that much data left to download for + // this torrent. + this.left -= piece.size(); + this.completedPieces.set(piece.getIndex()); + } + + /** PeerActivityListener handler(s). *************************************/ + + /** + * Peer choked handler. + * + *+ * When a peer chokes, the requests made to it are canceled and we need to + * mark the eventually piece we requested from it as available again for + * download tentative from another peer. + *
+ * + * @param peer The peer that choked. + */ + @Override + public synchronized void handlePeerChoked(SharingPeer peer) { + Piece piece = peer.getRequestedPiece(); + + if (piece != null) { + this.requestedPieces.set(piece.getIndex(), false); + } + + logger.trace("Peer {} choked, we now have {} outstanding " + + "request(s): {}", + new Object[] { + peer, + this.requestedPieces.cardinality(), + this.requestedPieces + }); + } + + /** + * Peer ready handler. + * + *+ * When a peer becomes ready to accept piece block requests, select a piece + * to download and go for it. + *
+ * + * @param peer The peer that became ready. + */ + @Override + public synchronized void handlePeerReady(SharingPeer peer) { + BitSet interesting = peer.getAvailablePieces(); + interesting.andNot(this.completedPieces); + interesting.andNot(this.requestedPieces); + + logger.trace("Peer {} is ready and has {} interesting piece(s).", + peer, interesting.cardinality()); + + // If we didn't find interesting pieces, we need to check if we're in + // an end-game situation. If yes, we request an already requested piece + // to try to speed up the end. + if (interesting.cardinality() == 0) { + interesting = peer.getAvailablePieces(); + interesting.andNot(this.completedPieces); + if (interesting.cardinality() == 0) { + logger.trace("No interesting piece from {}!", peer); + return; + } + + if (this.completedPieces.cardinality() < + ENG_GAME_COMPLETION_RATIO * this.pieces.length) { + logger.trace("Not far along enough to warrant end-game mode."); + return; + } + + logger.trace("Possible end-game, we're about to request a piece " + + "that was already requested from another peer."); + } + + Piece chosen = requestStrategy.choosePiece(rarest, interesting, pieces); + this.requestedPieces.set(chosen.getIndex()); + + logger.trace("Requesting {} from {}, we now have {} " + + "outstanding request(s): {}", + new Object[] { + chosen, + peer, + this.requestedPieces.cardinality(), + this.requestedPieces + }); + + peer.downloadPiece(chosen); + } + + /** + * Piece availability handler. + * + *+ * Handle updates in piece availability from a peer's HAVE message. When + * this happens, we need to mark that piece as available from the peer. + *
+ * + * @param peer The peer we got the update from. + * @param piece The piece that became available. + */ + @Override + public synchronized void handlePieceAvailability(SharingPeer peer, + Piece piece) { + // If we don't have this piece, tell the peer we're interested in + // getting it from him. + if (!this.completedPieces.get(piece.getIndex()) && + !this.requestedPieces.get(piece.getIndex())) { + peer.interesting(); + } + + this.rarest.remove(piece); + piece.seenAt(peer); + this.rarest.add(piece); + + logger.trace("Peer {} contributes {} piece(s) [{}/{}/{}].", + new Object[] { + peer, + peer.getAvailablePieces().cardinality(), + this.completedPieces.cardinality(), + this.getAvailablePieces().cardinality(), + this.pieces.length + }); + + if (!peer.isChoked() && + peer.isInteresting() && + !peer.isDownloading()) { + this.handlePeerReady(peer); + } + } + + /** + * Bit field availability handler. + * + *+ * Handle updates in piece availability from a peer's BITFIELD message. + * When this happens, we need to mark in all the pieces the peer has that + * they can be reached through this peer, thus augmenting the global + * availability of pieces. + *
+ * + * @param peer The peer we got the update from. + * @param availablePieces The pieces availability bit field of the peer. + */ + @Override + public synchronized void handleBitfieldAvailability(SharingPeer peer, + BitSet availablePieces) { + // Determine if the peer is interesting for us or not, and notify it. + BitSet interesting = (BitSet)availablePieces.clone(); + interesting.andNot(this.completedPieces); + interesting.andNot(this.requestedPieces); + + if (interesting.cardinality() == 0) { + peer.notInteresting(); + } else { + peer.interesting(); + } + + // Record that the peer has all the pieces it told us it had. + for (int i = availablePieces.nextSetBit(0); i >= 0; + i = availablePieces.nextSetBit(i+1)) { + this.rarest.remove(this.pieces[i]); + this.pieces[i].seenAt(peer); + this.rarest.add(this.pieces[i]); + } + + logger.trace("Peer {} contributes {} piece(s) ({} interesting) " + + "[completed={}; available={}/{}].", + new Object[] { + peer, + availablePieces.cardinality(), + interesting.cardinality(), + this.completedPieces.cardinality(), + this.getAvailablePieces().cardinality(), + this.pieces.length + }); + } + + /** + * Piece upload completion handler. + * + *
+ * When a piece has been sent to a peer, we just record that we sent that
+ * many bytes. If the piece is valid on the peer's side, it will send us a
+ * HAVE message and we'll record that the piece is available on the peer at
+ * that moment (see handlePieceAvailability()).
+ *
+ * If the complete piece downloaded is valid, we can record in the torrent + * completedPieces bit field that we know have this piece. + *
+ * + * @param peer The peer we got this piece from. + * @param piece The piece in question. + */ + @Override + public synchronized void handlePieceCompleted(SharingPeer peer, + Piece piece) throws IOException { + // Regardless of validity, record the number of bytes downloaded and + // mark the piece as not requested anymore + this.downloaded += piece.size(); + this.requestedPieces.set(piece.getIndex(), false); + + logger.trace("We now have {} piece(s) and {} outstanding request(s): {}", + new Object[] { + this.completedPieces.cardinality(), + this.requestedPieces.cardinality(), + this.requestedPieces + }); + } + + /** + * Peer disconnection handler. + * + *+ * When a peer disconnects, we need to mark in all of the pieces it had + * available that they can't be reached through this peer anymore. + *
+ * + * @param peer The peer we got this piece from. + */ + @Override + public synchronized void handlePeerDisconnected(SharingPeer peer) { + BitSet availablePieces = peer.getAvailablePieces(); + + for (int i = availablePieces.nextSetBit(0); i >= 0; + i = availablePieces.nextSetBit(i+1)) { + this.rarest.remove(this.pieces[i]); + this.pieces[i].noLongerAt(peer); + this.rarest.add(this.pieces[i]); + } + + Piece requested = peer.getRequestedPiece(); + if (requested != null) { + this.requestedPieces.set(requested.getIndex(), false); + } + + logger.debug("Peer {} went away with {} piece(s) [completed={}; available={}/{}]", + new Object[] { + peer, + availablePieces.cardinality(), + this.completedPieces.cardinality(), + this.getAvailablePieces().cardinality(), + this.pieces.length + }); + logger.trace("We now have {} piece(s) and {} outstanding request(s): {}", + new Object[] { + this.completedPieces.cardinality(), + this.requestedPieces.cardinality(), + this.requestedPieces + }); + } + + @Override + public synchronized void handleIOException(SharingPeer peer, + IOException ioe) { /* Do nothing */ } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/Announce.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/Announce.java new file mode 100644 index 0000000..66c5918 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/Announce.java @@ -0,0 +1,223 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceRequestMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * BitTorrent announce sub-system. + * + *+ * A BitTorrent client must check-in to the torrent's tracker(s) to get peers + * and to report certain events. + *
+ * + *+ * This Announce class implements a periodic announce request thread that will + * notify announce request event listeners for each tracker response. + *
+ * + * @see org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage + */ +public class Announce implements Runnable { + + protected static final Logger logger = + LoggerFactory.getLogger(Announce.class); + + private final Peer peer; + + private final TrackerClient client; + + /** Announce thread and control. */ + private Thread thread; + private boolean stop; + private boolean forceStop; + + /** Announce interval. */ + private int interval; + + private int currentTier; + private int currentClient; + + /** + * Initialize the base announce class members for the announcer. + * + * @param torrent The torrent we're announcing about. + * @param peer Our peer specification. + */ + public Announce(SharedTorrent torrent, Peer peer) { + this.peer = peer; + this.client = new TrackerClient(torrent, peer); + this.thread = null; + this.currentTier = 0; + this.currentClient = 0; + + logger.info("Initialized announce sub-system with 1 trackers on {}.", new Object[] { torrent }); + } + + /** + * Register a new announce response listener. + * + * @param listener The listener to register on this announcer events. + */ + public void register(AnnounceResponseListener listener) { + client.register(listener); + } + + /** + * Start the announce request thread. + */ + public void start() { + this.stop = false; + this.forceStop = false; + + if (this.thread == null || !this.thread.isAlive()) { + this.thread = new Thread(this); + this.thread.setName("bt-announce(" + + this.peer.getShortHexPeerId() + ")"); + this.thread.start(); + } + } + + /** + * Set the announce interval. + */ + public void setInterval(int interval) { + if (interval <= 0) { + this.stop(true); + return; + } + + if (this.interval == interval) { + return; + } + + logger.info("Setting announce interval to {}s per tracker request.", + interval); + this.interval = interval; + } + + /** + * Stop the announce thread. + * + *+ * One last 'stopped' announce event might be sent to the tracker to + * announce we're going away, depending on the implementation. + *
+ */ + public void stop() { + this.stop = true; + + if (this.thread != null && this.thread.isAlive()) { + this.thread.interrupt(); + this.client.close(); + + try { + this.thread.join(); + } catch (InterruptedException ie) { + // Ignore + } + } + + this.thread = null; + } + + /** + * Main announce loop. + * + *+ * The announce thread starts by making the initial 'started' announce + * request to register on the tracker and get the announce interval value. + * Subsequent announce requests are ordinary, event-less, periodic requests + * for peers. + *
+ * + *+ * Unless forcefully stopped, the announce thread will terminate by sending + * a 'stopped' announce request before stopping. + *
+ */ + @Override + public void run() { + logger.info("Starting announce loop..."); + + // Set an initial announce interval to 5 seconds. This will be updated + // in real-time by the tracker's responses to our announce requests. + this.interval = 5; + + AnnounceRequestMessage.RequestEvent event = + AnnounceRequestMessage.RequestEvent.STARTED; + + while (!this.stop) { + try { + this.getCurrentTrackerClient().announce(event, false); + event = AnnounceRequestMessage.RequestEvent.NONE; + } catch (AnnounceException ae) { + logger.info(ae.getMessage()); + logger.info("will re-announce with message " + event.getEventName()); + } + + try { + Thread.sleep(this.interval * 1000); + } catch (InterruptedException ie) { + // Ignore + } + } + + logger.info("Exited announce loop."); + + if (!this.forceStop) { + // Send the final 'stopped' event to the tracker after a little + // while. + event = AnnounceRequestMessage.RequestEvent.STOPPED; + try { + Thread.sleep(500); + } catch (InterruptedException ie) { + // Ignore + } + + try { + this.getCurrentTrackerClient().announce(event, true); + } catch (AnnounceException ae) { + logger.warn(ae.getMessage()); + } + } + } + + /** + * Returns the current tracker client used for announces. + * @throws AnnounceException When the current announce tier isn't defined + * in the torrent. + */ + public TrackerClient getCurrentTrackerClient() throws AnnounceException { + return this.client; + } + + /** + * Stop the announce thread. + * + * @param hard Whether to force stop the announce thread or not, i.e. not + * send the final 'stopped' announce request or not. + */ + private void stop(boolean hard) { + this.forceStop = hard; + this.stop(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceException.java new file mode 100644 index 0000000..7e28029 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceException.java @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + + +/** + * Exception thrown when an announce request failed. + * + */ +public class AnnounceException extends Exception { + + private static final long serialVersionUID = -1; + + public AnnounceException(String message) { + super(message); + } + + public AnnounceException(Throwable cause) { + super(cause); + } + + public AnnounceException(String message, Throwable cause) { + super(message, cause); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceResponseListener.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceResponseListener.java new file mode 100644 index 0000000..5bfcd7c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/AnnounceResponseListener.java @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.hadoop.yarn.server.broadcast.common.Peer; + +import java.util.EventListener; +import java.util.List; + + +/** + * EventListener interface for objects that want to receive tracker responses. + * + */ +public interface AnnounceResponseListener extends EventListener { + + /** + * Handle an announce response event. + * + * @param interval The announce interval requested by the tracker. + * @param complete The number of seeders on this torrent. + * @param incomplete The number of leechers on this torrent. + */ + public void handleAnnounceResponse(int interval, int complete, + int incomplete); + + /** + * Handle the discovery of new peers. + * + * @param peers The list of peers discovered (from the announce response or + * any other means like DHT/PEX, etc.). + */ + public void handleDiscoveredPeers(List+ * This method is called by {@link Announce#stop()} to make sure all connections + * are correctly closed when the announce thread is asked to stop. + *
+ */ + protected void close() { + // Do nothing by default, but can be overloaded. + } + + /** + * Formats an announce event into a usable string. + */ + protected String formatAnnounceEvent( + AnnounceRequestMessage.RequestEvent event) { + return AnnounceRequestMessage.RequestEvent.NONE.equals(event) + ? "" + : String.format(" %s", event.name()); + } + + /** + * Handle the announce response from the tracker. + * + *+ * Analyzes the response from the tracker and acts on it. If the response + * is an error, it is logged. Otherwise, the announce response is used + * to fire the corresponding announce and peer events to all announce + * listeners. + *
+ * + * @param message The incoming {@link TrackerMessage}. + * @param inhibitEvents Whether or not to prevent events from being fired. + */ + protected void handleTrackerAnnounceResponse(TrackerMessage message, + boolean inhibitEvents, String trackerAddr) throws AnnounceException { + logger.info("get tracker response, message: " + message.getType() + new String(message.getData().array())); + if (message instanceof ErrorMessage) { + ErrorMessage error = (ErrorMessage)message; + // if the error is due to unknown torrents, we post torrent to tracker + if (error.getReason().equals(FailureReason.UNKNOWN_TORRENT.getMessage())) { + logger.info("Tracker don't know this torrent. Post torrent to tracker"); + transport.postTorrent(trackerAddr, this.torrent.getEncoded()); + logger.info("finish post"); + } + /* this throw is must even message is UNKNOWN_TORRENT because it keep the announce loop from moving from START_MSG */ + throw new AnnounceException(error.getReason()); + } + + if (! (message instanceof AnnounceResponseMessage)) { + throw new AnnounceException("Unexpected tracker message type " + + message.getType().name() + "!"); + } + + if (inhibitEvents) { + return; + } + + AnnounceResponseMessage response = + (AnnounceResponseMessage)message; + this.fireAnnounceResponseEvent( + response.getComplete(), + response.getIncomplete(), + response.getInterval()); + this.fireDiscoveredPeersEvent(response.getPeers()); + } + + /** + * Fire the announce response event to all listeners. + * + * @param complete The number of seeders on this torrent. + * @param incomplete The number of leechers on this torrent. + * @param interval The announce interval requested by the tracker. + */ + protected void fireAnnounceResponseEvent(int complete, int incomplete, + int interval) { + for (AnnounceResponseListener listener : this.listeners) { + listener.handleAnnounceResponse(interval, complete, incomplete); + } + } + + /** + * Fire the new peer discovery event to all listeners. + * + * @param peers The list of peers discovered. + */ + protected void fireDiscoveredPeersEvent(List+ * This function first builds an announce request for the specified event + * with all the required parameters. Then, the request is made to the + * tracker and the response analyzed. + *
+ * + *+ * All registered {@link AnnounceResponseListener} objects are then fired + * with the decoded payload. + *
+ * + * @param event The announce event type (can be AnnounceEvent.NONE for + * periodic updates). + * @param inhibitEvents Prevent event listeners from being notified. + */ + public void announce(AnnounceRequestMessage.RequestEvent event, + boolean inhibitEvents) throws AnnounceException { + logger.info(this + "announce: Announcing{} to tracker with {}U/{}D/{}L bytes...", + new Object[] { + this.formatAnnounceEvent(event), + this.torrent.getUploaded(), + this.torrent.getDownloaded(), + this.torrent.getLeft() + }); + + HTTPAnnounceRequestMessage request; + + try { + request = this.buildAnnounceRequest(event); + } catch (IOException ioe) { + throw new AnnounceException("Error building announce request (" + + ioe.getMessage() + ")", ioe); + } catch (MessageValidationException mve) { + throw new AnnounceException("Announce request creation violated " + + "expected protocol (" + mve.getMessage() + ")", mve); + } + + String rackMaster = null; + do { + rackMaster = rackMaster == null ? topology.getRackMaster(this.torrent.getHexInfoHash()) : topology.getNextRackMaster(rackMaster); + try { + logger.info(this + "try announcing to rackMaster " + rackMaster); + announceTo(request, rackMaster, inhibitEvents); + } catch (AnnounceException ae) { + throw ae; + } catch (IOException e) { + logger.error("fail to announce to rackMaster " + rackMaster + ", try next"); + continue; + } + + // If we are rackMaster , we need to announce to globalMaster too + if (rackMaster.equals(topology.getLocalAddr())) { + String globalMaster = null; + do { + globalMaster = globalMaster == null ? topology.getGlobalMaster(this.torrent.getHexInfoHash()) : topology.getNextRackMaster(globalMaster); + // If we are also globalMaster, we cannot announce because otherwise we violate the interval + if (globalMaster.equals(rackMaster)) + break; + try { + logger.info("try announcing to globalMaster " + globalMaster); + announceTo(request, globalMaster, inhibitEvents); + } catch (AnnounceException e) { + throw e; + } catch (IOException e) { + logger.error("fail to announce to globalMaster " + globalMaster + ", try next"); + continue; + } + break; + } while (true); + } + break; + } while (true); + } + + /** + * Send and process a tracker announce request. + * + *+ * This function send the request to the + * tracker and then the response is analyzed. + *
+ * + *+ * All registered {@link AnnounceResponseListener} objects are then fired + * with the decoded payload. + *
+ * + * @param request the announce request + * @param trackerAddr the address of the tracker + */ + public void announceTo(HTTPAnnounceRequestMessage request, String trackerAddr, boolean inhibitEvents) throws IOException, AnnounceException { + logger.info(request.getPort() + "Announcing {} to tracker {} with {}U/{}D/{}L bytes..., type" + request.getEvent().getEventName(), + new Object[] { + request.getHexInfoHash(), + trackerAddr, + this.torrent.getUploaded(), + this.torrent.getDownloaded(), + this.torrent.getLeft() + }); + + try { + URL target = request.buildAnnounceURL(new URL("http://" + trackerAddr + ":6969/announce")); + HTTPTrackerMessage message = transport.announce(target, request); + this.handleTrackerAnnounceResponse(message, inhibitEvents, trackerAddr); + } catch (IOException ioe) { + logger.error(ioe.getClass().getCanonicalName()); + logger.error(ioe.getMessage()); + throw ioe; + } catch (MessageValidationException mve) { + logger.error(mve.getClass().getCanonicalName()); + logger.error(mve.getMessage()); + throw new AnnounceException("Tracker message violates expected " + + "protocol (" + mve.getMessage() + ")", mve); + } catch (AnnounceException ae) { + throw ae; + } + } + + /** + * Build the announce request tracker message. + * + * @param event The announce event (can be NONE or null) + * @return Returns an instance of a {@link HTTPAnnounceRequestMessage} + * that can be used to generate the fully qualified announce URL, with + * parameters, to make the announce request. + * @throws UnsupportedEncodingException + * @throws IOException + * @throws MessageValidationException + */ + private HTTPAnnounceRequestMessage buildAnnounceRequest( + AnnounceRequestMessage.RequestEvent event) + throws UnsupportedEncodingException, IOException, + MessageValidationException { + // Build announce request message + return HTTPAnnounceRequestMessage.craft( + this.torrent.getInfoHash(), + this.peer.getPeerId().array(), + this.peer.getPort(), + this.torrent.getUploaded(), + this.torrent.getDownloaded(), + this.torrent.getLeft(), + true, false, event, + this.peer.getIp(), + AnnounceRequestMessage.DEFAULT_NUM_WANT); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClientTransport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClientTransport.java new file mode 100644 index 0000000..809beab --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/announce/TrackerClientTransport.java @@ -0,0 +1,84 @@ +package org.apache.hadoop.yarn.server.broadcast.client.announce; + +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.MessageValidationException; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPAnnounceRequestMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.http.HTTPTrackerMessage; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.client.DefaultHttpClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; + +public class TrackerClientTransport { + protected static final Logger logger = LoggerFactory.getLogger(TrackerClientTransport.class); + + public HTTPTrackerMessage announce(URL target, HTTPAnnounceRequestMessage request) throws IOException, MessageValidationException { + InputStream in = null; + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) target.openConnection(); + in = conn.getInputStream(); + logger.info("YZY in connection opened"); + } catch (IOException e) { + in = conn.getErrorStream(); + logger.info("YZY err connection opened"); + if (in == null) + throw new IOException(); + } + + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(in); + HTTPTrackerMessage message = HTTPTrackerMessage.parse(ByteBuffer.wrap(baos.toByteArray())); + return message; + } catch (IOException ioe) { + throw ioe; + } catch (MessageValidationException mve) { + throw mve; + } finally { + // Make sure we close everything down at the end to avoid resource + // leaks. + try { + in.close(); + } catch (IOException ioe) { + logger.warn("Problem ensuring error stream closed!", ioe); + } + + // This means trying to close the error stream as well. + InputStream err = conn.getErrorStream(); + if (err != null) { + try { + err.close(); + } catch (IOException ioe) { + logger.warn("Problem ensuring error stream closed!", ioe); + } + } + + conn.disconnect(); + } + } + + public void postTorrent(String trackerAddr, byte[] torrentEncodedData) { + DefaultHttpClient cli = new DefaultHttpClient(); + HttpPost method = new HttpPost("http://" + trackerAddr + ":6969/preannounce"); + InputStreamEntity reqEntity = new InputStreamEntity(new ByteArrayInputStream(torrentEncodedData), torrentEncodedData.length); + reqEntity.setContentType("application/octet-stream"); + reqEntity.setChunked(false); + method.setEntity(reqEntity); + + try { + cli.execute(method); + } catch (IOException e) { + e.printStackTrace(); + } + cli.getConnectionManager().shutdown(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/MessageListener.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/MessageListener.java new file mode 100644 index 0000000..ba3a5ed --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/MessageListener.java @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage; + +import java.util.EventListener; + + +/** + * EventListener interface for objects that want to receive incoming messages + * from peers. + * + */ +public interface MessageListener extends EventListener { + + public void handleMessage(PeerMessage msg); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerActivityListener.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerActivityListener.java new file mode 100644 index 0000000..1f99c22 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerActivityListener.java @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import org.apache.hadoop.yarn.server.broadcast.client.Piece; + +import java.io.IOException; + +import java.util.BitSet; +import java.util.EventListener; + + +/** + * EventListener interface for objects that want to handle peer activity + * events like piece availability, or piece completion events, and more. + * + */ +public interface PeerActivityListener extends EventListener { + + /** + * Peer choked handler. + * + *+ * This handler is fired when a peer choked and now refuses to send data to + * us. This means we should not try to request or expect anything from it + * until it becomes ready again. + *
+ * + * @param peer The peer that choked. + */ + public void handlePeerChoked(SharingPeer peer); + + /** + * Peer ready handler. + * + *+ * This handler is fired when a peer notified that it is no longer choked. + * This means we can send piece block requests to it and start downloading. + *
+ * + * @param peer The peer that became ready. + */ + public void handlePeerReady(SharingPeer peer); + + /** + * Piece availability handler. + * + *+ * This handler is fired when an update in piece availability is received + * from a peer's HAVE message. + *
+ * + * @param peer The peer we got the update from. + * @param piece The piece that became available from this peer. + */ + public void handlePieceAvailability(SharingPeer peer, Piece piece); + + /** + * Bit field availability handler. + * + *+ * This handler is fired when an update in piece availability is received + * from a peer's BITFIELD message. + *
+ * + * @param peer The peer we got the update from. + * @param availablePieces The pieces availability bit field of the peer. + */ + public void handleBitfieldAvailability(SharingPeer peer, + BitSet availablePieces); + + /** + * Piece upload completion handler. + * + *+ * This handler is fired when a piece has been uploaded entirely to a peer. + *
+ * + * @param peer The peer the piece was sent to. + * @param piece The piece in question. + */ + public void handlePieceSent(SharingPeer peer, Piece piece); + + /** + * Piece download completion handler. + * + *+ * This handler is fired when a piece has been downloaded entirely and the + * piece data has been revalidated. + *
+ * + *+ * Note: the piece may not be valid after it has been + * downloaded, in which case appropriate action should be taken to + * redownload the piece. + *
+ * + * @param peer The peer we got this piece from. + * @param piece The piece in question. + */ + public void handlePieceCompleted(SharingPeer peer, Piece piece) + throws IOException; + + /** + * Peer disconnection handler. + * + *+ * This handler is fired when a peer disconnects, or is disconnected due to + * protocol violation. + *
+ * + * @param peer The peer we got this piece from. + */ + public void handlePeerDisconnected(SharingPeer peer); + + /** + * Handler for IOException during peer operation. + * + * @param peer The peer whose activity trigger the exception. + * @param ioe The IOException object, for reporting. + */ + public void handleIOException(SharingPeer peer, IOException ioe); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerExchange.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerExchange.java new file mode 100644 index 0000000..1e0c46c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/PeerExchange.java @@ -0,0 +1,428 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage.Type; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.text.ParseException; +import java.util.BitSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + + +/** + * Incoming and outgoing peer communication system. + * + *+ * The peer exchange is a wrapper around peer communication. It provides both + * incoming and outgoing communication channels to a connected peer after a + * successful handshake. + *
+ * + *+ * When a socket is bound to a sharing peer, a PeerExchange is automatically + * created to wrap this socket into a more usable system for communication with + * the remote peer. + *
+ * + *
+ * For incoming messages, the peer exchange provides message parsing and calls
+ * the handleMessage() method of the peer for each successfully
+ * parsed message.
+ *
+ * For outgoing message, the peer exchange offers a send() message
+ * that queues messages, and takes care of automatically sending a keep-alive
+ * message to the remote peer every two minutes when other message have been
+ * sent in that period of time, as recommended by the BitTorrent protocol
+ * specification.
+ *
+ * The message is queued in the outgoing message queue and will be + * processed as soon as possible. + *
+ * + * @param message The message object to send. + */ + public void send(PeerMessage message) { + try { + this.sendQueue.put(message); + } catch (InterruptedException ie) { + // Ignore, our send queue will only block if it contains + // MAX_INTEGER messages, in which case we're already in big + // trouble, and we'd have to be interrupted, too. + } + } + + /** + * Close and stop the peer exchange. + * + *+ * Closes the socket channel and stops both incoming and outgoing threads. + *
+ */ + public void close() { + this.stop = true; + + if (this.channel.isConnected()) { + IOUtils.closeQuietly(this.channel); + } + + logger.debug("Peer exchange with {} closed.", this.peer); + } + + /** + * Abstract Thread subclass that allows conditional rate limiting + * forPIECE messages.
+ *
+ * + * To impose rate limits, we only want to throttle when processing PIECE + * messages. All other peer messages should be exchanged as quickly as + * possible. + *
+ * + * @author ptgoetz + */ + private abstract class RateLimitThread extends Thread { + + protected final Rate rate = new Rate(); + protected long sleep = 1000; + + /** + * Dynamically determines an amount of time to sleep, based on the + * average read/write throughput. + * + *
+ * The algorithm is functional, but could certainly be improved upon.
+ * One obvious drawback is that with large changes in
+ * maxRate, it will take a while for the sleep time to
+ * adjust and the throttled rate to "smooth out."
+ *
+ * Ideally, it would calculate the optimal sleep time necessary to hit + * a desired throughput rather than continuously adjust toward a goal. + *
+ * + * @param maxRate the target rate in kB/second. + * @param messageSize the size, in bytes, of the last message read/written. + * @param message the lastPeerMessage read/written.
+ */
+ protected void rateLimit(double maxRate, long messageSize, PeerMessage message) {
+ if (message.getType() != Type.PIECE || maxRate <= 0) {
+ return;
+ }
+
+ try {
+ this.rate.add(messageSize);
+
+ // Continuously adjust the sleep time to try to hit our target
+ // rate limit.
+ if (rate.get() > (maxRate * 1024)) {
+ Thread.sleep(this.sleep);
+ this.sleep += 50;
+ } else {
+ this.sleep = this.sleep > 50
+ ? this.sleep - 50
+ : 0;
+ }
+ } catch (InterruptedException e) {
+ // Not critical, eat it.
+ }
+ }
+ }
+
+ /**
+ * Incoming messages thread.
+ *
+ *
+ * The incoming messages thread reads from the socket's input stream and
+ * waits for incoming messages. When a message is fully retrieve, it is
+ * parsed and passed to the peer's handleMessage() method that
+ * will act based on the message type.
+ *
+ * The outgoing messages thread waits for messages to appear in the send + * queue and processes them, in order, as soon as they arrive. + *
+ * + *+ * If no message is available for KEEP_ALIVE_IDLE_MINUTES minutes, it will + * automatically send a keep-alive message to the remote peer to keep the + * connection active. + *
+ * + * @author mpetazzoni + */ + private class OutgoingThread extends RateLimitThread { + + @Override + public void run() { + try { + // Loop until told to stop. When stop was requested, loop until + // the queue is served. + while (!stop || (stop && sendQueue.size() > 0)) { + try { + // Wait for two minutes for a message to send + PeerMessage message = sendQueue.poll( + 5, + TimeUnit.SECONDS); + + if (message == null) { + if (stop) { + return; + } + + message = PeerMessage.KeepAliveMessage.craft(); + } + + logger.trace("Sending {} to {}", message, peer); + + ByteBuffer data = message.getData(); + long size = 0; + while (!stop && data.hasRemaining()) { + int written = channel.write(data); + size += written; + if (written < 0) { + throw new EOFException( + "Reached end of stream while writing"); + } + } + + // Wait if needed to reach configured upload rate. + this.rateLimit(PeerExchange.this.torrent.getMaxUploadRate(), + size, message); + } catch (InterruptedException ie) { + // Ignore and potentially terminate + } + } + } catch (IOException ioe) { + logger.debug("Could not send message to {}: {}", + peer, + ioe.getMessage() != null + ? ioe.getMessage() + : ioe.getClass().getName()); + peer.unbind(true); + } + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/Rate.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/Rate.java new file mode 100644 index 0000000..594268c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/Rate.java @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import java.io.Serializable; +import java.util.Comparator; + + +/** + * A data exchange rate representation. + * + *+ * This is a utility class to keep track, and compare, of the data exchange + * rate (either download or upload) with a peer. + *
+ * + */ +public class Rate implements Comparable+ * The exchange rate is the number of bytes exchanged since the last + * reset and the last input. + *
+ */ + public synchronized float get() { + if (this.last - this.reset == 0) { + return 0; + } + + return this.bytes / ((this.last - this.reset) / 1000.0f); + } + + /** + * Reset the measurement. + */ + public synchronized void reset() { + this.bytes = 0; + this.reset = System.currentTimeMillis(); + this.last = this.reset; + } + + @Override + public int compareTo(Rate other) { + return RATE_COMPARATOR.compare(this, other); + } + + /** + * A rate comparator. + * + *+ * This class provides a comparator to sort peers by an exchange rate, + * comparing two rates and returning an ascending ordering. + *
+ * + *
+ * Note: we need to make sure here that we don't return 0, which
+ * would provide an ordering that is inconsistent with
+ * equals()'s behavior, and result in unpredictable behavior
+ * for sorted collections using this comparator.
+ *
+ * This method compares float, but we don't care too much about + * rounding errors. It's just to order peers so super-strict rate based + * order is not required. + *
+ * + * @param a + * @param b + */ + @Override + public int compare(Rate a, Rate b) { + if (a.get() > b.get()) { + return 1; + } + + return -1; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/SharingPeer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/SharingPeer.java new file mode 100644 index 0000000..4991cc7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/peer/SharingPeer.java @@ -0,0 +1,796 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.peer; + +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.PeerMessage; +import org.apache.hadoop.yarn.server.broadcast.client.Piece; +import org.apache.hadoop.yarn.server.broadcast.client.SharedTorrent; + +import java.io.IOException; +import java.io.Serializable; +import java.net.SocketException; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.util.BitSet; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * A peer exchanging on a torrent with the BitTorrent client. + * + *+ * A SharingPeer extends the base Peer class with all the data and logic needed + * by the BitTorrent client to interact with a peer exchanging on the same + * torrent. + *
+ * + *+ * Peers are defined by their peer ID, IP address and httpPort number, just like + * base peers. Peers we exchange with also contain four crucial attributes: + *
+ * + *choking, which means we are choking this peer and we're
+ * not willing to send him anything for now;interesting, which means we are interested in a piece
+ * this peer has;choked, if this peer is choking and won't send us
+ * anything right now;interested, if this peer is interested in something we
+ * have.+ * Peers start choked and uninterested. + *
+ * + */ +public class SharingPeer extends Peer implements MessageListener { + + private static final Logger logger = + LoggerFactory.getLogger(SharingPeer.class); + + private static final int MAX_PIPELINED_REQUESTS = 5; + + private boolean choking; + private boolean interesting; + + private boolean choked; + private boolean interested; + + private SharedTorrent torrent; + private BitSet availablePieces; + + private Piece requestedPiece; + private int lastRequestedOffset; + + private BlockingQueue+ * Initially, peers are considered choked, choking, and neither interested + * nor interesting. + *
+ */ + public synchronized void reset() { + this.choking = true; + this.interesting = false; + this.choked = true; + this.interested = false; + + this.exchange = null; + + this.requests = null; + this.lastRequestedOffset = 0; + this.downloading = false; + } + + /** + * Choke this peer. + * + *+ * We don't want to upload to this peer anymore, so mark that we're choking + * from this peer. + *
+ */ + public void choke() { + if (!this.choking) { + logger.trace("Choking {}", this); + this.send(PeerMessage.ChokeMessage.craft()); + this.choking = true; + } + } + + /** + * Unchoke this peer. + * + *+ * Mark that we are no longer choking from this peer and can resume + * uploading to it. + *
+ */ + public void unchoke() { + if (this.choking) { + logger.trace("Unchoking {}", this); + this.send(PeerMessage.UnchokeMessage.craft()); + this.choking = false; + } + } + + public boolean isChoking() { + return this.choking; + } + + + public void interesting() { + if (!this.interesting) { + logger.trace("Telling {} we're interested.", this); + this.send(PeerMessage.InterestedMessage.craft()); + this.interesting = true; + } + } + + public void notInteresting() { + if (this.interesting) { + logger.trace("Telling {} we're no longer interested.", this); + this.send(PeerMessage.NotInterestedMessage.craft()); + this.interesting = false; + } + } + + public boolean isInteresting() { + return this.interesting; + } + + + public boolean isChoked() { + return this.choked; + } + + public boolean isInterested() { + return this.interested; + } + + /** + * Returns the available pieces from this peer. + * + * @return A clone of the available pieces bit field from this peer. + */ + public BitSet getAvailablePieces() { + synchronized (this.availablePieces) { + return (BitSet)this.availablePieces.clone(); + } + } + + /** + * Returns the currently requested piece, if any. + */ + public Piece getRequestedPiece() { + return this.requestedPiece; + } + + /** + * Tells whether this peer is a seed. + * + * @return Returns true if the peer has all of the torrent's pieces + * available. + */ + public synchronized boolean isSeed() { + return this.torrent.getPieceCount() > 0 && + this.getAvailablePieces().cardinality() == + this.torrent.getPieceCount(); + } + + /** + * Bind a connected socket to this peer. + * + *+ * This will create a new peer exchange with this peer using the given + * socket, and register the peer as a message listener. + *
+ * + * @param channel The connected socket channel for this peer. + */ + public synchronized void bind(SocketChannel channel) throws SocketException { + this.unbind(true); + + this.exchange = new PeerExchange(this, this.torrent, channel); + this.exchange.register(this); + + this.download = new Rate(); + this.download.reset(); + + this.upload = new Rate(); + this.upload.reset(); + } + + /** + * Tells whether this peer as an active connection through a peer exchange. + */ + public boolean isConnected() { + synchronized (this.exchangeLock) { + return this.exchange != null && this.exchange.isConnected(); + } + } + + /** + * Unbind and disconnect this peer. + * + *+ * This terminates the eventually present and/or connected peer exchange + * with the peer and fires the peer disconnected event to any peer activity + * listeners registered on this peer. + *
+ * + * @param force Force unbind without sending cancel requests. + */ + public void unbind(boolean force) { + if (!force) { + // Cancel all outgoing requests, and send a NOT_INTERESTED message to + // the peer. + this.cancelPendingRequests(); + this.send(PeerMessage.NotInterestedMessage.craft()); + } + + synchronized (this.exchangeLock) { + if (this.exchange != null) { + this.exchange.close(); + this.exchange = null; + } + } + + this.firePeerDisconnected(); + this.requestedPiece = null; + } + + /** + * Send a message to the peer. + * + *+ * Delivery of the message can only happen if the peer is connected. + *
+ * + * @param message The message to send to the remote peer through our peer + * exchange. + */ + public void send(PeerMessage message) throws IllegalStateException { + if (this.isConnected()) { + this.exchange.send(message); + } else { + logger.warn("Attempting to send a message to non-connected peer {}!", this); + } + } + + /** + * Download the given piece from this peer. + * + *+ * Starts a block request queue and pre-fill it with MAX_PIPELINED_REQUESTS + * block requests. + *
+ * + *+ * Further requests will be added, one by one, every time a block is + * returned. + *
+ * + * @param piece The piece chosen to be downloaded from this peer. + */ + public synchronized void downloadPiece(Piece piece) + throws IllegalStateException { + if (this.isDownloading()) { + IllegalStateException up = new IllegalStateException( + "Trying to download a piece while previous " + + "download not completed!"); + logger.warn("What's going on? {}", up.getMessage(), up); + throw up; // ah ah. + } + + this.requests = new LinkedBlockingQueue+ * Re-fill the pipeline to get download the next blocks from the peer. + *
+ */ + private void requestNextBlocks() { + synchronized (this.requestsLock) { + if (this.requests == null || this.requestedPiece == null) { + // If we've been taken out of a piece download context it means our + // outgoing requests have been cancelled. Don't enqueue new + // requests until a proper piece download context is + // re-established. + return; + } + + while (this.requests.remainingCapacity() > 0 && + this.lastRequestedOffset < this.requestedPiece.size()) { + PeerMessage.RequestMessage request = PeerMessage.RequestMessage + .craft( + this.requestedPiece.getIndex(), + this.lastRequestedOffset, + Math.min( + (int)(this.requestedPiece.size() - + this.lastRequestedOffset), + PeerMessage.RequestMessage.DEFAULT_REQUEST_SIZE)); + this.requests.add(request); + this.send(request); + this.lastRequestedOffset += request.getLength(); + } + + this.downloading = this.requests.size() > 0; + } + } + + /** + * Remove the REQUEST message from the request pipeline matching this + * PIECE message. + * + *+ * Upon reception of a piece block with a PIECE message, remove the + * corresponding request from the pipeline to make room for the next block + * requests. + *
+ * + * @param message The PIECE message received. + */ + private void removeBlockRequest(PeerMessage.PieceMessage message) { + synchronized (this.requestsLock) { + if (this.requests == null) { + return; + } + + for (PeerMessage.RequestMessage request : this.requests) { + if (request.getPiece() == message.getPiece() && + request.getOffset() == message.getOffset()) { + this.requests.remove(request); + break; + } + } + + this.downloading = this.requests.size() > 0; + } + } + + /** + * Cancel all pending requests. + * + *+ * This queues CANCEL messages for all the requests in the queue, and + * returns the list of requests that were in the queue. + *
+ * + *+ * If no request queue existed, or if it was empty, an empty set of request + * messages is returned. + *
+ */ + public Set+ * The event contains the peer that chocked. + *
+ */ + private void firePeerChoked() { + for (PeerActivityListener listener : this.listeners) { + listener.handlePeerChoked(this); + } + } + + /** + * Fire the peer ready event to all registered listeners. + * + *+ * The event contains the peer that unchoked or became ready. + *
+ */ + private void firePeerReady() { + for (PeerActivityListener listener : this.listeners) { + listener.handlePeerReady(this); + } + } + + /** + * Fire the piece availability event to all registered listeners. + * + *+ * The event contains the peer (this), and the piece that became available. + *
+ */ + private void firePieceAvailabity(Piece piece) { + for (PeerActivityListener listener : this.listeners) { + listener.handlePieceAvailability(this, piece); + } + } + + /** + * Fire the bit field availability event to all registered listeners. + * + * The event contains the peer (this), and the bit field of available pieces + * from this peer. + */ + private void fireBitfieldAvailabity() { + for (PeerActivityListener listener : this.listeners) { + listener.handleBitfieldAvailability(this, + this.getAvailablePieces()); + } + } + + /** + * Fire the piece sent event to all registered listeners. + * + *+ * The event contains the peer (this), and the piece number that was + * sent to the peer. + *
+ * + * @param piece The completed piece. + */ + private void firePieceSent(Piece piece) { + for (PeerActivityListener listener : this.listeners) { + listener.handlePieceSent(this, piece); + } + } + + /** + * Fire the piece completion event to all registered listeners. + * + *+ * The event contains the peer (this), and the piece number that was + * completed. + *
+ * + * @param piece The completed piece. + */ + private void firePieceCompleted(Piece piece) throws IOException { + for (PeerActivityListener listener : this.listeners) { + listener.handlePieceCompleted(this, piece); + } + } + + /** + * Fire the peer disconnected event to all registered listeners. + * + *+ * The event contains the peer that disconnected (this). + *
+ */ + private void firePeerDisconnected() { + for (PeerActivityListener listener : this.listeners) { + listener.handlePeerDisconnected(this); + } + } + + /** + * Fire the IOException event to all registered listeners. + * + *+ * The event contains the peer that triggered the problem, and the + * exception object. + *
+ */ + private void fireIOException(IOException ioe) { + for (PeerActivityListener listener : this.listeners) { + listener.handleIOException(this, ioe); + } + } + + /** + * Download rate comparator. + * + *+ * Compares sharing peers based on their current download rate. + *
+ * + * @author mpetazzoni + * @see Rate.RateComparator + */ + public static class DLRateComparator + implements Comparator+ * Compares sharing peers based on their current upload rate. + *
+ * + * @author mpetazzoni + * @see Rate.RateComparator + */ + public static class ULRateComparator + implements Comparator+ * This implementation of the torrent byte storage provides support for + * multi-file torrents and completely abstracts the read/write operations from + * the notion of different files. The byte storage is represented as one + * continuous byte storage, directly accessible by offset regardless of which + * file this offset lands. + *
+ * + */ +public class FileCollectionStorage implements TorrentByteStorage { + + private static final Logger logger = + LoggerFactory.getLogger(FileCollectionStorage.class); + + private final List+ * This simple inner class holds the details for a read or write operation + * on one of the underlying {@link FileStorage}s. + *
+ * + * @author dgiffin + * @author mpetazzoni + */ + private static class FileOffset { + + public final FileStorage file; + public final long offset; + public final long length; + + FileOffset(FileStorage file, long offset, long length) { + this.file = file; + this.offset = offset; + this.length = length; + } + }; + + /** + * Select the group of files impacted by an operation. + * + *+ * This function selects which files are impacted by a read or write + * operation, with their respective relative offset and chunk length. + *
+ * + * @param offset The offset of the operation, in bytes, relative to the + * complete byte storage. + * @param length The number of bytes to read or write. + * @return A list of {@link FileOffset} objects representing the {@link + * FileStorage}s impacted by the operation, bundled with their + * respective relative offset and number of bytes to read or write. + * @throws IllegalArgumentException If the offset and length go over the + * byte storage size. + * @throws IllegalStateException If the files registered with this byte + * storage can't accommodate the request (should not happen, really). + */ + private List+ * This implementation of TorrentByteStorageFile provides a torrent byte data + * storage relying on a single underlying file and uses a RandomAccessFile + * FileChannel to expose thread-safe read/write methods. + *
+ * + */ +public class FileStorage implements TorrentByteStorage { + + private static final Logger logger = + LoggerFactory.getLogger(FileStorage.class); + + private final File target; + private final File partial; + private final long offset; + private final long size; + + private RandomAccessFile randomAccessFile; + private FileChannel channel; + private File current; + + public FileStorage(File file, long size) throws IOException { + this(file, 0, size); + } + + public FileStorage(File file, long offset, long size) + throws IOException { + this.target = file; + this.offset = offset; + this.size = size; + + this.partial = new File(this.target.getAbsolutePath() + + TorrentByteStorage.PARTIAL_FILE_NAME_SUFFIX); + + if (this.partial.exists()) { + logger.debug("Partial download found at {}. Continuing...", + this.partial.getAbsolutePath()); + this.current = this.partial; + this.randomAccessFile = new RandomAccessFile(this.current, "rw"); + // Set the file length to the appropriate size, eventually truncating + // or extending the file if it already exists with a different size. + this.randomAccessFile.setLength(this.size); + } else if (!this.target.exists()) { + logger.debug("Downloading new file to {}...", + this.partial.getAbsolutePath()); + this.current = this.partial; + this.randomAccessFile = new RandomAccessFile(this.current, "rw"); + // Set the file length to the appropriate size, eventually truncating + // or extending the file if it already exists with a different size. + this.randomAccessFile.setLength(this.size); + } else { + logger.debug("Using existing file {}.", + this.target.getAbsolutePath()); + this.current = this.target; + this.randomAccessFile = new RandomAccessFile(this.current, "r"); + } + + this.channel = randomAccessFile.getChannel(); + logger.info("Initialized byte storage file at {} " + + "({}+{} byte(s)).", + new Object[] { + this.current.getAbsolutePath(), + this.offset, + this.size, + }); + } + + protected long offset() { + return this.offset; + } + + @Override + public long size() { + return this.size; + } + + @Override + public int read(ByteBuffer buffer, long offset) throws IOException { + int requested = buffer.remaining(); + + if (offset + requested > this.size) { + throw new IllegalArgumentException("Invalid storage read request!"); + } + + int bytes = this.channel.read(buffer, offset); + if (bytes < requested) { + throw new IOException("Storage underrun!"); + } + + return bytes; + } + + @Override + public int write(ByteBuffer buffer, long offset) throws IOException { + int requested = buffer.remaining(); + + if (offset + requested > this.size) { + throw new IllegalArgumentException("Invalid storage write request!"); + } + + return this.channel.write(buffer, offset); + } + + @Override + public synchronized void close() throws IOException { + logger.debug("Closing file channel to " + this.current.getName() + "..."); + if (this.channel.isOpen()) { + this.channel.force(true); + } + this.randomAccessFile.close(); + } + + /** Move the partial file to its final location. + */ + @Override + public synchronized void finish() throws IOException { + logger.debug("Closing file channel to " + this.current.getName() + + " (download complete)."); + if (this.channel.isOpen()) { + this.channel.force(true); + } + + // Nothing more to do if we're already on the target file. + if (this.isFinished()) { + return; + } + + this.randomAccessFile.setLength(this.size); + this.randomAccessFile.close(); + FileUtils.deleteQuietly(this.target); + FileUtils.moveFile(this.current, this.target); + + logger.debug("Re-opening torrent byte storage at {}.", + this.target.getAbsolutePath()); + this.randomAccessFile = new RandomAccessFile(this.target, "r"); + + this.channel = this.randomAccessFile.getChannel(); + this.current = this.target; + FileUtils.deleteQuietly(this.partial); + logger.info("Moved torrent data from {} to {}.", + this.partial.getName(), + this.target.getName()); + } + + @Override + public boolean isFinished() { + return this.current.equals(this.target); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/TorrentByteStorage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/TorrentByteStorage.java new file mode 100644 index 0000000..ac7f083 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/storage/TorrentByteStorage.java @@ -0,0 +1,104 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.client.storage; + +import java.io.IOException; +import java.nio.ByteBuffer; + + +/** + * Abstract torrent byte storage. + * + *+ * This interface defines the methods for accessing an abstracted torrent byte + * storage. A torrent, especially when it contains multiple files, needs to be + * seen as one single continuous stream of bytes. Torrent pieces will most + * likely span accross file boundaries. This abstracted byte storage aims at + * providing a simple interface for read/write access to the torrent data, + * regardless of how it is composed underneath the piece structure. + *
+ * + */ +public interface TorrentByteStorage { + + public static final String PARTIAL_FILE_NAME_SUFFIX = ".part"; + + /** + * Returns the total size of the torrent storage. + */ + public long size(); + + /** + * Read from the byte storage. + * + *+ * Read {@code length} bytes at offset {@code offset} from the underlying + * byte storage and return them in a {@link ByteBuffer}. + *
+ * + * @param buffer The buffer to read the bytes into. The buffer's limit will + * control how many bytes are read from the storage. + * @param offset The offset, in bytes, to read from. This must be within + * the storage boundary. + * @return The number of bytes read from the storage. + * @throws IOException If an I/O error occurs while reading from the + * byte storage. + */ + public int read(ByteBuffer buffer, long offset) throws IOException; + + /** + * Write bytes to the byte storage. + * + *+ *
+ * + * @param block A {@link ByteBuffer} containing the bytes to write to the + * storage. The buffer limit is expected to be set correctly: all bytes + * from the buffer will be used. + * @param offset Offset in the underlying byte storage to write the block + * at. + * @return The number of bytes written to the storage. + * @throws IOException If an I/O error occurs while writing to the byte + * storage. + */ + public int write(ByteBuffer block, long offset) throws IOException; + + /** + * Close this byte storage. + * + * @throws IOException If closing the underlying storage (file(s) ?) + * failed. + */ + public void close() throws IOException; + + /** + * Finalize the byte storage when the download is complete. + * + *+ * This gives the byte storage the opportunity to perform finalization + * operations when the download completes, like moving the files from a + * temporary location to their destination. + *
+ * + * @throws IOException If the finalization failed. + */ + public void finish() throws IOException; + + /** + * Tells whether this byte storage has been finalized. + */ + public boolean isFinished(); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategy.java new file mode 100644 index 0000000..8907e5e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/client/strategy/RequestStrategy.java @@ -0,0 +1,29 @@ +package org.apache.hadoop.yarn.server.broadcast.client.strategy; + +import java.util.BitSet; +import java.util.SortedSet; + +import org.apache.hadoop.yarn.server.broadcast.client.Piece; + +/** + * Interface for a piece request strategy provider. + * + * @author cjmalloy + * + */ +public interface RequestStrategy { + + /** + * Choose a piece from the remaining pieces. + * + * @param rarest + * A set sorted by how rare the piece is + * @param interesting + * A set of the index of all interesting pieces + * @param pieces + * The complete array of pieces + * + * @return The chosen piece, ornull if no piece is interesting
+ */
+ Piece choosePiece(SortedSet+ * This class is meant to be a common base for the tracker and client, which + * would presumably subclass it to extend its functionality and fields. + *
+ * + */ +public class Peer { + + private final InetSocketAddress address; + private final String hostId; + + private ByteBuffer peerId; + private String hexPeerId; + + /** + * Instantiate a new peer. + * + * @param address The peer's address, with httpPort. + */ + public Peer(InetSocketAddress address) { + this(address, null); + } + + /** + * Instantiate a new peer. + * + * @param ip The peer's IP address. + * @param port The peer's httpPort. + */ + public Peer(String ip, int port) { + this(new InetSocketAddress(ip, port), null); + } + + /** + * Instantiate a new peer. + * + * @param ip The peer's IP address. + * @param port The peer's httpPort. + * @param peerId The byte-encoded peer ID. + */ + public Peer(String ip, int port, ByteBuffer peerId) { + this(new InetSocketAddress(ip, port), peerId); + } + + /** + * Instantiate a new peer. + * + * @param address The peer's address, with httpPort. + * @param peerId The byte-encoded peer ID. + */ + public Peer(InetSocketAddress address, ByteBuffer peerId) { + this.address = address; + this.hostId = String.format("%s:%d", + this.address.getAddress(), + this.address.getPort()); + + this.setPeerId(peerId); + } + + /** + * Tells whether this peer has a known peer ID yet or not. + */ + public boolean hasPeerId() { + return this.peerId != null; + } + + /** + * Returns the raw peer ID as a {@link ByteBuffer}. + */ + public ByteBuffer getPeerId() { + return this.peerId; + } + + /** + * Set a peer ID for this peer (usually during handshake). + * + * @param peerId The new peer ID for this peer. + */ + public void setPeerId(ByteBuffer peerId) { + if (peerId != null) { + this.peerId = peerId; + this.hexPeerId = Torrent.byteArrayToHexString(peerId.array()); + } else { + this.peerId = null; + this.hexPeerId = null; + } + } + + /** + * Get the hexadecimal-encoded string representation of this peer's ID. + */ + public String getHexPeerId() { + return this.hexPeerId; + } + + /** + * Get the shortened hexadecimal-encoded peer ID. + */ + public String getShortHexPeerId() { + return String.format("..%s", + this.hexPeerId.substring(this.hexPeerId.length()-6).toUpperCase()); + } + + /** + * Returns this peer's IP address. + */ + public String getIp() { + return this.address.getAddress().getHostAddress(); + } + + /** + * Returns this peer's InetAddress. + */ + public InetAddress getAddress() { + return this.address.getAddress(); + } + + /** + * Returns this peer's httpPort number. + */ + public int getPort() { + return this.address.getPort(); + } + + /** + * Returns this peer's host identifier ("host:httpPort"). + */ + public String getHostIdentifier() { + return this.hostId; + } + + /** + * Returns a binary representation of the peer's IP. + */ + public byte[] getRawIp() { + return this.address.getAddress().getAddress(); + } + + /** + * Returns a human-readable representation of this peer. + */ + public String toString() { + StringBuilder s = new StringBuilder("peer://") + .append(this.getIp()).append(":").append(this.getPort()) + .append("/"); + + if (this.hasPeerId()) { + s.append(this.hexPeerId.substring(this.hexPeerId.length()-6)); + } else { + s.append("?"); + } + + if (this.getPort() < 10000) { + s.append(" "); + } + + return s.toString(); + } + + /** + * Tells if two peers seem to look alike (i.e. they have the same IP, httpPort + * and peer ID if they have one). + */ + public boolean looksLike(Peer other) { + if (other == null) { + return false; + } + + return this.hostId.equals(other.hostId) && + (this.hasPeerId() + ? this.hexPeerId.equals(other.hexPeerId) + : true); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Torrent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Torrent.java new file mode 100644 index 0000000..9bb7a25 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/Torrent.java @@ -0,0 +1,823 @@ +/** + * Copyright (C) 2011-2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BDecoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEncoder; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A torrent file tracked by the controller's BitTorrent tracker. + * + *+ * This class represents an active torrent on the tracker. The torrent + * information is kept in-memory, and is created from the byte blob one would + * usually find in a .torrent file. + *
+ * + *+ * Each torrent also keeps a repository of the peers seeding and leeching this + * torrent from the tracker. + *
+ * + * @see Torrent meta-info file structure specification + */ +public class Torrent { + + private static final Logger logger = + LoggerFactory.getLogger(Torrent.class); + + public static final int DEFAULT_PIECE_LENGTH = 512 * 1024; + + public static final int PIECE_HASH_SIZE = 20; + + /** The query parameters encoding when parsing byte strings. */ + public static final String BYTE_ENCODING = "ISO-8859-1"; + + /** + * + * @author dgiffin + * @author mpetazzoni + */ + public static class TorrentFile { + + public final File file; + public final long size; + + public TorrentFile(File file, long size) { + this.file = file; + this.size = size; + } + } + + + protected final byte[] encoded; + protected final byte[] encoded_info; + protected final Map+ * If the torrent doesn't define an announce-list, use the mandatory + * announce field value as the single tracker in a single announce + * tier. Otherwise, the announce-list must be parsed and the trackers + * from each tier extracted. + *
+ * + * @see BitTorrent BEP#0012 "Multitracker Metadata Extension" + */ + try { + this.trackers = new ArrayList+ * For a single-file torrent, this is usually the name of the file. For a + * multi-file torrent, this is usually the name of a top-level directory + * containing those files. + *
+ */ + public String getName() { + return this.name; + } + + /** + * Get this torrent's comment string. + */ + public String getComment() { + return this.comment; + } + + /** + * Get this torrent's creator (user, software, whatever...). + */ + public String getCreatedBy() { + return this.createdBy; + } + + /** + * Get the total size of this torrent. + */ + public long getSize() { + return this.size; + } + + /** + * Get the file names from this torrent. + * + * @return The list of relative filenames of all the files described in + * this torrent. + */ + public List+ * The torrent's name is used. + *
+ */ + public String toString() { + return this.getName(); + } + + /** + * Return the B-encoded meta-info of this torrent. + */ + public byte[] getEncoded() { + return this.encoded; + } + + /** + * Return the trackers for this torrent. + */ + public List+ * If the environment variable TTORRENT_HASHING_THREADS is set to an + * integer value greater than 0, its value will be used. Otherwise, it + * defaults to the number of processors detected by the Java Runtime. + *
+ * + * @return How many threads to use for concurrent piece hashing. + */ + protected static int getHashingThreadsCount() { + String threads = System.getenv("TTORRENT_HASHING_THREADS"); + + if (threads != null) { + try { + int count = Integer.parseInt(threads); + if (count > 0) { + return count; + } + } catch (NumberFormatException nfe) { + // Pass + } + } + + return Runtime.getRuntime().availableProcessors(); + } + + /** Torrent loading ---------------------------------------------------- */ + + /** + * Load a torrent from the given torrent file. + * + *+ * This method assumes we are not a seeder and that local data needs to be + * validated. + *
+ * + * @param torrent The abstract {@link File} object representing the + * .torrent file to load. + * @throws IOException When the torrent file cannot be read. + */ + public static Torrent load(File torrent) throws IOException { + return Torrent.load(torrent, false); + } + + /** + * Load a torrent from the given torrent file. + * + * @param torrent The abstract {@link File} object representing the + * .torrent file to load. + * @param seeder Whether we are a seeder for this torrent or not (disables + * local data validation). + * @throws IOException When the torrent file cannot be read. + */ + public static Torrent load(File torrent, boolean seeder) + throws IOException { + byte[] data = FileUtils.readFileToByteArray(torrent); + return new Torrent(data, seeder); + } + + /** Torrent creation --------------------------------------------------- */ + + /** + * Create a {@link Torrent} object for a file. + * + *+ * Hash the given file to create the {@link Torrent} object representing + * the Torrent metainfo about this file, needed for announcing and/or + * sharing said file. + *
+ * + * @param source The file to use in the torrent. + * @param announce The announce URI that will be used for this torrent. + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + public static Torrent create(File source, URI announce, String createdBy) + throws InterruptedException, IOException { + return Torrent.create(source, null, DEFAULT_PIECE_LENGTH, + announce, null, createdBy); + } + + /** + * Create a {@link Torrent} object for a set of files. + * + *+ * Hash the given files to create the multi-file {@link Torrent} object + * representing the Torrent meta-info about them, needed for announcing + * and/or sharing these files. Since we created the torrent, we're + * considering we'll be a full initial seeder for it. + *
+ * + * @param parent The parent directory or location of the torrent files, + * also used as the torrent's name. + * @param files The files to add into this torrent. + * @param announce The announce URI that will be used for this torrent. + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + public static Torrent create(File parent, List+ * Hash the given file to create the {@link Torrent} object representing + * the Torrent metainfo about this file, needed for announcing and/or + * sharing said file. + *
+ * + * @param source The file to use in the torrent. + * @param announceList The announce URIs organized as tiers that will + * be used for this torrent + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + public static Torrent create(File source, int pieceLength, List+ * Hash the given files to create the multi-file {@link Torrent} object + * representing the Torrent meta-info about them, needed for announcing + * and/or sharing these files. Since we created the torrent, we're + * considering we'll be a full initial seeder for it. + *
+ * + * @param source The parent directory or location of the torrent files, + * also used as the torrent's name. + * @param files The files to add into this torrent. + * @param announceList The announce URIs organized as tiers that will + * be used for this torrent + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + public static Torrent create(File source, List+ * Hash the given files to create the multi-file {@link Torrent} object + * representing the Torrent meta-info about them, needed for announcing + * and/or sharing these files. Since we created the torrent, we're + * considering we'll be a full initial seeder for it. + *
+ * + * @param parent The parent directory or location of the torrent files, + * also used as the torrent's name. + * @param files The files to add into this torrent. + * @param announce The announce URI that will be used for this torrent. + * @param announceList The announce URIs organized as tiers that will + * be used for this torrent + * @param createdBy The creator's name, or any string identifying the + * torrent's creator. + */ + private static Torrent create(File parent, List+ * Hashes the given file piece by piece using the default Torrent piece + * length (see {@link #PIECE_LENGTH}) and returns the concatenation of + * these hashes, as a string. + *
+ * + *+ * This is used for creating Torrent meta-info structures from a file. + *
+ * + * @param file The file to hash. + */ + private static String hashFile(File file, int pieceLenght) + throws InterruptedException, IOException { + return Torrent.hashFiles(Arrays.asList(new File[] { file }), pieceLenght); + } + + private static String hashFiles(List+ * This class and its *Messages subclasses provide POJO + * representations of the peer protocol messages, along with easy parsing from + * an input ByteBuffer to quickly get a usable representation of an incoming + * message. + *
+ * + * @author mpetazzoni + * @see BitTorrent peer wire protocol + */ +public abstract class PeerMessage { + + /** The size, in bytes, of the length field in a message (one 32-bit + * integer). */ + public static final int MESSAGE_LENGTH_FIELD_SIZE = 4; + + /** + * Message type. + * + *+ * Note that the keep-alive messages don't actually have an type ID defined + * in the protocol as they are of length 0. + *
+ */ + public enum Type { + KEEP_ALIVE(-1), + CHOKE(0), + UNCHOKE(1), + INTERESTED(2), + NOT_INTERESTED(3), + HAVE(4), + BITFIELD(5), + REQUEST(6), + PIECE(7), + CANCEL(8); + + private byte id; + Type(int id) { + this.id = (byte)id; + } + + public boolean equals(byte c) { + return this.id == c; + } + + public byte getTypeByte() { + return this.id; + } + + public static Type get(byte c) { + for (Type t : Type.values()) { + if (t.equals(c)) { + return t; + } + } + return null; + } + }; + + private final Type type; + private final ByteBuffer data; + + private PeerMessage(Type type, ByteBuffer data) { + this.type = type; + this.data = data; + this.data.rewind(); + } + + public Type getType() { + return this.type; + } + + /** + * Returns a {@link ByteBuffer} backed by the same data as this message. + * + *+ * This method returns a duplicate of the buffer stored in this {@link + * PeerMessage} object to allow for multiple consumers to read from the + * same message without conflicting access to the buffer's position, mark + * and limit. + *
+ */ + public ByteBuffer getData() { + return this.data.duplicate(); + } + + /** + * Validate that this message makes sense for the torrent it's related to. + * + *+ * This method is meant to be overloaded by distinct message types, where + * it makes sense. Otherwise, it defaults to true. + *
+ * + * @param torrent The torrent this message is about. + */ + public PeerMessage validate(SharedTorrent torrent) + throws MessageValidationException { + return this; + } + + public String toString() { + return this.getType().name(); + } + + /** + * Parse the given buffer into a peer protocol message. + * + *+ * Parses the provided byte array and builds the corresponding PeerMessage + * subclass object. + *
+ * + * @param buffer The byte buffer containing the message data. + * @param torrent The torrent this message is about. + * @return A PeerMessage subclass instance. + * @throws ParseException When the message is invalid, can't be parsed or + * does not match the protocol requirements. + */ + public static PeerMessage parse(ByteBuffer buffer, SharedTorrent torrent) + throws ParseException { + int length = buffer.getInt(); + if (length == 0) { + return KeepAliveMessage.parse(buffer, torrent); + } else if (length != buffer.remaining()) { + throw new ParseException("Message size did not match announced " + + "size!", 0); + } + + Type type = Type.get(buffer.get()); + if (type == null) { + throw new ParseException("Unknown message ID!", + buffer.position()-1); + } + + switch (type) { + case CHOKE: + return ChokeMessage.parse(buffer.slice(), torrent); + case UNCHOKE: + return UnchokeMessage.parse(buffer.slice(), torrent); + case INTERESTED: + return InterestedMessage.parse(buffer.slice(), torrent); + case NOT_INTERESTED: + return NotInterestedMessage.parse(buffer.slice(), torrent); + case HAVE: + return HaveMessage.parse(buffer.slice(), torrent); + case BITFIELD: + return BitfieldMessage.parse(buffer.slice(), torrent); + case REQUEST: + return RequestMessage.parse(buffer.slice(), torrent); + case PIECE: + return PieceMessage.parse(buffer.slice(), torrent); + case CANCEL: + return CancelMessage.parse(buffer.slice(), torrent); + default: + throw new IllegalStateException("Message type should have " + + "been properly defined by now."); + } + } + + public static class MessageValidationException extends ParseException { + + static final long serialVersionUID = -1; + + public MessageValidationException(PeerMessage m) { + super("Message " + m + " is not valid!", 0); + } + + } + + + /** + * Keep alive message. + * + *+ * This class and its *TrackerMessage subclasses provide POJO + * representations of the tracker protocol messages, for at least HTTP and UDP + * trackers' protocols, along with easy parsing from an input ByteBuffer to + * quickly get a usable representation of an incoming message. + *
+ * + * @author mpetazzoni + */ +public abstract class TrackerMessage { + + /** + * Message type. + */ + public enum Type { + UNKNOWN(-1), + CONNECT_REQUEST(0), + CONNECT_RESPONSE(0), + ANNOUNCE_REQUEST(1), + ANNOUNCE_RESPONSE(1), + SCRAPE_REQUEST(2), + SCRAPE_RESPONSE(2), + ERROR(3); + + private final int id; + + Type(int id) { + this.id = id; + } + + public int getId() { + return this.id; + } + }; + + private final Type type; + private final ByteBuffer data; + + /** + * Constructor for the base tracker message type. + * + * @param type The message type. + * @param data A byte buffer containing the binary data of the message (a + * B-encoded map, a UDP packet data, etc.). + */ + protected TrackerMessage(Type type, ByteBuffer data) { + this.type = type; + this.data = data; + if (this.data != null) { + this.data.rewind(); + } + } + + /** + * Returns the type of this tracker message. + */ + public Type getType() { + return this.type; + } + + /** + * Returns the encoded binary data for this message. + */ + public ByteBuffer getData() { + return this.data; + } + + /** + * Generic exception for message format and message validation exceptions. + */ + public static class MessageValidationException extends Exception { + + static final long serialVersionUID = -1; + + public MessageValidationException(String s) { + super(s); + } + + public MessageValidationException(String s, Throwable cause) { + super(s, cause); + } + + } + + + /** + * Base interface for connection request messages. + * + *+ * This interface must be implemented by all subtypes of connection request + * messages for the various tracker protocols. + *
+ * + * @author mpetazzoni + */ + public interface ConnectionRequestMessage { + + }; + + + /** + * Base interface for connection response messages. + * + *+ * This interface must be implemented by all subtypes of connection + * response messages for the various tracker protocols. + *
+ * + * @author mpetazzoni + */ + public interface ConnectionResponseMessage { + + }; + + + /** + * Base interface for announce request messages. + * + *+ * This interface must be implemented by all subtypes of announce request + * messages for the various tracker protocols. + *
+ * + * @author mpetazzoni + */ + public interface AnnounceRequestMessage { + + public static final int DEFAULT_NUM_WANT = 50; + + /** + * Announce request event types. + * + *+ * When the client starts exchanging on a torrent, it must contact the + * torrent's tracker with a 'started' announce request, which notifies the + * tracker this client now exchanges on this torrent (and thus allows the + * tracker to report the existence of this peer to other clients). + *
+ * + *+ * When the client stops exchanging, or when its download completes, it must + * also send a specific announce request. Otherwise, the client must send an + * eventless (NONE), periodic announce request to the tracker at an + * interval specified by the tracker itself, allowing the tracker to + * refresh this peer's status and acknowledge that it is still there. + *
+ */ + public enum RequestEvent { + NONE(0), + COMPLETED(1), + STARTED(2), + STOPPED(3); + + private final int id; + RequestEvent(int id) { + this.id = id; + } + + public String getEventName() { + return this.name().toLowerCase(); + } + + public int getId() { + return this.id; + } + + public static RequestEvent getByName(String name) { + for (RequestEvent type : RequestEvent.values()) { + if (type.name().equalsIgnoreCase(name)) { + return type; + } + } + return null; + } + + public static RequestEvent getById(int id) { + for (RequestEvent type : RequestEvent.values()) { + if (type.getId() == id) { + return type; + } + } + return null; + } + }; + + public byte[] getInfoHash(); + public String getHexInfoHash(); + public byte[] getPeerId(); + public String getHexPeerId(); + public int getPort(); + public long getUploaded(); + public long getDownloaded(); + public long getLeft(); + public boolean getCompact(); + public boolean getNoPeerIds(); + public RequestEvent getEvent(); + + public String getIp(); + public int getNumWant(); + }; + + + /** + * Base interface for announce response messages. + * + *+ * This interface must be implemented by all subtypes of announce response + * messages for the various tracker protocols. + *
+ * + * @author mpetazzoni + */ + public interface AnnounceResponseMessage { + + public int getInterval(); + public int getComplete(); + public int getIncomplete(); + public List+ * This interface must be implemented by all subtypes of tracker error + * messages for the various tracker protocols. + *
+ * + * @author mpetazzoni + */ + public interface ErrorMessage { + + /** + * The various tracker error states. + * + *+ * These errors are reported by the tracker to a client when expected + * parameters or conditions are not present while processing an + * announce request from a BitTorrent client. + *
+ */ + public enum FailureReason { + UNKNOWN_TORRENT("The requested torrent does not exist on this tracker"), + MISSING_HASH("Missing info hash"), + MISSING_PEER_ID("Missing peer ID"), + MISSING_PORT("Missing httpPort"), + INVALID_EVENT("Unexpected event for peer state"), + NOT_IMPLEMENTED("Feature not implemented"); + + private String message; + + FailureReason(String message) { + this.message = message; + } + + public String getMessage() { + return this.message; + } + }; + + public String getReason(); + }; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceRequestMessage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceRequestMessage.java new file mode 100644 index 0000000..26f82b2 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/common/protocol/http/HTTPAnnounceRequestMessage.java @@ -0,0 +1,300 @@ +/** + * Copyright (C) 2012 Turn, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.broadcast.common.protocol.http; + +import org.apache.hadoop.yarn.server.broadcast.bcodec.BDecoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEValue; +import org.apache.hadoop.yarn.server.broadcast.bcodec.BEncoder; +import org.apache.hadoop.yarn.server.broadcast.bcodec.InvalidBEncodingException; +import org.apache.hadoop.yarn.server.broadcast.common.Peer; +import org.apache.hadoop.yarn.server.broadcast.common.Torrent; +import org.apache.hadoop.yarn.server.broadcast.common.protocol.TrackerMessage.AnnounceRequestMessage; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + + +/** + * The announce request message for the HTTP tracker protocol. + * + *+ * This class represents the announce request message in the HTTP tracker + * protocol. It doesn't add any specific fields compared to the generic + * announce request message, but it provides the means to parse such + * messages and craft them. + *
+ * + * @author mpetazzoni + */ +public class HTTPAnnounceRequestMessage extends HTTPTrackerMessage + implements AnnounceRequestMessage { + + private final byte[] infoHash; + private final Peer peer; + private final long uploaded; + private final long downloaded; + private final long left; + private final boolean compact; + private final boolean noPeerId; + private final RequestEvent event; + private final int numWant; + + private HTTPAnnounceRequestMessage(ByteBuffer data, + byte[] infoHash, Peer peer, long uploaded, long downloaded, + long left, boolean compact, boolean noPeerId, RequestEvent event, + int numWant) { + super(Type.ANNOUNCE_REQUEST, data); + this.infoHash = infoHash; + this.peer = peer; + this.downloaded = downloaded; + this.uploaded = uploaded; + this.left = left; + this.compact = compact; + this.noPeerId = noPeerId; + this.event = event; + this.numWant = numWant; + } + + @Override + public byte[] getInfoHash() { + return this.infoHash; + } + + @Override + public String getHexInfoHash() { + return Torrent.byteArrayToHexString(this.infoHash); + } + + @Override + public byte[] getPeerId() { + return this.peer.getPeerId().array(); + } + + @Override + public String getHexPeerId() { + return this.peer.getHexPeerId(); + } + + @Override + public int getPort() { + return this.peer.getPort(); + } + + @Override + public long getUploaded() { + return this.uploaded; + } + + @Override + public long getDownloaded() { + return this.downloaded; + } + + @Override + public long getLeft() { + return this.left; + } + + @Override + public boolean getCompact() { + return this.compact; + } + + @Override + public boolean getNoPeerIds() { + return this.noPeerId; + } + + @Override + public RequestEvent getEvent() { + return this.event; + } + + @Override + public String getIp() { + return this.peer.getIp(); + } + + @Override + public int getNumWant() { + return this.numWant; + } + + /** + * Build the announce request URL for the given tracker announce URL. + * + * @param trackerAnnounceURL The tracker's announce URL. + * @return The URL object representing the announce request URL. + */ + public URL buildAnnounceURL(URL trackerAnnounceURL) + throws UnsupportedEncodingException, MalformedURLException { + String base = trackerAnnounceURL.toString(); + StringBuilder url = new StringBuilder(base); + url.append(base.contains("?") ? "&" : "?") + .append("info_hash=") + .append(URLEncoder.encode( + new String(this.getInfoHash(), Torrent.BYTE_ENCODING), + Torrent.BYTE_ENCODING)) + .append("&peer_id=") + .append(URLEncoder.encode( + new String(this.getPeerId(), Torrent.BYTE_ENCODING), + Torrent.BYTE_ENCODING)) + .append("&httpPort=").append(this.getPort()) + .append("&uploaded=").append(this.getUploaded()) + .append("&downloaded=").append(this.getDownloaded()) + .append("&left=").append(this.getLeft()) + .append("&compact=").append(this.getCompact() ? 1 : 0) + .append("&no_peer_id=").append(this.getNoPeerIds() ? 1 : 0); + + if (this.getEvent() != null && + !RequestEvent.NONE.equals(this.getEvent())) { + url.append("&event=").append(this.getEvent().getEventName()); + } + + if (this.getIp() != null) { + url.append("&ip=").append(this.getIp()); + } + + return new URL(url.toString()); + } + + public static HTTPAnnounceRequestMessage parse(ByteBuffer data) + throws IOException, MessageValidationException { + BEValue decoded = BDecoder.bdecode(data); + if (decoded == null) { + throw new MessageValidationException( + "Could not decode tracker message (not B-encoded?)!"); + } + + Map+ * This method attemps to read and parse the incoming announce request into + * an announce request message, then creates the appropriate announce + * response message and sends it back to the client. + *
+ * @param request The incoming announce request. + * @param evt + */ + private void handleAnnounce(HttpRequest request, MessageEvent evt) throws IOException { + // Prepare the response headers. + Channel ch = evt.getChannel(); + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + response.addHeader(CONTENT_TYPE,"text/plain"); + response.addHeader(SERVER, version); + response.addHeader(DATE, System.currentTimeMillis()); + + /** + * Parse the query parameters into an announce request message. + * + * We need to rely on our own query parsing function because + * SimpleHTTP's Query map will contain UTF-8 decoded parameters, which + * doesn't work well for the byte-encoded strings we expect. + */ + HTTPAnnounceRequestMessage announceRequest = null; + try { + announceRequest = this.parseQuery(request, evt); + } catch (Exception e) { + e.printStackTrace(); + } + LOG.info("get announce request from " + ((InetSocketAddress)evt.getRemoteAddress()).getPort() + ",type: " + announceRequest.getEvent().getEventName()); + + // get torrent or notify client torrent is unknown by this tracker + TrackedTorrent torrent = torrents.get(announceRequest.getHexInfoHash()); + if (torrent == null) { + LOG.warn("Requested torrent hash was: " + announceRequest.getHexInfoHash()); + this.serveError(response, ch, BAD_REQUEST, ErrorMessage.FailureReason.UNKNOWN_TORRENT); + return; + } + + AnnounceRequestMessage.RequestEvent event = announceRequest.getEvent(); + String peerId = announceRequest.getHexPeerId(); + + // When no event is specified, it's a periodic update while the client + // is operating. If we don't have a peer for this announce, it means + // the tracker restarted while the client was running. Consider this + // announce request as a 'started' event. + if ((event == null || + AnnounceRequestMessage.RequestEvent.NONE.equals(event)) && + torrent.getPeer(peerId) == null) { + event = AnnounceRequestMessage.RequestEvent.STARTED; + } + + // If an event other than 'started' is specified and we also haven't + // seen the peer on this torrent before, something went wrong. A + // previous 'started' announce request should have been made by the + // client that would have had us register that peer on the torrent this + // request refers to. + if (event != null && torrent.getPeer(peerId) == null && + !AnnounceRequestMessage.RequestEvent.STARTED.equals(event)) { + this.serveError(response, ch, BAD_REQUEST, + ErrorMessage.FailureReason.INVALID_EVENT); + return; + } + + // if node is on the same rack but we didn't launch a client for this torrent, then launch a client + synchronized (infohashClientMap) { + LOG.info("map " + infohashClientMap); + if (topology.onSameRack(InetAddress.getLocalHost().getHostAddress(), ((InetSocketAddress) evt.getRemoteAddress()).getHostString()) && !infohashClientMap.containsKey(torrent.getHexInfoHash())) { + LOG.info("tracker launch a client for torrent " + torrent.getHexInfoHash()); + try { + Client c = new Client(InetAddress.getLocalHost(), torrent.getEncoded(), localDir); + infohashClientMap.put(torrent.getHexInfoHash(), c); + c.share(-1); + // FIXME track client for cleanup, need appid in either torrent or announcement + } catch (Exception e) { + e.printStackTrace(); + } + } else { + LOG.info("tracker don't need to launch a client for torrent " + torrent.getHexInfoHash()); + } + } + + // Update the torrent according to the announce event + TrackedPeer peer = null; + try { + peer = torrent.update(event, + ByteBuffer.wrap(announceRequest.getPeerId()), + announceRequest.getHexPeerId(), + announceRequest.getIp(), + announceRequest.getPort(), + announceRequest.getUploaded(), + announceRequest.getDownloaded(), + announceRequest.getLeft()); + } catch (IllegalArgumentException iae) { + this.serveError(response, ch, BAD_REQUEST, + ErrorMessage.FailureReason.INVALID_EVENT); + return; + } + + // Craft and output the answer + HTTPAnnounceResponseMessage announceResponse = null; + try { + announceResponse = HTTPAnnounceResponseMessage.craft( + torrent.getAnnounceInterval(), + TrackedTorrent.MIN_ANNOUNCE_INTERVAL_SECONDS, + version, + torrent.seeders(), + torrent.leechers(), + torrent.getSomePeers(peer)); + response.setContent(ChannelBuffers.wrappedBuffer(announceResponse.getData())); + } catch (Exception e) { + this.serveError(response, ch, INTERNAL_SERVER_ERROR, + e.getMessage()); + } + + evt.getChannel().write(response).addListener(ChannelFutureListener.CLOSE); + LOG.info("response write finished"); + } + + /** + * Parse the query parameters using our defined BYTE_ENCODING. + * + *+ * Because we're expecting byte-encoded strings as query parameters, we + * can't rely on SimpleHTTP's QueryParser which uses the wrong encoding for + * the job and returns us unparsable byte data. We thus have to implement + * our own little parsing method that uses BYTE_ENCODING to decode + * parameters from the URI. + *
+ * + *+ * Note: array parameters are not supported. If a key is present + * multiple times in the URI, the latest value prevails. We don't really + * need to implement this functionality as this never happens in the + * Tracker HTTP protocol. + *
+ * + * @param request The request's full URI, including query parameters. + * @param evt + * @return The {@link AnnounceRequestMessage} representing the client's + * announce request. + */ + private HTTPAnnounceRequestMessage parseQuery(HttpRequest request, MessageEvent evt) + throws IOException, MessageValidationException { + Map+ * Every PEER_COLLECTION_FREQUENCY_SECONDS, this thread will collect + * unfresh peers from all announced torrents. + *
+ */ + private class PeerCollectorThread extends Thread { + + private static final int PEER_COLLECTION_FREQUENCY_SECONDS = 15; + private boolean stop = false; + + @Override + public void run() { + LOG.info("Starting tracker peer collection for tracker at"); + + while (!stop) { + for (TrackedTorrent torrent : torrents.values()) { + torrent.collectUnfreshPeers(); + } + + try { + Thread.sleep(PeerCollectorThread + .PEER_COLLECTION_FREQUENCY_SECONDS * 1000); + } catch (InterruptedException ie) { + // Ignore + } + } + } + + public void setStop() { + this.stop = true; + } + } +} + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/Topology.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/Topology.java new file mode 100644 index 0000000..e99bd82 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-broadcast/src/main/java/org/apache/hadoop/yarn/server/broadcast/service/Topology.java @@ -0,0 +1,124 @@ +package org.apache.hadoop.yarn.server.broadcast.service; + + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.broadcast.client.announce.AnnounceException; + +import java.io.*; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; + +/** + * parse topology input and offer consistent hashing functionality + * FIXME if there is no topology input or topology input is not up to date, this won't work well + */ +public class Topology { + private static Log LOG = LogFactory.getLog(Topology.class); + /* hash to name dictionary */ + private Map+ * Represents a peer exchanging on a given torrent. In this implementation, + * we don't really care about the status of the peers and how much they + * have downloaded / exchanged because we are not a torrent exchange and + * don't need to keep track of what peers are doing while they're + * downloading. We only care about when they start, and when they are done. + *
+ * + *+ * We also never expire peers automatically. Unless peers send a STOPPED + * announce request, they remain as long as the torrent object they are a + * part of. + *
+ */ +public class TrackedPeer extends Peer { + + private static final Logger logger = + LoggerFactory.getLogger(TrackedPeer.class); + + private static final int FRESH_TIME_SECONDS = 30; + + private long uploaded; + private long downloaded; + private long left; + private Torrent torrent; + + /** + * Represents the state of a peer exchanging on this torrent. + * + *+ * Peers can be in the STARTED state, meaning they have announced + * themselves to us and are eventually exchanging data with other peers. + * Note that a peer starting with a completed file will also be in the + * started state and will never notify as being in the completed state. + * This information can be inferred from the fact that the peer reports 0 + * bytes left to download. + *
+ * + *+ * Peers enter the COMPLETED state when they announce they have entirely + * downloaded the file. As stated above, we may also elect them for this + * state if they report 0 bytes left to download. + *
+ * + *+ * Peers enter the STOPPED state very briefly before being removed. We + * still pass them to the STOPPED state in case someone else kept a + * reference on them. + *
+ */ + public enum PeerState { + UNKNOWN, + STARTED, + COMPLETED, + STOPPED; + }; + + private PeerState state; + private Date lastAnnounce; + + /** + * Instantiate a new tracked peer for the given torrent. + * + * @param torrent The torrent this peer exchanges on. + * @param ip The peer's IP address. + * @param port The peer's port. + * @param peerId The byte-encoded peer ID. + */ + public TrackedPeer(Torrent torrent, String ip, int port, + ByteBuffer peerId) { + super(ip, port, peerId); + this.torrent = torrent; + + // Instantiated peers start in the UNKNOWN state. + this.state = PeerState.UNKNOWN; + this.lastAnnounce = null; + + this.uploaded = 0; + this.downloaded = 0; + this.left = 0; + } + + /** + * Update this peer's state and information. + * + *+ * Note: if the peer reports 0 bytes left to download, its state will + * be automatically be set to COMPLETED. + *
+ * + * @param state The peer's state. + * @param uploaded Uploaded byte count, as reported by the peer. + * @param downloaded Downloaded byte count, as reported by the peer. + * @param left Left-to-download byte count, as reported by the peer. + */ + public void update(PeerState state, long uploaded, long downloaded, + long left) { + if (PeerState.STARTED.equals(state) && left == 0) { + state = PeerState.COMPLETED; + } + + if (!state.equals(this.state)) { + logger.info("Peer {} {} download of {}.", + new Object[] { + this, + state.name().toLowerCase(), + this.torrent, + }); + } + + this.state = state; + this.lastAnnounce = new Date(); + this.uploaded = uploaded; + this.downloaded = downloaded; + this.left = left; + } + + /** + * Tells whether this peer has completed its download and can thus be + * considered a seeder. + */ + public boolean isCompleted() { + return PeerState.COMPLETED.equals(this.state); + } + + /** + * Returns how many bytes the peer reported it has uploaded so far. + */ + public long getUploaded() { + return this.uploaded; + } + + /** + * Returns how many bytes the peer reported it has downloaded so far. + */ + public long getDownloaded() { + return this.downloaded; + } + + /** + * Returns how many bytes the peer reported it needs to retrieve before + * its download is complete. + */ + public long getLeft() { + return this.left; + } + + /** + * Tells whether this peer has checked in with the tracker recently. + * + *+ * Non-fresh peers are automatically terminated and collected by the + * Tracker. + *
+ */ + public boolean isFresh() { + return (this.lastAnnounce != null && + (this.lastAnnounce.getTime() + (FRESH_TIME_SECONDS * 1000) > + new Date().getTime())); + } + + /** + * Returns a BEValue representing this peer for inclusion in an + * announce reply from the tracker. + * + * The returned BEValue is a dictionary containing the peer ID (in its + * original byte-encoded form), the peer's IP and the peer's port. + */ + public BEValue toBEValue() throws UnsupportedEncodingException { + Map+ * {@link TrackedTorrent} objects are used by the BitTorrent tracker to + * represent a torrent that is announced by the tracker. As such, it is not + * expected to point to any valid local data like. It also contains some + * additional information used by the tracker to keep track of which peers + * exchange on it, etc. + *
+ * + * @author mpetazzoni + */ +public class TrackedTorrent extends Torrent { + + private static final Logger logger = + LoggerFactory.getLogger(TrackedTorrent.class); + + /** Minimum announce interval requested from peers, in seconds. */ + public static final int MIN_ANNOUNCE_INTERVAL_SECONDS = 5; + + /** Default number of peers included in a tracker response. */ + private static final int DEFAULT_ANSWER_NUM_PEERS = 30; + + /** Default announce interval requested from peers, in seconds. */ + private static final int DEFAULT_ANNOUNCE_INTERVAL_SECONDS = 10; + + private int answerPeers; + private int announceInterval; + + /** Peers currently exchanging on this torrent. */ + private ConcurrentMap+ * Collect and remove all non-fresh peers from this torrent. This is + * usually called by the periodic peer collector of the BitTorrent tracker. + *
+ */ + public void collectUnfreshPeers() { + for (TrackedPeer peer : this.peers.values()) { + if (!peer.isFresh()) { + this.peers.remove(peer.getHexPeerId()); + } + } + } + + /** + * Get the announce interval for this torrent. + */ + public int getAnnounceInterval() { + return this.announceInterval; + } + + /** + * Set the announce interval for this torrent. + * + * @param interval New announce interval, in seconds. + */ + public void setAnnounceInterval(int interval) { + if (interval <= 0) { + throw new IllegalArgumentException("Invalid announce interval"); + } + + this.announceInterval = interval; + } + + /** + * Update this torrent's swarm from an announce event. + * + *+ * This will automatically create a new peer on a 'started' announce event, + * and remove the peer on a 'stopped' announce event. + *
+ * + * @param event The reported event. If null, means a regular + * interval announce event, as defined in the BitTorrent specification. + * @param peerId The byte-encoded peer ID. + * @param hexPeerId The hexadecimal representation of the peer's ID. + * @param ip The peer's IP address. + * @param port The peer's inbound port. + * @param uploaded The peer's reported uploaded byte count. + * @param downloaded The peer's reported downloaded byte count. + * @param left The peer's reported left to download byte count. + * @return The peer that sent us the announce request. + */ + public TrackedPeer update(RequestEvent event, ByteBuffer peerId, + String hexPeerId, String ip, int port, long uploaded, long downloaded, + long left) throws UnsupportedEncodingException { + TrackedPeer peer; + TrackedPeer.PeerState state = TrackedPeer.PeerState.UNKNOWN; + + if (RequestEvent.STARTED.equals(event)) { + peer = new TrackedPeer(this, ip, port, peerId); + state = TrackedPeer.PeerState.STARTED; + this.addPeer(peer); + } else if (RequestEvent.STOPPED.equals(event)) { + peer = this.removePeer(hexPeerId); + state = TrackedPeer.PeerState.STOPPED; + } else if (RequestEvent.COMPLETED.equals(event)) { + peer = this.getPeer(hexPeerId); + state = TrackedPeer.PeerState.COMPLETED; + } else if (RequestEvent.NONE.equals(event)) { + peer = this.getPeer(hexPeerId); + state = TrackedPeer.PeerState.STARTED; + } else { + throw new IllegalArgumentException("Unexpected announce event type!"); + } + + peer.update(state, uploaded, downloaded, left); + return peer; + } + + /** + * Get a list of peers we can return in an announce response for this + * torrent. + * + * @param peer The peer making the request, so we can exclude it from the + * list of returned peers. + * @return A list of peers we can include in an announce response. + */ + public List