-
Type:
Sub-task
-
Status: Resolved
-
Priority:
Major
-
Resolution: Fixed
-
Affects Version/s: None
-
Fix Version/s: 1.1.0
-
Component/s: Shuffle, Spark Core
-
Labels:None
ShuffleCopier.getBlock gets one block at a time (in one request).
def getBlock(host: String, port: Int, blockId: BlockId, resultCollectCallback: (BlockId, Long, ByteBuf) => Unit) { val handler = new ShuffleCopier.ShuffleClientHandler(resultCollectCallback) val connectTimeout = conf.getInt("spark.shuffle.netty.connect.timeout", 60000) val fc = new FileClient(handler, connectTimeout) try { fc.init() fc.connect(host, port) fc.sendRequest(blockId.name) fc.waitForClose() fc.close() } catch { // Handle any socket-related exceptions in FileClient case e: Exception => { logError("Shuffle copy of block " + blockId + " from " + host + ":" + port + " failed", e) handler.handleError(blockId) } } }
- links to