diff --git llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java index fb1bcfedc5..63c91dacf9 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tez.runtime.library.common.Constants; import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord; @@ -42,12 +43,22 @@ private final LinkedBlockingQueue queue = new LinkedBlockingQueue(); + private FileSystem fs; public static final String INDEX_CACHE_MB = "llap.shuffle.indexcache.mb"; public IndexCache(Configuration conf) { this.conf = conf; totalMemoryAllowed = conf.getInt(INDEX_CACHE_MB, 10) * 1024 * 1024; LOG.info("IndexCache created with max memory = " + totalMemoryAllowed); + initLocalFs(); + } + + private void initLocalFs() { + try { + this.fs = FileSystem.getLocal(conf).getRaw(); + } catch (IOException e) { + throw new RuntimeException(e); + } } /** @@ -119,7 +130,7 @@ private IndexInformation readIndexFileToCache(Path indexFileName, LOG.debug("IndexCache MISS: MapId " + mapId + " not found") ; TezSpillRecord tmp = null; try { - tmp = new TezSpillRecord(indexFileName, conf, expectedIndexOwner); + tmp = new TezSpillRecord(indexFileName, fs, expectedIndexOwner); } catch (Throwable e) { tmp = new TezSpillRecord(0); cache.remove(mapId);