diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d50912b4e2..a29cb557dd 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4173,7 +4173,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "This is recommended to be used along-side NVDIMM (DAX) or NVMe flash storage."), LLAP_ALLOCATOR_MAPPED_PATH("hive.llap.io.allocator.mmap.path", "/tmp", new WritableDirectoryValidator(), - "The directory location for mapping NVDIMM/NVMe flash storage into the ORC low-level cache."), + "The directory of comma separated location for \n" + + "mapping NVDIMM/NVMe flash storage into the ORC low-level cache."), LLAP_ALLOCATOR_DISCARD_METHOD("hive.llap.io.allocator.discard.method", "both", new StringSet("freelist", "brute", "both"), "Which method to use to force-evict blocks to deal with fragmentation:\n" + diff --git a/common/src/java/org/apache/hadoop/hive/conf/Validator.java b/common/src/java/org/apache/hadoop/hive/conf/Validator.java index 1539a35d62..64d1b9cde1 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Validator.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Validator.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.conf; +import org.apache.commons.lang3.StringUtils; + import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; @@ -354,17 +356,23 @@ private String sizeString(long size) { @Override public String validate(String value) { - final Path path = FileSystems.getDefault().getPath(value); - if (path == null && value != null) { + if (StringUtils.isEmpty(value)) { return String.format("Path '%s' provided could not be located.", value); } - final boolean isDir = Files.isDirectory(path); - final boolean isWritable = Files.isWritable(path); - if (!isDir) { - return String.format("Path '%s' provided is not a directory.", value); - } - if (!isWritable) { - return String.format("Path '%s' provided is not writable.", value); + String[] allPaths = org.apache.hadoop.util.StringUtils.getStrings(value); + for (int i = 0; i < allPaths.length; i++) { + final Path path = FileSystems.getDefault().getPath(StringUtils.deleteWhitespace(allPaths[i])); + if (path == null && allPaths[i] != null) { + return String.format("Path '%s' provided could not be located.", allPaths[i]); + } + final boolean isDir = Files.isDirectory(path); + final boolean isWritable = Files.isWritable(path); + if (!isDir) { + return String.format("Path '%s' provided is not a directory.", allPaths[i]); + } + if (!isWritable) { + return String.format("Path '%s' provided is not writable.", allPaths[i]); + } } return null; } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java index 341da252c4..115aff4392 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.llap.cache; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -40,6 +42,7 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; import org.apache.hadoop.hive.conf.HiveConf; @@ -80,13 +83,13 @@ private static final int MAX_DISCARD_ATTEMPTS = 10, LOG_DISCARD_ATTEMPTS = 5; // Config settings - private final int minAllocLog2, maxAllocLog2, arenaSizeLog2, maxArenas; + private final int minAllocLog2, maxAllocLog2, arenaSizeLog2, maxArenas, maxArenasPerLoc; private final int minAllocation, maxAllocation, arenaSize; private final long maxSize; private final boolean isDirect; private final boolean isMapped; private final int maxForcedEvictionSize; - private final Path cacheDir; + private final List cacheDir = new ArrayList<>(); // These are only used for tests. private boolean enableDefragShortcut = true, oomLogging = true; @@ -190,19 +193,19 @@ public BuddyAllocator(boolean isDirectVal, boolean isMappedVal, int minAllocVal, maxAllocation = maxAllocVal; if (isMapped) { try { - Path path = FileSystems.getDefault().getPath(mapPath); - if (!Files.exists(path)) { - Files.createDirectory(path); + String[] allMapPaths = org.apache.hadoop.util.StringUtils.getStrings(mapPath); + for (int i = 0; i < allMapPaths.length; i++) { + Path path = FileSystems.getDefault().getPath(StringUtils.deleteWhitespace(allMapPaths[i])); + if (!Files.exists(path)) { + Files.createDirectory(path); + } + cacheDir.add(Files.createTempDirectory(path, "llap-", RWX)); } - cacheDir = Files.createTempDirectory(path, "llap-", RWX); } catch (IOException ioe) { // conf validator already checks this, so it will never trigger usually throw new AssertionError("Configured mmap directory should be writable", ioe); } - } else { - cacheDir = null; } - arenaSize = validateAndDetermineArenaSize(arenaCount, maxSizeVal); maxSize = validateAndDetermineMaxSize(maxSizeVal); memoryManager.updateMaxSize(determineMaxMmSize(defragHeadroom, maxSize)); @@ -211,6 +214,8 @@ public BuddyAllocator(boolean isDirectVal, boolean isMappedVal, int minAllocVal, maxAllocLog2 = 31 - Integer.numberOfLeadingZeros(maxAllocation); arenaSizeLog2 = 63 - Long.numberOfLeadingZeros(arenaSize); maxArenas = (int)(maxSize / arenaSize); + // Assuming cache size for every NVMe-SSD disks are same + maxArenasPerLoc = maxArenas/(Math.max(1, cacheDir.size())); arenas = new Arena[maxArenas]; for (int i = 0; i < maxArenas; ++i) { arenas[i] = new Arena(); @@ -851,13 +856,13 @@ public boolean isDirectAlloc() { return isDirect; } - private ByteBuffer preallocateArenaBuffer(int arenaSize) { + private ByteBuffer preallocateArenaBuffer(int arenaIx, int arenaSize) { if (isMapped) { RandomAccessFile rwf = null; File rf = null; Preconditions.checkArgument(isDirect, "All memory mapped allocations have to be direct buffers"); try { - rf = File.createTempFile("arena-", ".cache", cacheDir.toFile()); + rf = File.createTempFile("arena-", ".cache", cacheDir.get(arenaIx/maxArenasPerLoc).toFile()); rwf = new RandomAccessFile(rf, "rw"); rwf.setLength(arenaSize); // truncate (TODO: posix_fallocate?) // Use RW, not PRIVATE because the copy-on-write is irrelevant for a deleted file @@ -895,7 +900,7 @@ private ByteBuffer preallocateArenaBuffer(int arenaSize) { void init(int arenaIx) { this.arenaIx = arenaIx; try { - data = preallocateArenaBuffer(arenaSize); + data = preallocateArenaBuffer(arenaIx, arenaSize); } catch (OutOfMemoryError oom) { throw new OutOfMemoryError("Cannot allocate " + arenaSize + " bytes: " + oom.getMessage() + "; make sure your xmx and process size are set correctly."); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java index 4662139593..0b9a353555 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java @@ -106,6 +106,19 @@ public void testSameSizes() throws Exception { allocSameSize(a, arenaCount * 2, allocLog2); } + @Test + public void testMultipleSSDLocation() throws Exception { + int max = 8, maxAlloc = 1 << max, allocLog2 = max - 1, arenaCount = 10; + String baseLoc = tmpDir; + String multiplePaths = baseLoc + "tmp1"; + multiplePaths = multiplePaths + "," + baseLoc + "tmp2"; + BuddyAllocator a = new BuddyAllocator(isDirect, isMapped, 1 << 3, maxAlloc, maxAlloc, + maxAlloc * arenaCount, 0, multiplePaths, new DummyMemoryManager(), + LlapDaemonCacheMetrics.create("test", "1"), null, true); + allocSameSize(a, arenaCount * 2, allocLog2); + } + + @Test public void testMTT() { final int min = 3, max = 8, maxAlloc = 1 << max, allocsPerSize = 3;