diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index 047f401..31b0e5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -82,10 +82,8 @@ static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT = YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir"; - @VisibleForTesting /** Default value for storage location on local disk. */ - static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT = - "/tmp/timeline_service_data"; + private static final String STORAGE_DIR_ROOT = "timeline_service_data"; private final CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN"); @@ -159,13 +157,13 @@ private static void fillFields(TimelineEntity finalEntity, private String getFlowRunPath(String userId, String clusterId, String flowName, Long flowRunId, String appId) throws IOException { if (userId != null && flowName != null && flowRunId != null) { - return userId + "/" + flowName + "/" + flowRunId; + return userId + File.separator + flowName + File.separator + flowRunId; } if (clusterId == null || appId == null) { throw new IOException("Unable to get flow info"); } - String appFlowMappingFile = rootPath + "/" + ENTITIES_DIR + "/" + - clusterId + "/" + APP_FLOW_MAPPING_FILE; + String appFlowMappingFile = rootPath + File.separator + ENTITIES_DIR + + File.separator + clusterId + File.separator + APP_FLOW_MAPPING_FILE; try (BufferedReader reader = new BufferedReader(new InputStreamReader( new FileInputStream( @@ -180,8 +178,8 @@ private String getFlowRunPath(String userId, String clusterId, !applicationId.trim().equals(appId)) { continue; } - return record.get(1).trim() + "/" + record.get(2).trim() + "/" + - record.get(3).trim(); + return record.get(1).trim() + File.separator + record.get(2).trim() + + File.separator + record.get(3).trim(); } parser.close(); } @@ -364,7 +362,7 @@ public int compare(Long l1, Long l2) { @Override public void serviceInit(Configuration conf) throws Exception { rootPath = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT, - DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT); + conf.get("hadoop.tmp.dir") + File.separator + STORAGE_DIR_ROOT); super.serviceInit(conf); } @@ -375,8 +373,8 @@ public TimelineEntity getEntity(TimelineReaderContext context, context.getClusterId(), context.getFlowName(), context.getFlowRunId(), context.getAppId()); File dir = new File(new File(rootPath, ENTITIES_DIR), - context.getClusterId() + "/" + flowRunPath + "/" + context.getAppId() + - "/" + context.getEntityType()); + context.getClusterId() + File.separator + flowRunPath + File.separator + + context.getAppId() + File.separator + context.getEntityType()); File entityFile = new File( dir, context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION); try (BufferedReader reader = @@ -401,8 +399,9 @@ public TimelineEntity getEntity(TimelineReaderContext context, context.getAppId()); File dir = new File(new File(rootPath, ENTITIES_DIR), - context.getClusterId() + "/" + flowRunPath + "/" + - context.getAppId() + "/" + context.getEntityType()); + context.getClusterId() + File.separator + flowRunPath + + File.separator + context.getAppId() + File.separator + + context.getEntityType()); return getEntities(dir, context.getEntityType(), filters, dataToRetrieve); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java index 1bb77a0..136a61f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java @@ -53,15 +53,14 @@ public static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT = YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir"; - /** default value for storage location on local disk. */ - public static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT - = "/tmp/timeline_service_data"; - public static final String ENTITIES_DIR = "entities"; /** Default extension for output files. */ public static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist"; + /** default value for storage location on local disk. */ + private static final String STORAGE_DIR_ROOT = "timeline_service_data"; + FileSystemTimelineWriterImpl() { super((FileSystemTimelineWriterImpl.class.getName())); } @@ -124,7 +123,7 @@ public String getOutputRoot() { @Override public void serviceInit(Configuration conf) throws Exception { outputRoot = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT, - DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT); + conf.get("hadoop.tmp.dir") + File.separator + STORAGE_DIR_ROOT); } @Override @@ -140,7 +139,7 @@ public void flush() throws IOException { private static String mkdirs(String... dirStrs) throws IOException { StringBuilder path = new StringBuilder(); for (String dirStr : dirStrs) { - path.append(dirStr).append('/'); + path.append(dirStr).append(File.separatorChar); File dir = new File(path.toString()); if (!dir.exists()) { if (!dir.mkdirs()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java index b58bbe3..dfcc1ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java @@ -43,32 +43,32 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp; -import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter; -import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; public class TestFileSystemTimelineReaderImpl { - private static final String ROOT_DIR = - FileSystemTimelineReaderImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT; - private FileSystemTimelineReaderImpl reader; + private static String rootDir; + private static FileSystemTimelineReaderImpl reader; @BeforeClass public static void setup() throws Exception { + init(); // initialize the reader and rootDir loadEntityData(); // Create app flow mapping file. CSVFormat format = CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN"); - String appFlowMappingFile = ROOT_DIR + "/entities/cluster1/" + + String appFlowMappingFile = rootDir + File.separator + "entities" + + File.separator + "cluster1" + File.separator + FileSystemTimelineReaderImpl.APP_FLOW_MAPPING_FILE; try (PrintWriter out = new PrintWriter(new BufferedWriter( @@ -78,21 +78,19 @@ public static void setup() throws Exception { printer.printRecord("app2", "user1", "flow1,flow", 1); printer.close(); } - (new File(ROOT_DIR)).deleteOnExit(); + (new File(rootDir)).deleteOnExit(); } @AfterClass public static void tearDown() throws Exception { - FileUtils.deleteDirectory(new File(ROOT_DIR)); + FileUtils.deleteDirectory(new File(rootDir)); } - @Before - public void init() throws Exception { + private static void init() throws Exception { reader = new FileSystemTimelineReaderImpl(); Configuration conf = new YarnConfiguration(); - conf.set(FileSystemTimelineReaderImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT, - ROOT_DIR); reader.init(conf); + rootDir = reader.getRootPath(); } private static void writeEntityFile(TimelineEntity entity, File dir) @@ -102,7 +100,8 @@ private static void writeEntityFile(TimelineEntity entity, File dir) throw new IOException("Could not create directories for " + dir); } } - String fileName = dir.getAbsolutePath() + "/" + entity.getId() + ".thist"; + String fileName = dir.getAbsolutePath() + File.separator + entity.getId() + + ".thist"; try (PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(fileName, true)))){ out.println(TimelineUtils.dumpTimelineRecordtoJSON(entity)); @@ -112,8 +111,10 @@ private static void writeEntityFile(TimelineEntity entity, File dir) } private static void loadEntityData() throws Exception { - File appDir = new File(ROOT_DIR + - "/entities/cluster1/user1/flow1/1/app1/app/"); + File appDir = new File(rootDir + File.separator + "entities" + + File.separator + "cluster1" + File.separator + "user1" + + File.separator + "flow1" + File.separator + "1" + File.separator + + "app1" + File.separator + "app" + File.separator); TimelineEntity entity11 = new TimelineEntity(); entity11.setId("id_1"); entity11.setType("app"); @@ -254,8 +255,10 @@ private static void loadEntityData() throws Exception { entity4.addEvent(event44); writeEntityFile(entity4, appDir); - File appDir2 = new File(ROOT_DIR + - "/entities/cluster1/user1/flow1,flow/1/app2/app/"); + File appDir2 = new File(rootDir + File.separator + "entities" + + File.separator + "cluster1" + File.separator + "user1" + + File.separator + "flow1,flow" + File.separator + "1" + File.separator + + "app2" + File.separator + "app" + File.separator); TimelineEntity entity5 = new TimelineEntity(); entity5.setId("id_5"); entity5.setType("app");