diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index ed04555..cc9dea7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -40,5 +40,9 @@ org.apache.hadoop hadoop-yarn-server-common + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/HistoryWriter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/HistoryWriter.java new file mode 100644 index 0000000..032d09e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/HistoryWriter.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * The base class of writing the application history + * + */ +public abstract class HistoryWriter { + + public static final Log LOG = LogFactory.getLog(HistoryWriter.class); + + protected ExecutorService taskRunner; + + public HistoryWriter() { + taskRunner = Executors.newCachedThreadPool(new ThreadFactoryBuilder() + .setNameFormat("WritingHistoryTask #%d").build()); + } + + public void writeApplicationAsync(RMApp app) { + taskRunner.submit(new WritingHistoryTask( + new WritingAppHistoryEvent(app))); + } + + protected abstract void writeApplication(RMApp app) throws Exception; + + public void writeApplicationAttemptAsync(RMAppAttempt appAttempt) { + taskRunner.submit(new WritingHistoryTask( + new WritingAppAttemptHistoryEvent(appAttempt))); + } + + protected abstract void writeApplicationAttempt( + RMAppAttempt app) throws Exception; + + public void writeContainerAsync(RMContainer container) { + taskRunner.submit(new WritingHistoryTask( + new WritingContainerHistoryEvent(container))); + } + + protected abstract void writeContainer( + RMContainer container) throws Exception; + + /** + * The worker thread of executing the writing history task + * + */ + private final class WritingHistoryTask implements Callable { + + private WritingHistoryEvent event; + + public WritingHistoryTask(WritingHistoryEvent event) { + this.event = event; + } + + @Override + public Throwable call() { + try { + switch(event.getType()) { + case WRITING_APP: + WritingAppHistoryEvent appEvent = (WritingAppHistoryEvent) event; + writeApplication(appEvent.getApplication()); + break; + case WRITING_APP_ATTEMPT: + WritingAppAttemptHistoryEvent appAttemptEvent = + (WritingAppAttemptHistoryEvent) event; + writeApplicationAttempt(appAttemptEvent.getApplicationAttempt()); + break; + case WRITING_CONTAINER: + WritingContainerHistoryEvent containerEvent = + (WritingContainerHistoryEvent) event; + writeContainer(containerEvent.getContainer()); + break; + default: + LOG.error("Unknown WritingHistoryEvent type: " + event.getType()); + } + } catch (Throwable t) { + return t; + } + return null; + } + + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingAppAttemptHistoryEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingAppAttemptHistoryEvent.java new file mode 100644 index 0000000..8bd30f9 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingAppAttemptHistoryEvent.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; + + +public class WritingAppAttemptHistoryEvent extends WritingHistoryEvent { + + private RMAppAttempt appAttempt; + + public WritingAppAttemptHistoryEvent(RMAppAttempt appAttempt) { + super(WritingHistoryEventType.WRITING_APP_ATTEMPT); + this.appAttempt = appAttempt; + } + + public RMAppAttempt getApplicationAttempt() { + return appAttempt; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingAppHistoryEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingAppHistoryEvent.java new file mode 100644 index 0000000..51d43b5 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingAppHistoryEvent.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; + + +public class WritingAppHistoryEvent extends WritingHistoryEvent { + + private RMApp app; + + public WritingAppHistoryEvent(RMApp app) { + super(WritingHistoryEventType.WRITING_APP); + this.app = app; + } + + public RMApp getApplication() { + return app; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingContainerHistoryEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingContainerHistoryEvent.java new file mode 100644 index 0000000..c7ec930 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingContainerHistoryEvent.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; + + +public class WritingContainerHistoryEvent extends WritingHistoryEvent { + + private RMContainer container; + + public WritingContainerHistoryEvent(RMContainer container) { + super(WritingHistoryEventType.WRITING_CONTAINER); + this.container = container; + } + + public RMContainer getContainer() { + return container; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingHistoryEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingHistoryEvent.java new file mode 100644 index 0000000..486aa06 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingHistoryEvent.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.yarn.event.AbstractEvent; + + +public class WritingHistoryEvent extends AbstractEvent { + + public WritingHistoryEvent(WritingHistoryEventType type) { + super(type); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingHistoryEventType.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingHistoryEventType.java new file mode 100644 index 0000000..70443cf --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/WritingHistoryEventType.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + + +public enum WritingHistoryEventType { + WRITING_APP, + WRITING_APP_ATTEMPT, + WRITING_CONTAINER +}