diff --git hadoop-project/pom.xml hadoop-project/pom.xml index 3c49182..7764eaa 100644 --- hadoop-project/pom.xml +++ hadoop-project/pom.xml @@ -428,6 +428,12 @@ org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + ${project.version} + + + + org.apache.hadoop hadoop-yarn-services-core ${project.version} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml new file mode 100644 index 0000000..d0dfde2 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml @@ -0,0 +1,98 @@ + + + + + hadoop-yarn-server + org.apache.hadoop + 3.1.0-SNAPSHOT + + 4.0.0 + org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + 3.1.0-SNAPSHOT + hadoop-yarn-server-globalpolicygenerator + + + + ${project.parent.basedir} + + + + + + org.apache.hadoop + hadoop-yarn-server-common + + + + org.apache.hadoop + hadoop-common + + + + org.apache.hadoop + hadoop-yarn-api + + + + org.apache.hadoop + hadoop-yarn-common + + + + org.apache.hadoop + hadoop-common + test-jar + test + + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + + junit + junit + test + + + + org.apache.hadoop + hadoop-yarn-server-common + test-jar + test + + + + org.hsqldb + hsqldb + test + + + + + + + + org.apache.rat + apache-rat-plugin + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java new file mode 100644 index 0000000..fc4bef9 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; + +public interface GPGContext { + + FederationStateStoreFacade getStateStoreFacade(); + + void setStateStoreFacade(FederationStateStoreFacade facade); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java new file mode 100644 index 0000000..aa1d5ba --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; + +public class GPGContextImpl implements GPGContext { + + private FederationStateStoreFacade facade; + + @Override + public FederationStateStoreFacade getStateStoreFacade() { + return facade; + } + + @Override + public void setStateStoreFacade(FederationStateStoreFacade facade) { + this.facade = facade; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java new file mode 100644 index 0000000..17dd28c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GlobalPolicyGenerator extends CompositeService { + + public static final Logger LOG = + LoggerFactory.getLogger(GlobalPolicyGenerator.class); + + // YARN Variables + private static CompositeServiceShutdownHook gpgShutdownHook; + public static final int SHUTDOWN_HOOK_PRIORITY = 30; + private AtomicBoolean isStopping = new AtomicBoolean(false); + private static final String METRICS_NAME = "Global Policy Generator"; + + // Federation Variables + private GPGContext gpgContext; + + public GlobalPolicyGenerator() { + super(GlobalPolicyGenerator.class.getName()); + this.gpgContext = new GPGContextImpl(); + } + + protected void initAndStart(Configuration conf, boolean hasToReboot) { + try { + // Remove the old hook if we are rebooting. + if (hasToReboot && null != gpgShutdownHook) { + ShutdownHookManager.get().removeShutdownHook(gpgShutdownHook); + } + + gpgShutdownHook = new CompositeServiceShutdownHook(this); + ShutdownHookManager.get().addShutdownHook(gpgShutdownHook, + SHUTDOWN_HOOK_PRIORITY); + + this.init(conf); + this.start(); + } catch (Throwable t) { + LOG.error("Error starting globalpolicygenerator", t); + System.exit(-1); + } + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + // Set up the context + this.gpgContext + .setStateStoreFacade(FederationStateStoreFacade.getInstance()); + + DefaultMetricsSystem.initialize(METRICS_NAME); + + // super.serviceInit after all services are added + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.isStopping.getAndSet(true)) { + return; + } + DefaultMetricsSystem.shutdown(); + super.serviceStop(); + } + + public String getName() { + return "FederationGlobalPolicyGenerator"; + } + + public GPGContext getGPGContext() { + return this.gpgContext; + } + + @SuppressWarnings("resource") + public static void main(String argv[]) { + Configuration conf = new YarnConfiguration(); + boolean federationEnabled = + conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED, + YarnConfiguration.DEFAULT_FEDERATION_ENABLED); + if (federationEnabled) { + Thread.setDefaultUncaughtExceptionHandler( + new YarnUncaughtExceptionHandler()); + StringUtils.startupShutdownMessage(GlobalPolicyGenerator.class, argv, + LOG); + GlobalPolicyGenerator globalPolicyGenerator = new GlobalPolicyGenerator(); + globalPolicyGenerator.initAndStart(conf, false); + } else { + LOG.warn("Federation is not enabled. The gpg cannot start."); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java new file mode 100644 index 0000000..abaa57c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java new file mode 100644 index 0000000..1f3a252 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +/** + * Unit test for GlobalPolicyGenerator. + */ +public class TestGlobalPolicyGenerator { +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml index 8156016..ceef10a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -46,5 +46,6 @@ hadoop-yarn-server-timelineservice-hbase hadoop-yarn-server-timelineservice-hbase-tests hadoop-yarn-server-router + hadoop-yarn-server-globalpolicygenerator diff --git hadoop-yarn-project/pom.xml hadoop-yarn-project/pom.xml index 3ef9c45..ea6e9bf 100644 --- hadoop-yarn-project/pom.xml +++ hadoop-yarn-project/pom.xml @@ -80,6 +80,10 @@ org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + + + org.apache.hadoop hadoop-yarn-services-core