+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.applications.mawo.server.common; +/** + * Helper classes for the mawo server common operations. + */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/master/job/JobId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/master/job/JobId.java new file mode 100644 index 00000000000..f056f5787fe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/master/job/JobId.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.applications.mawo.server.master.job; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Random; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; + +/** + * Define MaWo JobId. + */ +public class JobId implements Writable { + + /** + * MaWo job prefix. + */ + private static final String JOB_PREFIX = "mawo_job_"; + + /** + * Create unique random JobId. + * @return unique random JobId + */ + static JobId newJobId() { + Random rn = new Random(); + final int range = 900000; + final int randomadd = 100000; + int randomNum = rn.nextInt(range) + randomadd; + return new JobId(randomNum); + } + + /** + * Unique Id. + */ + private int jobIdentifier; + + /** + * JobId default constructor. + */ + public JobId() { + + } + + /** + * JobId constructor with Id. + * @param id : unique id + */ + public JobId(final int id) { + this.jobIdentifier = id; + } + + /** + * Get JobId. + * @return unique ID + */ + public final int getID() { + return jobIdentifier; + } + + /** + * Print JobId. + * @return JobId + */ + public final String toString() { + return JOB_PREFIX + jobIdentifier; + } + + @Override + /** + * Hashcode for jobId. + */ + public final int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + jobIdentifier; + return result; + } + + @Override + /** + * Implement equals method for jobId. + */ + public final boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + JobId other = (JobId) obj; + if (jobIdentifier != other.jobIdentifier) { + return false; + } + return true; + } + + /** {@inheritDoc} */ + public final void write(final DataOutput out) throws IOException { + WritableUtils.writeVInt(out, jobIdentifier); + } + + /** {@inheritDoc} */ + public final void readFields(final DataInput in) throws IOException { + this.jobIdentifier = WritableUtils.readVInt(in); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/master/job/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/master/job/package-info.java new file mode 100644 index 00000000000..c9453805d1a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/master/job/package-info.java @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.applications.mawo.server.master.job; +/** + * Helper classes for the mawo master job. + */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/worker/WorkerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/worker/WorkerId.java new file mode 100644 index 00000000000..f20b8f563f6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/worker/WorkerId.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.applications.mawo.server.worker; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; + +/** + * Define WorkerId for Workers. + */ +public class WorkerId implements Writable { + /** + * WorkerId is a unique identifier for workers. + */ + private Text workerId = new Text(); + /** + * Hostname of worker node. + */ + private Text hostname = new Text(); + /** + * Ip address of worker node. + */ + private Text ipAdd = new Text(); + + /** + * Default constructor for workerId. + * Set Hostname and Ip address of the machine where worker is running. + */ + public WorkerId() { + try { + this.hostname = + new Text(InetAddress.getLocalHost().getLocalHost().getHostName()); + this.ipAdd = + new Text(InetAddress.getLocalHost().getHostAddress().toString()); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + } + + /** + * Get hostname for Worker. + * @return hostname of worker node + */ + public final Text getHostname() { + return hostname; + } + + /** + * Set hostname for Worker. + * @param wkhostname : Hostname of worker + */ + public final void setHostname(final Text wkhostname) { + this.hostname = wkhostname; + } + + /** + * Get Worker IP address. + * @return IP address of worker node + */ + public final String getIPAddress() { + return this.ipAdd.toString(); + } + + /** + * Print workerId. + * @return workeId in string + */ + public final String toString() { + return workerId.toString(); + } + + /** + * Get workerId. + * @return workerId : Worker identifier + */ + public final String getWorkerId() { + return this.workerId.toString(); + } + + /** + * Set workerId. + * @param localworkerId : Worker identifier + */ + public final void setWorkerId(final String localworkerId) { + this.workerId = new Text(localworkerId); + } + + @Override + /** + * Implememt equals method for WorkerId. + */ + public final boolean equals(final Object o) { + WorkerId x = (WorkerId) o; + return x.getHostname().equals(this.hostname); + } + + /** {@inheritDoc} */ + public final void write(final DataOutput dataOutput) throws IOException { + workerId.write(dataOutput); + hostname.write(dataOutput); + ipAdd.write(dataOutput); + } + + /** {@inheritDoc} */ + public final void readFields(final DataInput dataInput) throws IOException { + workerId.readFields(dataInput); + hostname.readFields(dataInput); + ipAdd.readFields(dataInput); + } + + @Override + /** + * Override hashcode method for WorkerId. + */ + public final int hashCode() { + final int prime = 31; + int result = 1; + int workerHash = 0; + if (workerId == null) { + workerHash = 0; + } else { + workerHash = workerId.hashCode(); + } + int hostHash = 0; + if (hostname == null) { + hostHash = 0; + } else { + hostHash = hostname.hashCode(); + } + int ipHash = 0; + if (ipAdd == null) { + ipHash = 0; + } else { + ipHash = ipAdd.hashCode(); + } + result = prime * result + workerHash; + result = prime * result + hostHash; + result = prime * result + ipHash; + return result; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/worker/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/worker/package-info.java new file mode 100644 index 00000000000..7fa0d55b6e9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/java/org/apache/hadoop/applications/mawo/server/worker/package-info.java @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.applications.mawo.server.worker; +/** + * Helper classes for the mawo worker. + */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/resources/log4j.properties new file mode 100644 index 00000000000..e8643cd59a5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/resources/log4j.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/resources/mawo-default.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/resources/mawo-default.properties new file mode 100644 index 00000000000..5ebe59eaa1a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/resources/mawo-default.properties @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rpc.server.hostname=localhost +rpc.server.port=5120 + +#curator related configurations +zookeeper.parent.path=/mawoRoot +zookeeper.address=localhost:2181 +zookeeper.retry.interval.ms=1000 +zookeeper.session.timeout.ms=10000 +zookeeper.retries.num=1000 +zookeeper.acl=world:anyone:rwcda +worker.num.tasks=10 +mawo.job-queue-storage.enabled=true +mawo.job-builder.class=org.apache.hadoop.applications.mawo.server.master.job.SimpleTaskJsonJobBuilder +worker.workspace=/tmp + +ycloud.url=0.0.0.0:9191 + +task.block-size.min=10 +task.uber-time.min=12 + +master.tasks-status.log.path=/tmp + +master.teardown-worker.validity-interval.ms=120000 +#a comma-separated list of environment variables needed by all the tasks +worker.whitelist.env= diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/java/org/apache/hadoop/applications/mawo/server/common/TestMaWoConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/java/org/apache/hadoop/applications/mawo/server/common/TestMaWoConfiguration.java new file mode 100644 index 00000000000..f2af48ed54b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/java/org/apache/hadoop/applications/mawo/server/common/TestMaWoConfiguration.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.applications.mawo.server.common; + + +import org.junit.Assert; +import org.junit.Test; +import org.junit.internal.ArrayComparisonFailure; + +import java.lang.*; +import java.util.ArrayList; +import java.util.Properties; + +/** + * Test MaWo configuration. + */ +public class TestMaWoConfiguration { + + /** + * Validate default MaWo Configurations. + */ + @Test + public void testMaWoConfiguration() { + + MawoConfiguration mawoConf = new MawoConfiguration(); + + // validate Rpc server port + Assert.assertEquals(mawoConf.getRpcServerPort(), 5120); + + // validate Rpc hostname + Assert.assertTrue("localhost".equals(mawoConf.getRpcHostName())); + + // validate job queue storage conf + boolean jobQueueStorage = mawoConf.getJobQueueStorageEnabled(); + Assert.assertTrue(jobQueueStorage); + + // validate default teardownWorkerValidity Interval + Assert.assertEquals(mawoConf.getTeardownWorkerValidityInterval(), 120000); + + // validate Zk related configs + Assert.assertTrue("/tmp/mawoRoot".equals(mawoConf.getZKParentPath())); + Assert.assertTrue("localhost:2181".equals(mawoConf.getZKAddress())); + Assert.assertEquals(1000, mawoConf.getZKRetryIntervalMS()); + Assert.assertEquals(10000, mawoConf.getZKSessionTimeoutMS()); + Assert.assertEquals(1000, mawoConf.getZKRetriesNum()); + } + + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/resources/log4j.properties new file mode 100644 index 00000000000..e8643cd59a5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/resources/log4j.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/resources/mawo.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/resources/mawo.properties new file mode 100644 index 00000000000..0d5ea315894 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/test/resources/mawo.properties @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rpc.server.hostname=localhost +rpc.server.port=5120 + +#curator related configurations +zookeeper.parent.path=/tmp/mawoRoot +zookeeper.address=localhost:2181 +zookeeper.retry.interval.ms=1000 +zookeeper.session.timeout.ms=10000 +zookeeper.retries.num=1000 +zookeeper.acl=world:anyone:rwcda +worker.num.tasks=10 +mawo.job-queue-storage.enabled=true