Property changes on: .
___________________________________________________________________
Modified: svn:ignore
- .project
.classpath
build
.fbprefs
src-gen
hama-0.1.0-dev.jar
hama-0.1.0-dev-examples.jar
hama-0.1.0-dev-test.jar
logs
.settings
+ .project
.classpath
build
.fbprefs
src-gen
hama-0.1.0-dev.jar
hama-0.1.0-dev-examples.jar
hama-0.1.0-dev-test.jar
logs
.settings
lib
target
*.ipr
*.iml
Index: conf/hama-env.sh.template
===================================================================
--- conf/hama-env.sh.template (revision 1152138)
+++ conf/hama-env.sh.template (working copy)
@@ -1,37 +0,0 @@
-#
-#/**
-# * Copyright 2007 The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-# Set environment variables here.
-
-# The java implementation to use. Required.
-# export JAVA_HOME=/usr/lib/jvm/java-6-sun
-
-# Where log files are stored. $HAMA_HOME/logs by default.
-# export HAMA_LOG_DIR=${HAMA_HOME}/logs
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-# export HAMA_HEAPSIZE=1000
-
-# Extra ssh options. Empty by default.
-# export HAMA_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HAMA_CONF_DIR"
-
-# Tell Hama whether it should manage it's own instance of Zookeeper or not.
-# export HAMA_MANAGES_ZK=true
Index: conf/hama-site.xml.template
===================================================================
--- conf/hama-site.xml.template (revision 1152138)
+++ conf/hama-site.xml.template (working copy)
@@ -1,25 +0,0 @@
-
-
-
-
-
Index: conf/groomservers.template
===================================================================
--- conf/groomservers.template (revision 1152138)
+++ conf/groomservers.template (working copy)
@@ -1 +0,0 @@
-localhost
\ No newline at end of file
Index: conf/hama-default.xml
===================================================================
--- conf/hama-default.xml (revision 1152138)
+++ conf/hama-default.xml (working copy)
@@ -1,165 +0,0 @@
-
-
-
-
-
- bsp.master.address
- local
- The address of the bsp master server. Either the
- literal string "local" or a host[:port] (where host is a name or
- IP address) for distributed mode.
-
-
-
- bsp.master.port
- 40000
- The port master should bind to.
-
-
- bsp.peer.port
- 61000
- The port an groom server binds to.
-
-
- bsp.groom.rpc.port
- 50000
- The port an groom rpc binds to.
-
-
- bsp.http.infoserver.port
- 40013
- The port where the web-interface can be seen.
-
-
-
- bsp.groom.report.address
- 127.0.0.1:0
- The interface and port that groom server listens on.
- Since it is only connected to by the tasks, it uses the local interface.
- EXPERT ONLY. Should only be changed if your host does not have the loopback
- interface.
-
-
- bsp.system.dir
- ${hadoop.tmp.dir}/bsp/system
- The shared directory where BSP stores control files.
-
-
-
- bsp.local.dir
- ${hadoop.tmp.dir}/bsp/local
- local directory for temporal store.
-
-
- hama.tmp.dir
- /tmp/hama-${user.name}
- Temporary directory on the local filesystem.
-
-
- bsp.child.java.opts
- -Xmx1024m
- Java opts for the groom server child processes.
- The following symbol, if present, will be interpolated: @taskid@ is replaced
- by current TaskID. Any other occurrences of '@' will go unchanged.
- For example, to enable verbose gc logging to a file named for the taskid in
- /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
- -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
- The configuration variable bsp.child.ulimit can be used to control the
- maximum virtual memory of the child processes.
-
-
-
- bsp.local.tasks.maximum
- 20
- Number of tasks that run in parallel when in local mode.
-
-
-
-
- hama.zookeeper.property.initLimit
- 10
- Property from ZooKeeper's config zoo.cfg.
- The number of ticks that the initial synchronization phase can take.
-
-
-
- hama.zookeeper.property.syncLimit
- 5
- Property from ZooKeeper's config zoo.cfg.
- The number of ticks that can pass between sending a request and getting an
- acknowledgment.
-
-
-
- hama.zookeeper.property.dataDir
- ${hama.tmp.dir}/zookeeper
- Property from ZooKeeper's config zoo.cfg.
- The directory where the snapshot is stored.
-
-
-
- hama.zookeeper.property.clientPort
- 21810
- Property from ZooKeeper's config zoo.cfg.
- The port at which the clients will connect.
-
-
-
- hama.zookeeper.property.tickTime
- 3000
-
-
-
- hama.zookeeper.property.maxClientCnxns
- 30
- Property from ZooKeeper's config zoo.cfg.
- Limit on number of concurrent connections (at the socket level) that a
- single client, identified by IP address, may make to a single member of
- the ZooKeeper ensemble. Set high to avoid zk connection issues running
- standalone and pseudo-distributed.
-
-
-
-
- hama.zookeeper.quorum
- localhost
- Comma separated list of servers in the ZooKeeper quorum.
- For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
- By default this is set to localhost for local and pseudo-distributed modes
- of operation. For a fully-distributed setup, this should be set to a full
- list of ZooKeeper quorum servers. If HAMA_MANAGES_ZK is set in hama-env.sh
- this is the list of servers which we will start/stop ZooKeeper on.
-
-
-
-
-
Index: conf/log4j.properties
===================================================================
--- conf/log4j.properties (revision 1152138)
+++ conf/log4j.properties (working copy)
@@ -1,81 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Define some default values that can be overridden by system properties
-hama.root.logger=INFO,console
-hama.log.dir=.
-hama.log.file=hama.log
-
-# Define the root logger to the system property "hama.root.logger".
-log4j.rootLogger=${hama.root.logger}
-
-# Logging Threshold
-log4j.threshhold=ALL
-
-#
-# Daily Rolling File Appender
-#
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hama.log.dir}/${hama.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-#
-# TaskLog Appender
-#
-
-#Default values
-hama.tasklog.taskid=null
-hama.tasklog.noKeepSplits=4
-hama.tasklog.totalLogFileSize=100
-hama.tasklog.purgeLogSplits=true
-hama.tasklog.logsRetainHours=12
-
-log4j.appender.TLA=org.apache.hama.bsp.TaskLogAppender
-log4j.appender.TLA.taskId=${hama.tasklog.taskid}
-log4j.appender.TLA.totalLogFileSize=${hama.tasklog.totalLogFileSize}
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-# Custom Logging levels
-
-#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
-#log4j.logger.org.apache.hadoop.dfs=DEBUG
-#log4j.logger.org.apache.hama=DEBUG
Index: src/test/org/apache/hama/HamaTestCase.java
===================================================================
--- src/test/org/apache/hama/HamaTestCase.java (revision 1152138)
+++ src/test/org/apache/hama/HamaTestCase.java (working copy)
@@ -1,179 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.File;
-import java.io.IOException;
-
-import junit.framework.AssertionFailedError;
-import junit.framework.TestCase;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hama.util.Bytes;
-
-public abstract class HamaTestCase extends TestCase {
- private static Log LOG = LogFactory.getLog(HamaTestCase.class);
-
- /** configuration parameter name for test directory */
- public static final String TEST_DIRECTORY_KEY = "test.build.data";
-
- private boolean localfs = false;
- protected Path testDir = null;
- protected FileSystem fs = null;
-
- static {
- initialize();
- }
-
- public volatile HamaConfiguration conf;
-
- /** constructor */
- public HamaTestCase() {
- super();
- init();
- }
-
- /**
- * @param name
- */
- public HamaTestCase(String name) {
- super(name);
- init();
- }
-
- private void init() {
- conf = new HamaConfiguration();
- conf.setStrings("bsp.local.dir", "/tmp/hama-test");
- conf.set("bsp.master.address", "localhost");
- conf.set("bsp.groom.report.address", "127.0.0.1:0");
- }
-
- /**
- * Note that this method must be called after the mini hdfs cluster has
- * started or we end up with a local file system.
- */
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- localfs =
- (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
-
- if (fs == null) {
- this.fs = FileSystem.get(conf);
- }
- try {
- if (localfs) {
- this.testDir = getUnitTestdir(getName());
- if (fs.exists(testDir)) {
- fs.delete(testDir, true);
- }
- } else {
- this.testDir =
- this.fs.makeQualified(new Path("/tmp/hama-test"));
- }
- } catch (Exception e) {
- LOG.fatal("error during setup", e);
- throw e;
- }
- }
-
- @Override
- protected void tearDown() throws Exception {
- try {
- if (localfs) {
- if (this.fs.exists(testDir)) {
- this.fs.delete(testDir, true);
- }
- }
- } catch (Exception e) {
- LOG.fatal("error during tear down", e);
- }
- super.tearDown();
- }
-
- protected Path getUnitTestdir(String testName) {
- return new Path(
- conf.get(TEST_DIRECTORY_KEY, "/tmp/hama-test/build/data"), testName);
- }
-
- /**
- * Initializes parameters used in the test environment:
- *
- * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set.
- * Sets the boolean debugging if "DEBUGGING" is set in the environment.
- * If debugging is enabled, reconfigures logging so that the root log level is
- * set to WARN and the logging level for the package is set to DEBUG.
- */
- public static void initialize() {
- if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
- System.setProperty(TEST_DIRECTORY_KEY, new File(
- "build/hama/test").getAbsolutePath());
- }
- }
-
- /**
- * Common method to close down a MiniDFSCluster and the associated file system
- *
- * @param cluster
- */
- public static void shutdownDfs(MiniDFSCluster cluster) {
- if (cluster != null) {
- LOG.info("Shutting down Mini DFS ");
- try {
- cluster.shutdown();
- } catch (Exception e) {
- /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
- // here because of an InterruptedException. Don't let exceptions in
- // here be cause of test failure.
- }
- try {
- FileSystem fs = cluster.getFileSystem();
- if (fs != null) {
- LOG.info("Shutting down FileSystem");
- fs.close();
- }
- FileSystem.closeAll();
- } catch (IOException e) {
- LOG.error("error closing file system", e);
- }
- }
- }
-
- public void assertByteEquals(byte[] expected,
- byte[] actual) {
- if (Bytes.compareTo(expected, actual) != 0) {
- throw new AssertionFailedError("expected:<" +
- Bytes.toString(expected) + "> but was:<" +
- Bytes.toString(actual) + ">");
- }
- }
-
- public static void assertEquals(byte[] expected,
- byte[] actual) {
- if (Bytes.compareTo(expected, actual) != 0) {
- throw new AssertionFailedError("expected:<" +
- Bytes.toStringBinary(expected) + "> but was:<" +
- Bytes.toStringBinary(actual) + ">");
- }
- }
-
-}
Index: src/test/org/apache/hama/MiniBSPCluster.java
===================================================================
--- src/test/org/apache/hama/MiniBSPCluster.java (revision 1152138)
+++ src/test/org/apache/hama/MiniBSPCluster.java (working copy)
@@ -1,252 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.IOException;
-
-import java.net.ServerSocket;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import static java.util.concurrent.TimeUnit.*;
-
-import static junit.framework.Assert.*;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hama.bsp.BSPMaster;
-import org.apache.hama.bsp.GroomServer;
-import org.apache.hama.HamaConfiguration;
-
-
-public class MiniBSPCluster {
-
- public static final Log LOG = LogFactory.getLog(MiniBSPCluster.class);
-
- private ScheduledExecutorService scheduler;
-
- private HamaConfiguration configuration;
- private BSPMasterRunner master;
- private List groomServerList =
- new CopyOnWriteArrayList();
- private int grooms;
-
- public class BSPMasterRunner implements Runnable{
- BSPMaster bspm;
- HamaConfiguration conf;
-
- public BSPMasterRunner(HamaConfiguration conf){
- this.conf = conf;
- if(null == this.conf)
- throw new NullPointerException("No Configuration for BSPMaster.");
- }
-
- public void run(){
- try{
- LOG.info("Starting BSP Master.");
- this.bspm = BSPMaster.startMaster(this.conf);
- this.bspm.offerService();
- }catch(IOException ioe){
- LOG.error("Fail to startup BSP Master.", ioe);
- }catch(InterruptedException ie){
- LOG.error("BSP Master fails in offerService().", ie);
- Thread.currentThread().interrupt();
- }
- }
-
- public void shutdown(){
- if(null != this.bspm) this.bspm.shutdown();
- }
-
- public boolean isRunning(){
- if(null == this.bspm) return false;
-
- if(this.bspm.currentState().equals(BSPMaster.State.RUNNING)){
- return true;
- }
- return false;
- }
-
- public BSPMaster getMaster(){
- return this.bspm;
- }
- }
-
- public class GroomServerRunner implements Runnable{
- GroomServer gs;
- HamaConfiguration conf;
-
- public GroomServerRunner(HamaConfiguration conf){
- this.conf = conf;
- }
-
- public void run(){
- try{
- this.gs = GroomServer.constructGroomServer(GroomServer.class, conf);
- GroomServer.startGroomServer(this.gs).join();
- }catch(InterruptedException ie){
- LOG.error("Fail to start GroomServer. ", ie);
- Thread.currentThread().interrupt();
- }
- }
-
- public void shutdown(){
- try{
- if(null != this.gs) this.gs.shutdown();
- }catch(IOException ioe){
- LOG.info("Fail to shutdown GroomServer.", ioe);
- }
- }
-
- public boolean isRunning(){
- if(null == this.gs) return false;
- return this.gs.isRunning();
- }
-
- public GroomServer getGroomServer(){
- return this.gs;
- }
- }
-
- public MiniBSPCluster(HamaConfiguration conf, int groomServers) {
- this.configuration = conf;
- this.grooms = groomServers;
- if(1 > this.grooms) {
- this.grooms = 2;
- }
- LOG.info("Groom server number "+this.grooms);
- int threadpool = conf.getInt("bsp.test.threadpool", 10);
- LOG.info("Thread pool value "+threadpool);
- scheduler = Executors.newScheduledThreadPool(threadpool);
- }
-
- public void startBSPCluster(){
- startMaster();
- startGroomServers();
- }
-
- public void shutdownBSPCluster(){
- if(null != this.master && this.master.isRunning())
- this.master.shutdown();
- if(0 < groomServerList.size()){
- for(GroomServerRunner groom: groomServerList){
- if(groom.isRunning()) groom.shutdown();
- }
- }
- }
-
-
- public void startMaster(){
- if(null == this.scheduler)
- throw new NullPointerException("No ScheduledExecutorService exists.");
- this.master = new BSPMasterRunner(this.configuration);
- scheduler.schedule(this.master, 0, SECONDS);
- }
-
- public void startGroomServers(){
- if(null == this.scheduler)
- throw new NullPointerException("No ScheduledExecutorService exists.");
- if(null == this.master)
- throw new NullPointerException("No BSPMaster exists.");
- int cnt=0;
- while(!this.master.isRunning()){
- LOG.info("Waiting BSPMaster up.");
- try{
- Thread.sleep(1000);
- cnt++;
- if(100 < cnt){
- fail("Fail to launch BSPMaster.");
- }
- }catch(InterruptedException ie){
- LOG.error("Fail to check BSP Master's state.", ie);
- Thread.currentThread().interrupt();
- }
- }
- for(int i=0; i < this.grooms; i++){
- HamaConfiguration c = new HamaConfiguration(this.configuration);
- randomPort(c);
- GroomServerRunner gsr = new GroomServerRunner(c);
- groomServerList.add(gsr);
- scheduler.schedule(gsr, 0, SECONDS);
- cnt = 0;
- while(!gsr.isRunning()){
- LOG.info("Waitin for GroomServer up.");
- try{
- Thread.sleep(1000);
- cnt++;
- if(10 < cnt){
- fail("Fail to launch groom server.");
- }
- }catch(InterruptedException ie){
- LOG.error("Fail to check Groom Server's state.", ie);
- Thread.currentThread().interrupt();
- }
- }
- }
-
- }
-
- private void randomPort(HamaConfiguration conf){
- try{
- ServerSocket skt = new ServerSocket(0);
- int p = skt.getLocalPort();
- skt.close();
- conf.set(Constants.PEER_PORT, new Integer(p).toString());
- conf.setInt(Constants.GROOM_RPC_PORT, p+100);
- }catch(IOException ioe){
- LOG.error("Can not find a free port for BSPPeer.", ioe);
- }
- }
-
- public void shutdown() {
- shutdownBSPCluster();
- scheduler.shutdown();
- }
-
- public List getGroomServerThreads() {
- List list = new ArrayList();
- for(GroomServerRunner gsr: groomServerList){
- list.add(new Thread(gsr));
- }
- return list;
- }
-
- public Thread getMaster() {
- return new Thread(this.master);
- }
-
- public List getGroomServers(){
- List list = new ArrayList();
- for(GroomServerRunner gsr: groomServerList){
- list.add(gsr.getGroomServer());
- }
- return list;
- }
-
- public BSPMaster getBSPMaster(){
- if(null != this.master)
- return this.master.getMaster();
- return null;
- }
-
- public ScheduledExecutorService getScheduler(){
- return this.scheduler;
- }
-}
Index: src/test/org/apache/hama/zookeeper/TestZKTools.java
===================================================================
--- src/test/org/apache/hama/zookeeper/TestZKTools.java (revision 1152138)
+++ src/test/org/apache/hama/zookeeper/TestZKTools.java (working copy)
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.zookeeper;
-
-import org.apache.hama.Constants;
-import org.apache.hama.HamaConfiguration;
-
-import junit.framework.TestCase;
-
-public class TestZKTools extends TestCase {
-
- public void testZKProps() {
- HamaConfiguration conf = new HamaConfiguration();
- conf.set(Constants.ZOOKEEPER_QUORUM, "test.com:123");
- conf.set(Constants.ZOOKEEPER_CLIENT_PORT, "2222");
-
- assertEquals("test.com:2222", QuorumPeer.getZKQuorumServersString(conf));
- }
-}
Index: src/test/org/apache/hama/HamaClusterTestCase.java
===================================================================
--- src/test/org/apache/hama/HamaClusterTestCase.java (revision 1152138)
+++ src/test/org/apache/hama/HamaClusterTestCase.java (working copy)
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.File;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-
-public abstract class HamaClusterTestCase extends HamaTestCase {
- public static final Log LOG = LogFactory.getLog(HamaClusterTestCase.class);
- protected MiniDFSCluster dfsCluster;
- protected MiniBSPCluster bspCluster;
- protected MiniZooKeeperCluster zooKeeperCluster;
- protected boolean startDfs;
- protected int numOfGroom = 2;
-
- /** default constructor */
- public HamaClusterTestCase() {
- this(false);
- }
-
- public HamaClusterTestCase(boolean startDfs) {
- super();
- this.startDfs = startDfs;
- }
-
- /**
- * Actually start the MiniBSP instance.
- */
- protected void hamaClusterSetup() throws Exception {
- File testDir = new File(getUnitTestdir(getName()).toString());
-
- // Note that this is done before we create the MiniHamaCluster because we
- // need to edit the config to add the ZooKeeper servers.
- this.zooKeeperCluster = new MiniZooKeeperCluster();
- int clientPort = this.zooKeeperCluster.startup(testDir);
- conf.set("hama.zookeeper.property.clientPort", Integer.toString(clientPort));
- bspCluster = new MiniBSPCluster(this.conf, numOfGroom);
- bspCluster.startBSPCluster();
- }
-
- @Override
- protected void setUp() throws Exception {
- try {
- if (this.startDfs) {
- // This spews a bunch of warnings about missing scheme. TODO: fix.
- this.dfsCluster = new MiniDFSCluster(0, this.conf, 2, true, true, true,
- null, null, null, null);
-
- // mangle the conf so that the fs parameter points to the minidfs we
- // just started up
- FileSystem filesystem = dfsCluster.getFileSystem();
- conf.set("fs.defaultFS", filesystem.getUri().toString());
- Path parentdir = filesystem.getHomeDirectory();
-
- filesystem.mkdirs(parentdir);
- }
-
- // do the super setup now. if we had done it first, then we would have
- // gotten our conf all mangled and a local fs started up.
- super.setUp();
-
- // start the instance
- hamaClusterSetup();
- } catch (Exception e) {
- if (zooKeeperCluster != null) {
- zooKeeperCluster.shutdown();
- }
- if (dfsCluster != null) {
- shutdownDfs(dfsCluster);
- }
- throw e;
- }
- }
-
- @Override
- protected void tearDown() throws Exception {
- super.tearDown();
- try {
- if (startDfs) {
- shutdownDfs(dfsCluster);
- }
- bspCluster.shutdown();
- } catch (Exception e) {
- LOG.error(e);
- }
- }
-}
Index: src/test/org/apache/hama/bsp/TestBSPMessageBundle.java
===================================================================
--- src/test/org/apache/hama/bsp/TestBSPMessageBundle.java (revision 1152138)
+++ src/test/org/apache/hama/bsp/TestBSPMessageBundle.java (working copy)
@@ -1,81 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import junit.framework.TestCase;
-
-public class TestBSPMessageBundle extends TestCase {
-
- public void testEmpty() throws IOException {
- BSPMessageBundle bundle = new BSPMessageBundle();
- // Serialize it.
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- bundle.write(new DataOutputStream(baos));
- baos.close();
- // Deserialize it.
- BSPMessageBundle readBundle = new BSPMessageBundle();
- readBundle.readFields(new DataInputStream(new ByteArrayInputStream(baos
- .toByteArray())));
- assertEquals(0, readBundle.getMessages().size());
- }
-
- public void testSerializationDeserialization() throws IOException {
- BSPMessageBundle bundle = new BSPMessageBundle();
- ByteMessage[] testMessages = new ByteMessage[16];
- for (int i = 0; i < testMessages.length; ++i) {
- // Create a one byte tag containing the number of the message.
- byte[] tag = new byte[1];
- tag[0] = (byte) i;
- // Create a four bytes data part containing serialized number of the
- // message.
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- baos.write(i);
- baos.close();
- byte[] data = baos.toByteArray();
- testMessages[i] = new ByteMessage(tag, data);
- bundle.addMessage(testMessages[i]);
- }
- // Serialize it.
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- bundle.write(new DataOutputStream(baos));
- baos.close();
- // Deserialize it.
- BSPMessageBundle readBundle = new BSPMessageBundle();
- readBundle.readFields(new DataInputStream(new ByteArrayInputStream(baos
- .toByteArray())));
- // Check contents.
- int messageNumber = 0;
- for (BSPMessage message : readBundle.getMessages()) {
- ByteMessage byteMessage = (ByteMessage) message;
- assertTrue(Arrays.equals(testMessages[messageNumber].getTag(),
- byteMessage.getTag()));
- assertTrue(Arrays.equals(testMessages[messageNumber].getData(),
- byteMessage.getData()));
- ++messageNumber;
- }
- assertEquals(testMessages.length, messageNumber);
- }
-}
Index: src/test/org/apache/hama/bsp/TestClusterStatus.java
===================================================================
--- src/test/org/apache/hama/bsp/TestClusterStatus.java (revision 1152138)
+++ src/test/org/apache/hama/bsp/TestClusterStatus.java (working copy)
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hama.bsp.BSPMaster;
-import org.apache.hama.bsp.ClusterStatus;
-
-public class TestClusterStatus extends TestCase {
- Random rnd = new Random();
-
- protected void setUp() throws Exception {
- super.setUp();
- }
-
- public final void testWriteAndReadFields() throws IOException {
- DataOutputBuffer out = new DataOutputBuffer();
- DataInputBuffer in = new DataInputBuffer();
-
- ClusterStatus status1;
- Map grooms = new HashMap();
-
- for (int i = 0; i < 10; i++) {
- int num = rnd.nextInt();
- String groomName = "groom_" + num;
- String peerName = "peerhost:" + num;
- grooms.put(groomName, peerName);
- }
-
- int tasks = rnd.nextInt(100);
- int maxTasks = rnd.nextInt(100);
- BSPMaster.State state = BSPMaster.State.RUNNING;
-
- status1 = new ClusterStatus(grooms, tasks, maxTasks, state);
- status1.write(out);
-
- in.reset(out.getData(), out.getLength());
-
- ClusterStatus status2 = new ClusterStatus();
- status2.readFields(in);
-
- Map grooms_s = new HashMap(status1
- .getActiveGroomNames());
- Map grooms_o = new HashMap(status2
- .getActiveGroomNames());
-
- assertEquals(status1.getGroomServers(), status2.getGroomServers());
-
- assertTrue(grooms_s.entrySet().containsAll(grooms_o.entrySet()));
- assertTrue(grooms_o.entrySet().containsAll(grooms_s.entrySet()));
-
- assertEquals(status1.getTasks(), status2.getTasks());
- assertEquals(status1.getMaxTasks(), status2.getMaxTasks());
- }
-}
Index: src/test/org/apache/hama/bsp/TestMessages.java
===================================================================
--- src/test/org/apache/hama/bsp/TestMessages.java (revision 1152138)
+++ src/test/org/apache/hama/bsp/TestMessages.java (working copy)
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import junit.framework.TestCase;
-
-import org.apache.hama.util.Bytes;
-
-public class TestMessages extends TestCase {
-
- public void testByteMessage() {
- int dataSize = (int) (Runtime.getRuntime().maxMemory() * 0.60);
- ByteMessage msg = new ByteMessage(Bytes.toBytes("tag"), new byte[dataSize]);
- assertEquals(msg.getData().length, dataSize);
- msg = null;
-
- byte[] dummyData = new byte[1024];
- ByteMessage msg2 = new ByteMessage(Bytes.tail(dummyData, 128), dummyData);
- assertEquals(
- Bytes.compareTo(msg2.getTag(), 0, 128, msg2.getData(),
- msg2.getData().length - 128, 128), 0);
- }
-}
Index: src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java
===================================================================
--- src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java (revision 1152138)
+++ src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java (working copy)
@@ -1,94 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hama.Constants;
-import org.apache.hama.HamaCluster;
-import org.apache.hama.HamaConfiguration;
-
-public class TestBSPMasterGroomServer extends HamaCluster {
-
- private static Log LOG = LogFactory.getLog(TestBSPMasterGroomServer.class);
- private static String TMP_OUTPUT = "/tmp/test-example/";
- private HamaConfiguration configuration;
- private String TEST_JOB = "src/test/testjar/testjob.jar";
-
- public TestBSPMasterGroomServer() {
- configuration = new HamaConfiguration();
- configuration.set("bsp.master.address", "localhost");
- assertEquals("Make sure master addr is set to localhost:", "localhost",
- configuration.get("bsp.master.address"));
- configuration.setStrings("bsp.local.dir", "/tmp/hama-test");
- configuration.set(Constants.ZOOKEEPER_QUORUM, "localhost");
- configuration.setInt(Constants.ZOOKEEPER_CLIENT_PORT, 21810);
- }
-
- public void setUp() throws Exception {
- super.setUp();
- }
-
- public void testSubmitJob() throws Exception {
- BSPJob bsp = new BSPJob(configuration);
- bsp.setJobName("Test Serialize Printing");
- bsp.setBspClass(testjar.ClassSerializePrinting.HelloBSP.class);
- bsp.setJar(TEST_JOB);
-
- // Set the task size as a number of GroomServer
- BSPJobClient jobClient = new BSPJobClient(configuration);
- configuration.setInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 600);
- ClusterStatus cluster = jobClient.getClusterStatus(false);
- assertEquals(this.numOfGroom, cluster.getMaxTasks());
-
- // TODO test the multi-tasks
- bsp.setNumBspTask(1);
-
- FileSystem fileSys = FileSystem.get(conf);
-
- if (bsp.waitForCompletion(true)) {
- checkOutput(fileSys, cluster, conf);
- }
- LOG.info("Client finishes execution job.");
- }
-
- private static void checkOutput(FileSystem fileSys, ClusterStatus cluster,
- HamaConfiguration conf) throws Exception {
- for (int i = 0; i < 1; i++) { // TODO test the multi-tasks
- SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, new Path(
- TMP_OUTPUT + i), conf);
- LongWritable timestamp = new LongWritable();
- Text message = new Text();
- reader.next(timestamp, message);
- assertTrue("Check if `Hello BSP' gets printed.", message.toString()
- .indexOf("Hello BSP from") >= 0);
- reader.close();
- }
- }
-
- public void tearDown() throws Exception {
- super.tearDown();
- }
-}
Index: src/test/org/apache/hama/HamaCluster.java
===================================================================
--- src/test/org/apache/hama/HamaCluster.java (revision 1152138)
+++ src/test/org/apache/hama/HamaCluster.java (working copy)
@@ -1,47 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * Forming up the miniDfs and miniZooKeeper
- */
-public abstract class HamaCluster extends HamaClusterTestCase {
- public static final Log LOG = LogFactory.getLog(HamaCluster.class);
- protected final static HamaConfiguration conf = new HamaConfiguration();
-
- public HamaCluster(){
- super();
- }
-
- public HamaCluster(boolean startDfs) {
- super(startDfs);
- }
-
- protected void setUp() throws Exception {
- super.setUp();
- }
-
- protected static HamaConfiguration getConf() {
- return conf;
- }
-}
Index: src/test/org/apache/hama/ipc/TestRPC.java
===================================================================
--- src/test/org/apache/hama/ipc/TestRPC.java (revision 1152138)
+++ src/test/org/apache/hama/ipc/TestRPC.java (working copy)
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.VersionedProtocol;
-
-import junit.framework.TestCase;
-
-public class TestRPC extends TestCase {
- private static final int PORT = 1234;
- private static final String ADDRESS = "0.0.0.0";
-
- public static final Log LOG = LogFactory
- .getLog("org.apache.hadoop.ipc.TestRPC");
-
- private static Configuration conf = new Configuration();
-
- public TestRPC(String name) {
- super(name);
- }
-
- public interface TestProtocol extends VersionedProtocol {
- public static final long versionID = 1L;
-
- void ping() throws IOException;
-
- String echo(String value) throws IOException;
-
- String[] echo(String[] value) throws IOException;
-
- Writable echo(Writable value) throws IOException;
-
- int add(int v1, int v2) throws IOException;
-
- int add(int[] values) throws IOException;
-
- int error() throws IOException;
-
- void testServerGet() throws IOException;
- }
-
- public class TestImpl implements TestProtocol {
-
- public long getProtocolVersion(String protocol, long clientVersion) {
- return TestProtocol.versionID;
- }
-
- public void ping() {
- }
-
- public String echo(String value) throws IOException {
- return value;
- }
-
- public String[] echo(String[] values) throws IOException {
- return values;
- }
-
- public Writable echo(Writable writable) {
- return writable;
- }
-
- public int add(int v1, int v2) {
- return v1 + v2;
- }
-
- public int add(int[] values) {
- int sum = 0;
- for (int i = 0; i < values.length; i++) {
- sum += values[i];
- }
- return sum;
- }
-
- public int error() throws IOException {
- throw new IOException("bobo");
- }
-
- public void testServerGet() throws IOException {
- if (!(Server.get() instanceof RPC.Server)) {
- throw new IOException("Server.get() failed");
- }
- }
-
- }
-
- public void testCalls() throws Exception {
- Server server = RPC.getServer(new TestImpl(), ADDRESS, PORT, conf);
- server.start();
-
- InetSocketAddress addr = new InetSocketAddress(PORT);
- TestProtocol proxy = (TestProtocol) RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
-
- proxy.ping();
-
- String stringResult = proxy.echo("foo");
- assertEquals(stringResult, "foo");
-
- stringResult = proxy.echo((String) null);
- assertEquals(stringResult, null);
-
- String[] stringResults = proxy.echo(new String[] { "foo", "bar" });
- assertTrue(Arrays.equals(stringResults, new String[] { "foo", "bar" }));
-
- stringResults = proxy.echo((String[]) null);
- assertTrue(Arrays.equals(stringResults, null));
-
- int intResult = proxy.add(1, 2);
- assertEquals(intResult, 3);
-
- intResult = proxy.add(new int[] { 1, 2 });
- assertEquals(intResult, 3);
-
- boolean caught = false;
- try {
- proxy.error();
- } catch (IOException e) {
- LOG.debug("Caught " + e);
- caught = true;
- }
- assertTrue(caught);
-
- proxy.testServerGet();
-
- // try some multi-calls
- Method echo = TestProtocol.class.getMethod("echo",
- new Class[] { String.class });
- String[] strings = (String[]) RPC.call(echo, new String[][] { { "a" },
- { "b" } }, new InetSocketAddress[] { addr, addr }, null, conf);
- assertTrue(Arrays.equals(strings, new String[] { "a", "b" }));
-
- Method ping = TestProtocol.class.getMethod("ping", new Class[] {});
- Object[] voids = (Object[]) RPC.call(ping, new Object[][] { {}, {} },
- new InetSocketAddress[] { addr, addr }, null, conf);
- assertEquals(voids, null);
-
- server.stop();
- }
-
- public static void main(String[] args) throws Exception {
-
- new TestRPC("test").testCalls();
-
- }
-
-}
Index: src/test/org/apache/hama/ipc/TestIPC.java
===================================================================
--- src/test/org/apache/hama/ipc/TestIPC.java (revision 1152138)
+++ src/test/org/apache/hama/ipc/TestIPC.java (working copy)
@@ -1,232 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Random;
-
-import junit.framework.TestCase;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.StringUtils;
-
-public class TestIPC extends TestCase {
- public static final Log LOG = LogFactory.getLog(TestIPC.class);
-
- final private static Configuration conf = new Configuration();
- final static private int PING_INTERVAL = 1000;
-
- static {
- Client.setPingInterval(conf, PING_INTERVAL);
- }
-
- public TestIPC(String name) {
- super(name);
- }
-
- private static final Random RANDOM = new Random();
-
- private static final String ADDRESS = "0.0.0.0";
-
- private static class TestServer extends Server {
- private boolean sleep;
-
- public TestServer(int handlerCount, boolean sleep) throws IOException {
- super(ADDRESS, 0, LongWritable.class, handlerCount, conf);
- this.sleep = sleep;
- }
-
- @Override
- public Writable call(Class> protocol, Writable param, long receiveTime)
- throws IOException {
- if (sleep) {
- try {
- Thread.sleep(RANDOM.nextInt(2 * PING_INTERVAL)); // sleep a bit
- } catch (InterruptedException e) {
- }
- }
- return param; // echo param as result
- }
- }
-
- private static class SerialCaller extends Thread {
- private Client client;
- private InetSocketAddress server;
- private int count;
- private boolean failed;
-
- public SerialCaller(Client client, InetSocketAddress server, int count) {
- this.client = client;
- this.server = server;
- this.count = count;
- }
-
- public void run() {
- for (int i = 0; i < count; i++) {
- try {
- LongWritable param = new LongWritable(RANDOM.nextLong());
- LongWritable value = (LongWritable) client.call(param, server, null,
- null);
- if (!param.equals(value)) {
- LOG.fatal("Call failed!");
- failed = true;
- break;
- }
- } catch (Exception e) {
- LOG.fatal("Caught: " + StringUtils.stringifyException(e));
- failed = true;
- }
- }
- }
- }
-
- private static class ParallelCaller extends Thread {
- private Client client;
- private int count;
- private InetSocketAddress[] addresses;
- private boolean failed;
-
- public ParallelCaller(Client client, InetSocketAddress[] addresses,
- int count) {
- this.client = client;
- this.addresses = addresses;
- this.count = count;
- }
-
- public void run() {
- for (int i = 0; i < count; i++) {
- try {
- Writable[] params = new Writable[addresses.length];
- for (int j = 0; j < addresses.length; j++)
- params[j] = new LongWritable(RANDOM.nextLong());
- Writable[] values = client.call(params, addresses, null, null);
- for (int j = 0; j < addresses.length; j++) {
- if (!params[j].equals(values[j])) {
- LOG.fatal("Call failed!");
- failed = true;
- break;
- }
- }
- } catch (Exception e) {
- LOG.fatal("Caught: " + StringUtils.stringifyException(e));
- failed = true;
- }
- }
- }
- }
-
- public void testSerial() throws Exception {
- testSerial(3, false, 2, 5, 100);
- }
-
- public void testSerial(int handlerCount, boolean handlerSleep,
- int clientCount, int callerCount, int callCount) throws Exception {
- Server server = new TestServer(handlerCount, handlerSleep);
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
- server.start();
-
- Client[] clients = new Client[clientCount];
- for (int i = 0; i < clientCount; i++) {
- clients[i] = new Client(LongWritable.class, conf);
- }
-
- SerialCaller[] callers = new SerialCaller[callerCount];
- for (int i = 0; i < callerCount; i++) {
- callers[i] = new SerialCaller(clients[i % clientCount], addr, callCount);
- callers[i].start();
- }
- for (int i = 0; i < callerCount; i++) {
- callers[i].join();
- assertFalse(callers[i].failed);
- }
- for (int i = 0; i < clientCount; i++) {
- clients[i].stop();
- }
- server.stop();
- }
-
- public void testParallel() throws Exception {
- testParallel(10, false, 2, 4, 2, 4, 100);
- }
-
- public void testParallel(int handlerCount, boolean handlerSleep,
- int serverCount, int addressCount, int clientCount, int callerCount,
- int callCount) throws Exception {
- Server[] servers = new Server[serverCount];
- for (int i = 0; i < serverCount; i++) {
- servers[i] = new TestServer(handlerCount, handlerSleep);
- servers[i].start();
- }
-
- InetSocketAddress[] addresses = new InetSocketAddress[addressCount];
- for (int i = 0; i < addressCount; i++) {
- addresses[i] = NetUtils.getConnectAddress(servers[i % serverCount]);
- }
-
- Client[] clients = new Client[clientCount];
- for (int i = 0; i < clientCount; i++) {
- clients[i] = new Client(LongWritable.class, conf);
- }
-
- ParallelCaller[] callers = new ParallelCaller[callerCount];
- for (int i = 0; i < callerCount; i++) {
- callers[i] = new ParallelCaller(clients[i % clientCount], addresses,
- callCount);
- callers[i].start();
- }
- for (int i = 0; i < callerCount; i++) {
- callers[i].join();
- assertFalse(callers[i].failed);
- }
- for (int i = 0; i < clientCount; i++) {
- clients[i].stop();
- }
- for (int i = 0; i < serverCount; i++) {
- servers[i].stop();
- }
- }
-
- public void testStandAloneClient() throws Exception {
- testParallel(10, false, 2, 4, 2, 4, 100);
- Client client = new Client(LongWritable.class, conf);
- InetSocketAddress address = new InetSocketAddress("127.0.0.1", 10);
- try {
- client.call(new LongWritable(RANDOM.nextLong()), address, null, null);
- fail("Expected an exception to have been thrown");
- } catch (IOException e) {
- String message = e.getMessage();
- String addressText = address.toString();
- assertTrue("Did not find " + addressText + " in " + message, message
- .contains(addressText));
- Throwable cause = e.getCause();
- assertNotNull("No nested exception in " + e, cause);
- String causeText = cause.getMessage();
- assertTrue("Did not find " + causeText + " in " + message, message
- .contains(causeText));
- }
- }
-
-}
Index: src/test/org/apache/hama/util/TestNumeric.java
===================================================================
--- src/test/org/apache/hama/util/TestNumeric.java (revision 1152138)
+++ src/test/org/apache/hama/util/TestNumeric.java (working copy)
@@ -1,34 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import junit.framework.TestCase;
-
-public class TestNumeric extends TestCase {
- final static int TEST_INT = 3;
- final static double TEST_DOUBLE = 0.4;
-
- /**
- * Double conversion test
- */
- public void testDouble() {
- assertEquals(Bytes.toDouble(Bytes.toBytes(TEST_DOUBLE)), TEST_DOUBLE);
- }
-}
Index: src/test/org/apache/hama/util/TestBytes.java
===================================================================
--- src/test/org/apache/hama/util/TestBytes.java (revision 1152138)
+++ src/test/org/apache/hama/util/TestBytes.java (working copy)
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import junit.framework.TestCase;
-
-public class TestBytes extends TestCase {
- public void testNullHashCode() {
- byte [] b = null;
- Exception ee = null;
- try {
- Bytes.hashCode(b);
- } catch (Exception e) {
- ee = e;
- }
- assertNotNull(ee);
- }
-
- public void testSplit() throws Exception {
- byte [] lowest = Bytes.toBytes("AAA");
- byte [] middle = Bytes.toBytes("CCC");
- byte [] highest = Bytes.toBytes("EEE");
- byte [][] parts = Bytes.split(lowest, highest, 1);
- for (int i = 0; i < parts.length; i++) {
- System.out.println(Bytes.toString(parts[i]));
- }
- assertEquals(3, parts.length);
- assertTrue(Bytes.equals(parts[1], middle));
- // Now divide into three parts. Change highest so split is even.
- highest = Bytes.toBytes("DDD");
- parts = Bytes.split(lowest, highest, 2);
- for (int i = 0; i < parts.length; i++) {
- System.out.println(Bytes.toString(parts[i]));
- }
- assertEquals(4, parts.length);
- // Assert that 3rd part is 'CCC'.
- assertTrue(Bytes.equals(parts[2], middle));
- }
-
- public void testSplit2() throws Exception {
- // More split tests.
- byte [] lowest = Bytes.toBytes("http://A");
- byte [] highest = Bytes.toBytes("http://z");
- byte [] middle = Bytes.toBytes("http://]");
- byte [][] parts = Bytes.split(lowest, highest, 1);
- for (int i = 0; i < parts.length; i++) {
- System.out.println(Bytes.toString(parts[i]));
- }
- assertEquals(3, parts.length);
- assertTrue(Bytes.equals(parts[1], middle));
- }
-
- public void testToLong() throws Exception {
- long [] longs = {-1l, 123l, 122232323232l};
- for (int i = 0; i < longs.length; i++) {
- byte [] b = Bytes.toBytes(longs[i]);
- assertEquals(longs[i], Bytes.toLong(b));
- }
- }
-
- public void testToFloat() throws Exception {
- float [] floats = {-1f, 123.123f, Float.MAX_VALUE};
- for (int i = 0; i < floats.length; i++) {
- byte [] b = Bytes.toBytes(floats[i]);
- assertEquals(floats[i], Bytes.toFloat(b));
- }
- }
-
- public void testToDouble() throws Exception {
- double [] doubles = {Double.MIN_VALUE, Double.MAX_VALUE};
- for (int i = 0; i < doubles.length; i++) {
- byte [] b = Bytes.toBytes(doubles[i]);
- assertEquals(doubles[i], Bytes.toDouble(b));
- }
- }
-
- public void testBinarySearch() throws Exception {
- byte [][] arr = {
- {1},
- {3},
- {5},
- {7},
- {9},
- {11},
- {13},
- {15},
- };
- byte [] key1 = {3,1};
- byte [] key2 = {4,9};
- byte [] key2_2 = {4};
- byte [] key3 = {5,11};
-
- assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(-(2+1), Arrays.binarySearch(arr, key2_2,
- Bytes.BYTES_COMPARATOR));
- assertEquals(-(2+1), Bytes.binarySearch(arr, key2, 0, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- }
-
- public void testIncrementBytes() throws IOException {
-
- assertTrue(checkTestIncrementBytes(10, 1));
- assertTrue(checkTestIncrementBytes(12, 123435445));
- assertTrue(checkTestIncrementBytes(124634654, 1));
- assertTrue(checkTestIncrementBytes(10005460, 5005645));
- assertTrue(checkTestIncrementBytes(1, -1));
- assertTrue(checkTestIncrementBytes(10, -1));
- assertTrue(checkTestIncrementBytes(10, -5));
- assertTrue(checkTestIncrementBytes(1005435000, -5));
- assertTrue(checkTestIncrementBytes(10, -43657655));
- assertTrue(checkTestIncrementBytes(-1, 1));
- assertTrue(checkTestIncrementBytes(-26, 5034520));
- assertTrue(checkTestIncrementBytes(-10657200, 5));
- assertTrue(checkTestIncrementBytes(-12343250, 45376475));
- assertTrue(checkTestIncrementBytes(-10, -5));
- assertTrue(checkTestIncrementBytes(-12343250, -5));
- assertTrue(checkTestIncrementBytes(-12, -34565445));
- assertTrue(checkTestIncrementBytes(-1546543452, -34565445));
- }
-
- private static boolean checkTestIncrementBytes(long val, long amount)
- throws IOException {
- byte[] value = Bytes.toBytes(val);
- byte [] testValue = {-1, -1, -1, -1, -1, -1, -1, -1};
- if (value[0] > 0) {
- testValue = new byte[Bytes.SIZEOF_LONG];
- }
- System.arraycopy(value, 0, testValue, testValue.length - value.length,
- value.length);
-
- long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount));
-
- return (Bytes.toLong(testValue) + amount) == incrementResult;
- }
-}
Index: src/test/org/apache/hama/util/TestRandomVariable.java
===================================================================
--- src/test/org/apache/hama/util/TestRandomVariable.java (revision 1152138)
+++ src/test/org/apache/hama/util/TestRandomVariable.java (working copy)
@@ -1,74 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import org.apache.log4j.Logger;
-
-import junit.framework.TestCase;
-
-/**
- * Random variable generation test
- */
-public class TestRandomVariable extends TestCase {
- static final Logger LOG = Logger.getLogger(TestRandomVariable.class);
- final static int COUNT = 50;
-
- /**
- * Random object test
- *
- * @throws Exception
- */
- public void testRand() throws Exception {
- for (int i = 0; i < COUNT; i++) {
- double result = RandomVariable.rand();
- assertTrue(result >= 0.0d && result <= 1.0);
- }
- }
-
- /**
- * Random integer test
- *
- * @throws Exception
- */
- public void testRandInt() throws Exception {
- final int min = 122;
- final int max = 561;
-
- for (int i = 0; i < COUNT; i++) {
- int result = RandomVariable.randInt(min, max);
- assertTrue(result >= min && result <= max);
- }
- }
-
- /**
- * Uniform test
- *
- * @throws Exception
- */
- public void testUniform() throws Exception {
- final double min = 1.0d;
- final double max = 3.0d;
-
- for (int i = 0; i < COUNT; i++) {
- double result = RandomVariable.uniform(min, max);
- assertTrue(result >= min && result <= max);
- }
- }
-}
Index: src/test/testjar/testjob.jar
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: src/test/testjar/ClassSerializePrinting.java
===================================================================
--- src/test/testjar/ClassSerializePrinting.java (revision 1152138)
+++ src/test/testjar/ClassSerializePrinting.java (working copy)
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package testjar;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.Text;
-import org.apache.hama.bsp.BSP;
-import org.apache.hama.bsp.BSPPeer;
-import org.apache.zookeeper.KeeperException;
-
-public class ClassSerializePrinting {
- private static String TMP_OUTPUT = "/tmp/test-example/";
-
- public static class HelloBSP extends BSP {
- public static final Log LOG = LogFactory.getLog(HelloBSP.class);
- private Configuration conf;
- private final static int PRINT_INTERVAL = 1000;
- private FileSystem fileSys;
- private int num;
-
- public void bsp(BSPPeer bspPeer) throws IOException,
- KeeperException, InterruptedException {
-
- int i = 0;
- for (String otherPeer : bspPeer.getAllPeerNames()) {
- String peerName = bspPeer.getPeerName();
- if (peerName.equals(otherPeer)) {
- writeLogToFile(peerName, i);
- }
-
- Thread.sleep(PRINT_INTERVAL);
- bspPeer.sync();
- i++;
- }
- }
-
- private void writeLogToFile(String string, int i) throws IOException {
- SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf,
- new Path(TMP_OUTPUT + i), LongWritable.class, Text.class,
- CompressionType.NONE);
- writer.append(new LongWritable(System.currentTimeMillis()), new Text(
- "Hello BSP from " + (i + 1) + " of " + num + ": " + string));
- writer.close();
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- num = Integer.parseInt(conf.get("bsp.peers.num"));
- try {
- fileSys = FileSystem.get(conf);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- }
-
-}
Index: src/java/org/apache/hama/http/HttpServer.java
===================================================================
--- src/java/org/apache/hama/http/HttpServer.java (revision 1152138)
+++ src/java/org/apache/hama/http/HttpServer.java (working copy)
@@ -1,501 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.http;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.BindException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.log.LogLevel;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.mortbay.jetty.Connector;
-import org.mortbay.jetty.Handler;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
-import org.mortbay.jetty.nio.SelectChannelConnector;
-import org.mortbay.jetty.security.SslSocketConnector;
-import org.mortbay.jetty.servlet.Context;
-import org.mortbay.jetty.servlet.DefaultServlet;
-import org.mortbay.jetty.servlet.FilterHolder;
-import org.mortbay.jetty.servlet.FilterMapping;
-import org.mortbay.jetty.servlet.ServletHandler;
-import org.mortbay.jetty.servlet.ServletHolder;
-import org.mortbay.jetty.webapp.WebAppContext;
-import org.mortbay.thread.BoundedThreadPool;
-import org.mortbay.util.MultiException;
-
-/**
- * Create a Jetty embedded server to answer http requests. The primary goal is
- * to serve up status information for the server. There are three contexts:
- * "/logs/" -> points to the log directory "/static/" -> points to common static
- * files (src/webapps/static) "/" -> the jsp server code from
- * (src/webapps/)
- */
-public class HttpServer {
- public static final Log LOG = LogFactory.getLog(HttpServer.class);
-
- static final String FILTER_INITIALIZER_PROPERTY = "hama.http.filter.initializers";
-
- protected final Server webServer;
- protected final Connector listener;
- protected final WebAppContext webAppContext;
- protected final boolean findPort;
- protected final Map defaultContexts = new HashMap();
- protected final List filterNames = new ArrayList();
- private static final int MAX_RETRIES = 10;
-
- /** Same as this(name, bindAddress, port, findPort, null); */
- public HttpServer(String name, String bindAddress, int port, boolean findPort)
- throws IOException {
- this(name, bindAddress, port, findPort, new Configuration());
- }
-
- /**
- * Create a status server on the given port. The jsp scripts are taken from
- * src/webapps/.
- *
- * @param name The name of the server
- * @param port The port to use on the server
- * @param findPort whether the server should start at the given port and
- * increment by 1 until it finds a free port.
- * @param conf Configuration
- */
- public HttpServer(String name, String bindAddress, int port,
- boolean findPort, Configuration conf) throws IOException {
- webServer = new Server();
- this.findPort = findPort;
-
- listener = createBaseListener(conf);
- listener.setHost(bindAddress);
- listener.setPort(port);
- webServer.addConnector(listener);
-
- webServer.setThreadPool(new BoundedThreadPool());
-
- final String appDir = getWebAppsPath();
- ContextHandlerCollection contexts = new ContextHandlerCollection();
- webServer.setHandler(contexts);
- webAppContext = new WebAppContext();
-
- System.setProperty("java.naming.factory.initial",
- "org.mortbay.naming.InitialContextFactory");
- System.setProperty("java.naming.factory.url.pkgs", "org.mortbay.naming");
-
- webAppContext.setContextPath("/");
- webAppContext.setWar(appDir + "/" + name);
- webServer.addHandler(webAppContext);
-
- addDefaultApps(contexts, appDir);
-
- addDefaultServlets();
- }
-
- /**
- * Create a required listener for the Jetty instance listening on the port
- * provided. This wrapper and all subclasses must create at least one
- * listener.
- */
- protected Connector createBaseListener(Configuration conf) throws IOException {
- SelectChannelConnector ret = new SelectChannelConnector();
- ret.setLowResourceMaxIdleTime(10000);
- ret.setAcceptQueueSize(128);
- ret.setResolveNames(false);
- ret.setUseDirectBuffers(false);
- return ret;
- }
-
- /**
- * Add default apps.
- *
- * @param appDir The application directory
- * @throws IOException
- */
- protected void addDefaultApps(ContextHandlerCollection parent,
- final String appDir) throws IOException {
- // set up the context for "/logs/" if "hama.log.dir" property is defined.
- String logDir = System.getProperty("hama.log.dir");
- if (logDir != null) {
- Context logContext = new Context(parent, "/logs");
- logContext.setResourceBase(logDir);
- logContext.addServlet(DefaultServlet.class, "/");
- defaultContexts.put(logContext, true);
- }
- // set up the context for "/static/*"
- Context staticContext = new Context(parent, "/static");
- staticContext.setResourceBase(appDir + "/static");
- staticContext.addServlet(DefaultServlet.class, "/*");
- defaultContexts.put(staticContext, true);
- }
-
- /**
- * Add default servlets.
- */
- protected void addDefaultServlets() {
- // set up default servlets
- addServlet("stacks", "/stacks", StackServlet.class);
- addServlet("logLevel", "/logLevel", LogLevel.Servlet.class);
- }
-
- public void addContext(Context ctxt, boolean isFiltered) throws IOException {
- webServer.addHandler(ctxt);
- defaultContexts.put(ctxt, isFiltered);
- }
-
- /**
- * Add a context
- *
- * @param pathSpec The path spec for the context
- * @param dir The directory containing the context
- * @param isFiltered if true, the servlet is added to the filter path mapping
- * @throws IOException
- */
- protected void addContext(String pathSpec, String dir, boolean isFiltered)
- throws IOException {
- if (0 == webServer.getHandlers().length) {
- throw new RuntimeException("Couldn't find handler");
- }
- WebAppContext webAppCtx = new WebAppContext();
- webAppCtx.setContextPath(pathSpec);
- webAppCtx.setWar(dir);
- addContext(webAppCtx, true);
- }
-
- /**
- * Set a value in the webapp context. These values are available to the jsp
- * pages as "application.getAttribute(name)".
- *
- * @param name The name of the attribute
- * @param value The value of the attribute
- */
- public void setAttribute(String name, Object value) {
- webAppContext.setAttribute(name, value);
- }
-
- /**
- * Add a servlet in the server.
- *
- * @param name The name of the servlet (can be passed as null)
- * @param pathSpec The path spec for the servlet
- * @param clazz The servlet class
- */
- public void addServlet(String name, String pathSpec,
- Class extends HttpServlet> clazz) {
- addInternalServlet(name, pathSpec, clazz);
- addFilterPathMapping(pathSpec, webAppContext);
- }
-
- /**
- * Add an internal servlet in the server.
- *
- * @param name The name of the servlet (can be passed as null)
- * @param pathSpec The path spec for the servlet
- * @param clazz The servlet class
- * @deprecated this is a temporary method
- */
- @Deprecated
- public void addInternalServlet(String name, String pathSpec,
- Class extends HttpServlet> clazz) {
- ServletHolder holder = new ServletHolder(clazz);
- if (name != null) {
- holder.setName(name);
- }
- webAppContext.addServlet(holder, pathSpec);
- }
-
- /**
- * Define a filter for a context and set up default url mappings.
- */
- protected void defineFilter(Context ctx, String name, String classname,
- Map parameters, String[] urls) {
-
- FilterHolder holder = new FilterHolder();
- holder.setName(name);
- holder.setClassName(classname);
- holder.setInitParameters(parameters);
- FilterMapping fmap = new FilterMapping();
- fmap.setPathSpecs(urls);
- fmap.setDispatches(Handler.ALL);
- fmap.setFilterName(name);
- ServletHandler handler = ctx.getServletHandler();
- handler.addFilter(holder, fmap);
- }
-
- /**
- * Add the path spec to the filter path mapping.
- *
- * @param pathSpec The path spec
- * @param webAppCtx The WebApplicationContext to add to
- */
- protected void addFilterPathMapping(String pathSpec, Context webAppCtx) {
- ServletHandler handler = webAppCtx.getServletHandler();
- for (String name : filterNames) {
- FilterMapping fmap = new FilterMapping();
- fmap.setPathSpec(pathSpec);
- fmap.setFilterName(name);
- fmap.setDispatches(Handler.ALL);
- handler.addFilterMapping(fmap);
- }
- }
-
- /**
- * Get the value in the webapp context.
- *
- * @param name The name of the attribute
- * @return The value of the attribute
- */
- public Object getAttribute(String name) {
- return webAppContext.getAttribute(name);
- }
-
- /**
- * Get the pathname to the webapps files.
- *
- * @return the pathname as a URL
- * @throws IOException if 'webapps' directory cannot be found on CLASSPATH.
- */
- protected String getWebAppsPath() throws IOException {
- // URL url = BSPMaster.class.getClassLoader().getResource("webapps");
- // if (url == null)
- // throw new IOException("webapps not found in CLASSPATH");
- // return url.toString();
- return "src/webapps";
- }
-
- /**
- * Get the port that the server is on
- *
- * @return the port
- */
- public int getPort() {
- return webServer.getConnectors()[0].getLocalPort();
- }
-
- /**
- * Set the min, max number of worker threads (simultaneous connections).
- */
- public void setThreads(int min, int max) {
- BoundedThreadPool pool = (BoundedThreadPool) webServer.getThreadPool();
- pool.setMinThreads(min);
- pool.setMaxThreads(max);
- }
-
- /**
- * Configure an ssl listener on the server.
- *
- * @param addr address to listen on
- * @param keystore location of the keystore
- * @param storPass password for the keystore
- * @param keyPass password for the key
- * @deprecated Use
- * {@link #addSslListener(InetSocketAddress, Configuration, boolean)}
- */
- @Deprecated
- public void addSslListener(InetSocketAddress addr, String keystore,
- String storPass, String keyPass) throws IOException {
- if (webServer.isStarted()) {
- throw new IOException("Failed to add ssl listener");
- }
- SslSocketConnector sslListener = new SslSocketConnector();
- sslListener.setHost(addr.getHostName());
- sslListener.setPort(addr.getPort());
- sslListener.setKeystore(keystore);
- sslListener.setPassword(storPass);
- sslListener.setKeyPassword(keyPass);
- webServer.addConnector(sslListener);
- }
-
- /**
- * Configure an ssl listener on the server.
- *
- * @param addr address to listen on
- * @param sslConf conf to retrieve ssl options
- * @param needClientAuth whether client authentication is required
- */
- public void addSslListener(InetSocketAddress addr, Configuration sslConf,
- boolean needClientAuth) throws IOException {
- if (webServer.isStarted()) {
- throw new IOException("Failed to add ssl listener");
- }
- if (needClientAuth) {
- // setting up SSL truststore for authenticating clients
- System.setProperty("javax.net.ssl.trustStore", sslConf.get(
- "ssl.server.truststore.location", ""));
- System.setProperty("javax.net.ssl.trustStorePassword", sslConf.get(
- "ssl.server.truststore.password", ""));
- System.setProperty("javax.net.ssl.trustStoreType", sslConf.get(
- "ssl.server.truststore.type", "jks"));
- }
- SslSocketConnector sslListener = new SslSocketConnector();
- sslListener.setHost(addr.getHostName());
- sslListener.setPort(addr.getPort());
- sslListener.setKeystore(sslConf.get("ssl.server.keystore.location"));
- sslListener.setPassword(sslConf.get("ssl.server.keystore.password", ""));
- sslListener.setKeyPassword(sslConf.get("ssl.server.keystore.keypassword",
- ""));
- sslListener.setKeystoreType(sslConf.get("ssl.server.keystore.type", "jks"));
- sslListener.setNeedClientAuth(needClientAuth);
- webServer.addConnector(sslListener);
- }
-
- /**
- * Start the server. Does not wait for the server to start.
- */
- public void start() throws IOException {
- try {
- int port = 0;
- int oriPort = listener.getPort(); // The original requested port
- while (true) {
- try {
- port = webServer.getConnectors()[0].getLocalPort();
- LOG.info("Port returned by webServer.getConnectors()[0]."
- + "getLocalPort() before open() is " + port
- + ". Opening the listener on " + oriPort);
- listener.open();
- port = listener.getLocalPort();
- LOG.info("listener.getLocalPort() returned "
- + listener.getLocalPort()
- + " webServer.getConnectors()[0].getLocalPort() returned "
- + webServer.getConnectors()[0].getLocalPort());
- // Workaround to handle the problem reported in HADOOP-4744
- if (port < 0) {
- Thread.sleep(100);
- int numRetries = 1;
- while (port < 0) {
- LOG.warn("listener.getLocalPort returned " + port);
- if (numRetries++ > MAX_RETRIES) {
- throw new Exception(" listener.getLocalPort is returning "
- + "less than 0 even after " + numRetries + " resets");
- }
- for (int i = 0; i < 2; i++) {
- LOG.info("Retrying listener.getLocalPort()");
- port = listener.getLocalPort();
- if (port > 0) {
- break;
- }
- Thread.sleep(200);
- }
- if (port > 0) {
- break;
- }
- LOG.info("Bouncing the listener");
- listener.close();
- Thread.sleep(1000);
- listener.setPort(oriPort == 0 ? 0 : (oriPort += 1));
- listener.open();
- Thread.sleep(100);
- port = listener.getLocalPort();
- }
- } // Workaround end
- LOG.info("Jetty bound to port " + port);
- webServer.start();
- // Workaround for HADOOP-6386
- port = listener.getLocalPort();
- if (port < 0) {
- LOG.warn("Bounds port is " + port + " after webserver start");
- for (int i = 0; i < MAX_RETRIES / 2; i++) {
- try {
- webServer.stop();
- } catch (Exception e) {
- LOG.warn("Can't stop web-server", e);
- }
- Thread.sleep(1000);
-
- listener.setPort(oriPort == 0 ? 0 : (oriPort += 1));
- listener.open();
- Thread.sleep(100);
- webServer.start();
- LOG.info(i + "attempts to restart webserver");
- port = listener.getLocalPort();
- if (port > 0)
- break;
- }
- if (port < 0)
- throw new Exception("listener.getLocalPort() is returning "
- + "less than 0 even after " + MAX_RETRIES + " resets");
- }
- // End of HADOOP-6386 workaround
- break;
- } catch (IOException ex) {
- // if this is a bind exception,
- // then try the next port number.
- if (ex instanceof BindException) {
- if (!findPort) {
- throw (BindException) ex;
- }
- } else {
- LOG.info("HttpServer.start() threw a non Bind IOException");
- throw ex;
- }
- } catch (MultiException ex) {
- LOG.info("HttpServer.start() threw a MultiException");
- throw ex;
- }
- listener.setPort((oriPort += 1));
- }
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException("Problem starting http server", e);
- }
- }
-
- /**
- * stop the server
- */
- public void stop() throws Exception {
- listener.close();
- webServer.stop();
- }
-
- public void join() throws InterruptedException {
- webServer.join();
- }
-
- /**
- * A very simple servlet to serve up a text representation of the current
- * stack traces. It both returns the stacks to the caller and logs them.
- * Currently the stack traces are done sequentially rather than exactly the
- * same data.
- */
- public static class StackServlet extends HttpServlet {
- private static final long serialVersionUID = -6284183679759467039L;
-
- @Override
- public void doGet(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
-
- PrintWriter out = new PrintWriter(response.getOutputStream());
- ReflectionUtils.printThreadInfo(out, "");
- out.close();
- ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
- }
- }
-}
Index: src/java/org/apache/hama/http/package.html
===================================================================
--- src/java/org/apache/hama/http/package.html (revision 1152138)
+++ src/java/org/apache/hama/http/package.html (working copy)
@@ -1,23 +0,0 @@
-
-
-
-
-
-Contains the administrative web interfaces.
-
-
Index: src/java/org/apache/hama/MiniZooKeeperCluster.java
===================================================================
--- src/java/org/apache/hama/MiniZooKeeperCluster.java (revision 1152138)
+++ src/java/org/apache/hama/MiniZooKeeperCluster.java (working copy)
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Reader;
-import java.net.BindException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.zookeeper.server.NIOServerCnxn;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.apache.zookeeper.server.persistence.FileTxnLog;
-
-/**
- * This class starts and runs the MiniZookeeperCluster.
- */
-public class MiniZooKeeperCluster {
- private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class);
-
- private static final int TICK_TIME = 2000;
- private static final int CONNECTION_TIMEOUT = 30000;
-
- private boolean started;
- private int clientPort = 21810; // use non-standard port
-
- private NIOServerCnxn.Factory standaloneServerFactory;
- private int tickTime = 0;
-
- /** Create mini Zookeeper cluster. */
- public MiniZooKeeperCluster() {
- this.started = false;
- }
-
- public void setClientPort(int clientPort) {
- this.clientPort = clientPort;
- }
-
- public void setTickTime(int tickTime) {
- this.tickTime = tickTime;
- }
-
- private static void setupTestEnv() {
- // during the tests we run with 100K prealloc in the logs.
- // on windows systems prealloc of 64M was seen to take ~15seconds
- // resulting in test failure (client timeout on first session).
- // set env and directly in order to handle static init/gc issues
- System.setProperty("zookeeper.preAllocSize", "100");
- FileTxnLog.setPreallocSize(100);
- }
-
- /**
- * @param baseDir
- * @return ClientPort server bound to.
- * @throws IOException
- * @throws InterruptedException
- */
- public int startup(File baseDir) throws IOException,
- InterruptedException {
-
- setupTestEnv();
-
- shutdown();
-
- File dir = new File(baseDir, "zookeeper").getAbsoluteFile();
- recreateDir(dir);
-
- int tickTimeToUse;
- if (this.tickTime > 0) {
- tickTimeToUse = this.tickTime;
- } else {
- tickTimeToUse = TICK_TIME;
- }
- ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
- while (true) {
- try {
- standaloneServerFactory =
- new NIOServerCnxn.Factory(new InetSocketAddress(clientPort));
- } catch (BindException e) {
- LOG.info("Faild binding ZK Server to client port: " + clientPort);
- //this port is already in use. try to use another
- clientPort++;
- continue;
- }
- break;
- }
- standaloneServerFactory.startup(server);
-
- if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for startup of standalone server");
- }
-
- started = true;
-
- return clientPort;
- }
-
- private void recreateDir(File dir) throws IOException {
- if (dir.exists()) {
- FileUtil.fullyDelete(dir);
- }
- try {
- dir.mkdirs();
- } catch (SecurityException e) {
- throw new IOException("creating dir: " + dir, e);
- }
- }
-
- /**
- * @throws IOException
- */
- public void shutdown() throws IOException {
- if (!started) {
- return;
- }
-
- standaloneServerFactory.shutdown();
- if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for shutdown of standalone server");
- }
-
- started = false;
- }
-
- // XXX: From o.a.zk.t.ClientBase
- private static boolean waitForServerDown(int port, long timeout) {
- long start = System.currentTimeMillis();
- while (true) {
- try {
- Socket sock = new Socket("localhost", port);
- try {
- OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes());
- outstream.flush();
- } finally {
- sock.close();
- }
- } catch (IOException e) {
- return true;
- }
-
- if (System.currentTimeMillis() > start + timeout) {
- break;
- }
- try {
- Thread.sleep(250);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- return false;
- }
-
- // XXX: From o.a.zk.t.ClientBase
- private static boolean waitForServerUp(int port, long timeout) {
- long start = System.currentTimeMillis();
- while (true) {
- try {
- Socket sock = new Socket("localhost", port);
- BufferedReader reader = null;
- try {
- OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes());
- outstream.flush();
-
- Reader isr = new InputStreamReader(sock.getInputStream());
- reader = new BufferedReader(isr);
- String line = reader.readLine();
- if (line != null && line.startsWith("Zookeeper version:")) {
- return true;
- }
- } finally {
- sock.close();
- if (reader != null) {
- reader.close();
- }
- }
- } catch (IOException e) {
- // ignore as this is expected
- LOG.info("server localhost:" + port + " not up " + e);
- }
-
- if (System.currentTimeMillis() > start + timeout) {
- break;
- }
- try {
- Thread.sleep(250);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- return false;
- }
-}
Index: src/java/org/apache/hama/zookeeper/QuorumPeer.java
===================================================================
--- src/java/org/apache/hama/zookeeper/QuorumPeer.java (revision 1152138)
+++ src/java/org/apache/hama/zookeeper/QuorumPeer.java (working copy)
@@ -1,354 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.zookeeper;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.Properties;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.DNS;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hama.Constants;
-import org.apache.hama.HamaConfiguration;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.apache.zookeeper.server.quorum.QuorumPeerMain;
-
-/**
- * This class starts and runs the QuorumPeers
- */
-public class QuorumPeer implements Constants {
- private static final Log LOG = LogFactory.getLog(QuorumPeer.class);
-
- private static final String VARIABLE_START = "${";
- private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
- private static final String VARIABLE_END = "}";
- private static final int VARIABLE_END_LENGTH = VARIABLE_END.length();
-
- private static final String ZK_CFG_PROPERTY = "hama.zookeeper.property.";
- private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
-
- /**
- * Parse ZooKeeper configuration from Hama XML config and run a QuorumPeer.
- * @param baseConf Hadoop Configuration.
- */
- public static void run(Configuration baseConf) {
- Configuration conf = new HamaConfiguration(baseConf);
- try {
- Properties zkProperties = makeZKProps(conf);
- writeMyID(zkProperties);
- QuorumPeerConfig zkConfig = new QuorumPeerConfig();
- zkConfig.parseProperties(zkProperties);
- runZKServer(zkConfig);
- } catch (Exception e) {
- LOG.error("Exception during ZooKeeper startup - exiting...",e);
- System.exit(-1);
- }
- }
-
- private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException {
- if (zkConfig.isDistributed()) {
- QuorumPeerMain qp = new QuorumPeerMain();
- qp.runFromConfig(zkConfig);
- } else {
- ZooKeeperServerMain zk = new ZooKeeperServerMain();
- ServerConfig serverConfig = new ServerConfig();
- serverConfig.readFrom(zkConfig);
- zk.runFromConfig(serverConfig);
- }
- }
-
- private static boolean addressIsLocalHost(String address) {
- return address.equals("localhost") || address.equals("127.0.0.1");
- }
-
- private static void writeMyID(Properties properties) throws IOException {
- long myId = -1;
-
- Configuration conf = new HamaConfiguration();
- String myAddress = DNS.getDefaultHost(
- conf.get("hama.zookeeper.dns.interface","default"),
- conf.get("hama.zookeeper.dns.nameserver","default"));
-
- List ips = new ArrayList();
-
- // Add what could be the best (configured) match
- ips.add(myAddress.contains(".") ?
- myAddress :
- StringUtils.simpleHostname(myAddress));
-
- // For all nics get all hostnames and IPs
- Enumeration> nics = NetworkInterface.getNetworkInterfaces();
- while(nics.hasMoreElements()) {
- Enumeration> rawAdrs =
- ((NetworkInterface)nics.nextElement()).getInetAddresses();
- while(rawAdrs.hasMoreElements()) {
- InetAddress inet = (InetAddress) rawAdrs.nextElement();
- ips.add(StringUtils.simpleHostname(inet.getHostName()));
- ips.add(inet.getHostAddress());
- }
- }
-
- for (Entry
- *
- *
Clients can query for the latest ClusterStatus, via
- * {@link BSPJobClient#getClusterStatus(boolean)}.
- *
- * @see BSPMaster
- */
-public class ClusterStatus implements Writable {
-
- private int numActiveGrooms;
- private Map activeGrooms = new HashMap();
- private int tasks;
- private int maxTasks;
- private BSPMaster.State state;
-
- /**
- *
- */
- public ClusterStatus() {}
-
- public ClusterStatus(int grooms, int tasks, int maxTasks, BSPMaster.State state) {
- this.numActiveGrooms = grooms;
- this.tasks = tasks;
- this.maxTasks = maxTasks;
- this.state = state;
- }
-
- public ClusterStatus(Map activeGrooms, int tasks, int maxTasks,
- BSPMaster.State state) {
- this(activeGrooms.size(), tasks, maxTasks, state);
- this.activeGrooms = activeGrooms;
- }
-
- /**
- * Get the number of groom servers in the cluster.
- *
- * @return the number of groom servers in the cluster.
- */
- public int getGroomServers() {
- return numActiveGrooms;
- }
-
- /**
- * Get the names of groom servers, and their peers, in the cluster.
- *
- * @return the active groom servers in the cluster.
- */
- public Map getActiveGroomNames() {
- return activeGrooms;
- }
-
- /**
- * Get the number of currently running tasks in the cluster.
- *
- * @return the number of currently running tasks in the cluster.
- */
- public int getTasks() {
- return tasks;
- }
-
- /**
- * Get the maximum capacity for running tasks in the cluster.
- *
- * @return the maximum capacity for running tasks in the cluster.
- */
- public int getMaxTasks() {
- return maxTasks;
- }
-
- /**
- * Get the current state of the BSPMaster,
- * as {@link BSPMaster.State}
- *
- * @return the current state of the BSPMaster.
- */
- public BSPMaster.State getBSPMasterState() {
- return state;
- }
-
- //////////////////////////////////////////////
- // Writable
- //////////////////////////////////////////////
- @Override
- public void write(DataOutput out) throws IOException {
- if (activeGrooms.isEmpty()) {
- out.writeInt(numActiveGrooms);
- out.writeBoolean(false);
- } else {
- out.writeInt(activeGrooms.size());
- out.writeBoolean(true);
-
- String[] groomNames = activeGrooms.keySet().toArray(new String[0]);
- List peerNames = new ArrayList();
-
- for (String groomName : groomNames) {
- peerNames.add(activeGrooms.get(groomName));
- }
-
- WritableUtils.writeCompressedStringArray(out, groomNames);
- WritableUtils.writeCompressedStringArray(out, peerNames.toArray(new String[0]));
- }
- out.writeInt(tasks);
- out.writeInt(maxTasks);
- WritableUtils.writeEnum(out, state);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- numActiveGrooms = in.readInt();
- boolean groomListFollows = in.readBoolean();
-
- if (groomListFollows) {
- String[] groomNames = WritableUtils.readCompressedStringArray(in);
- String[] peerNames = WritableUtils.readCompressedStringArray(in);
- activeGrooms = new HashMap(groomNames.length);
-
- for (int i = 0; i < groomNames.length; i++) {
- activeGrooms.put(groomNames[i], peerNames[i]);
- }
- }
-
- tasks = in.readInt();
- maxTasks = in.readInt();
- state = WritableUtils.readEnum(in, BSPMaster.State.class);
- }
-}
Index: src/java/org/apache/hama/bsp/FCFSQueue.java
===================================================================
--- src/java/org/apache/hama/bsp/FCFSQueue.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/FCFSQueue.java (working copy)
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.util.Collection;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-class FCFSQueue implements Queue {
-
- public static final Log LOG = LogFactory.getLog(FCFSQueue.class);
- private final String name;
- private BlockingQueue queue = new LinkedBlockingQueue();
-
- public FCFSQueue(String name) {
- this.name = name;
- }
-
- @Override
- public String getName() {
- return this.name;
- }
-
- @Override
- public void addJob(JobInProgress job) {
- try {
- queue.put(job);
- } catch (InterruptedException ie) {
- LOG.error("Fail to add a job to the " + this.name + " queue.", ie);
- }
- }
-
- @Override
- public void removeJob(JobInProgress job) {
- queue.remove(job);
- }
-
- @Override
- public JobInProgress removeJob() {
- try {
- return queue.take();
- } catch (InterruptedException ie) {
- LOG.error("Fail to remove a job from the " + this.name + " queue.", ie);
- }
- return null;
- }
-
- @Override
- public Collection jobs() {
- return queue;
- }
-
-}
Index: src/java/org/apache/hama/bsp/JobInProgress.java
===================================================================
--- src/java/org/apache/hama/bsp/JobInProgress.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobInProgress.java (working copy)
@@ -1,360 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-
-/**
- * JobInProgress maintains all the info for keeping a Job on the straight and
- * narrow. It keeps its JobProfile and its latest JobStatus, plus a set of
- * tables for doing bookkeeping of its Tasks.ss
- */
-class JobInProgress {
-
- /**
- * Used when the a kill is issued to a job which is initializing.
- */
- static class KillInterruptedException extends InterruptedException {
- private static final long serialVersionUID = 1L;
-
- public KillInterruptedException(String msg) {
- super(msg);
- }
- }
-
- static final Log LOG = LogFactory.getLog(JobInProgress.class);
- boolean tasksInited = false;
- boolean jobInited = false;
-
- Configuration conf;
- JobProfile profile;
- JobStatus status;
- Path jobFile = null;
- Path localJobFile = null;
- Path localJarFile = null;
- private LocalFileSystem localFs;
- // Indicates how many times the job got restarted
- private int restartCount;
-
- long startTime;
- long launchTime;
- long finishTime;
-
- private String jobName;
-
- // private LocalFileSystem localFs;
- private BSPJobID jobId;
- final BSPMaster master;
- TaskInProgress tasks[] = new TaskInProgress[0];
- private long superstepCounter;
-
- int numBSPTasks = 0;
- int clusterSize;
-
- public JobInProgress(BSPJobID jobId, Path jobFile, BSPMaster master,
- Configuration conf) throws IOException {
- this.conf = conf;
- this.jobId = jobId;
- this.localFs = FileSystem.getLocal(conf);
- this.jobFile = jobFile;
- this.master = master;
- this.status = new JobStatus(jobId, null, 0L, 0L,
- JobStatus.State.PREP.value());
- this.startTime = System.currentTimeMillis();
- this.superstepCounter = 0;
- this.restartCount = 0;
-
- this.localJobFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId
- + ".xml");
- this.localJarFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId
- + ".jar");
-
- Path jobDir = master.getSystemDirectoryForJob(jobId);
- FileSystem fs = jobDir.getFileSystem(conf);
- fs.copyToLocalFile(jobFile, localJobFile);
- BSPJob job = new BSPJob(jobId, localJobFile.toString());
- this.numBSPTasks = job.getNumBspTask();
-
- this.profile = new JobProfile(job.getUser(), jobId, jobFile.toString(),
- job.getJobName());
-
- this.setJobName(job.getJobName());
-
- status.setUsername(job.getUser());
- status.setStartTime(startTime);
-
- String jarFile = job.getJar();
- if (jarFile != null) {
- fs.copyToLocalFile(new Path(jarFile), localJarFile);
- }
-
- }
-
- public JobProfile getProfile() {
- return profile;
- }
-
- public JobStatus getStatus() {
- return status;
- }
-
- public synchronized long getLaunchTime() {
- return launchTime;
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public long getFinishTime() {
- return finishTime;
- }
-
- /**
- * @return the number of desired tasks.
- */
- public int desiredBSPTasks() {
- return numBSPTasks;
- }
-
- /**
- * @return The JobID of this JobInProgress.
- */
- public BSPJobID getJobID() {
- return jobId;
- }
-
- public synchronized TaskInProgress findTaskInProgress(TaskID id) {
- if (areTasksInited()) {
- for (TaskInProgress tip : tasks) {
- if (tip.getTaskId().equals(id)) {
- return tip;
- }
- }
- }
- return null;
- }
-
- public synchronized boolean areTasksInited() {
- return this.tasksInited;
- }
-
- public String toString() {
- return "jobName:" + profile.getJobName() + "\n" + "submit user:"
- + profile.getUser() + "\n" + "JobId:" + jobId + "\n" + "JobFile:"
- + jobFile + "\n";
- }
-
- // ///////////////////////////////////////////////////
- // Create/manage tasks
- // ///////////////////////////////////////////////////
-
- public synchronized void initTasks() throws IOException {
- if (tasksInited) {
- return;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("numBSPTasks: " + numBSPTasks);
- }
-
- // adjust number of map tasks to actual number of splits
- this.tasks = new TaskInProgress[numBSPTasks];
- for (int i = 0; i < numBSPTasks; i++) {
- tasks[i] = new TaskInProgress(getJobID(), this.jobFile.toString(),
- this.master, this.conf, this, i);
- }
-
- // Update job status
- this.status = new JobStatus(this.status.getJobID(), this.profile.getUser(),
- 0L, 0L, JobStatus.RUNNING);
-
- tasksInited = true;
- LOG.debug("Job is initialized.");
- }
-
- public synchronized Task obtainNewTask(GroomServerStatus status,
- int clusterSize) {
- this.clusterSize = clusterSize;
-
- if (this.status.getRunState() != JobStatus.RUNNING) {
- LOG.info("Cannot create task split for " + profile.getJobID());
- return null;
- }
-
- Task result = null;
- try {
- for (int i = 0; i < tasks.length; i++) {
- if (!tasks[i].isRunning() && !tasks[i].isComplete()) {
- result = tasks[i].getTaskToRun(status);
- break;
- }
- }
-
- } catch (IOException e) {
- LOG.error("Exception while obtaining new task!", e);
- }
- return result;
- }
-
- public synchronized void completedTask(TaskInProgress tip, TaskStatus status) {
- TaskAttemptID taskid = status.getTaskId();
- updateTaskStatus(tip, status);
- LOG.info("Taskid '" + taskid + "' has finished successfully.");
- tip.completed(taskid);
-
- //
- // If all tasks are complete, then the job is done!
- //
-
- boolean allDone = true;
- for (TaskInProgress taskInProgress : tasks) {
- if (!taskInProgress.isComplete()) {
- allDone = false;
- break;
- }
- }
-
- if (allDone) {
- this.status = new JobStatus(this.status.getJobID(),
- this.profile.getUser(), superstepCounter, superstepCounter,
- superstepCounter, JobStatus.SUCCEEDED, superstepCounter);
- this.finishTime = System.currentTimeMillis();
- this.status.setFinishTime(this.finishTime);
-
- LOG.debug("Job successfully done.");
-
- garbageCollect();
- }
- }
-
- public void failedTask(TaskInProgress tip, TaskStatus status) {
- TaskAttemptID taskid = status.getTaskId();
- updateTaskStatus(tip, status);
- LOG.info("Taskid '" + taskid + "' has failed.");
- tip.terminated(taskid);
-
- //
- // If all tasks are complete, then the job is done!
- //
-
- boolean allDone = true;
- for (TaskInProgress taskInProgress : tasks) {
- if (!taskInProgress.isFailed()) {
- allDone = false;
- break;
- }
- }
-
- if (allDone) {
- this.status = new JobStatus(this.status.getJobID(),
- this.profile.getUser(), superstepCounter, superstepCounter,
- superstepCounter, JobStatus.FAILED, superstepCounter);
- this.finishTime = System.currentTimeMillis();
- this.status.setFinishTime(this.finishTime);
-
- LOG.debug("Job failed.");
-
- garbageCollect();
- }
- }
-
- public synchronized void updateTaskStatus(TaskInProgress tip,
- TaskStatus taskStatus) {
- tip.updateStatus(taskStatus); // update tip
-
- if (superstepCounter < taskStatus.getSuperstepCount()) {
- superstepCounter = taskStatus.getSuperstepCount();
- // TODO Later, we have to update JobInProgress status here
-
- }
- }
-
- public synchronized void kill() {
- if (status.getRunState() != JobStatus.KILLED) {
- this.status = new JobStatus(status.getJobID(), this.profile.getUser(),
- 0L, 0L, 0L, JobStatus.KILLED);
- this.finishTime = System.currentTimeMillis();
- this.status.setFinishTime(this.finishTime);
- //
- // kill all TIPs.
- //
- for (int i = 0; i < tasks.length; i++) {
- tasks[i].kill();
- }
-
- garbageCollect();
- }
-
- }
-
- /**
- * The job is dead. We're now GC'ing it, getting rid of the job from all
- * tables. Be sure to remove all of this job's tasks from the various tables.
- */
- synchronized void garbageCollect() {
- try {
- // Definitely remove the local-disk copy of the job file
- if (localJobFile != null) {
- localFs.delete(localJobFile, true);
- localJobFile = null;
- }
- if (localJarFile != null) {
- localFs.delete(localJarFile, true);
- localJarFile = null;
- }
-
- // JobClient always creates a new directory with job files
- // so we remove that directory to cleanup
- FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(profile.getJobFile()).getParent(), true);
-
- } catch (IOException e) {
- LOG.info("Error cleaning up " + profile.getJobID() + ": " + e);
- }
- }
-
- /**
- * Get the number of times the job has restarted
- */
- int getNumRestarts() {
- return restartCount;
- }
-
- /**
- * @param jobName the jobName to set
- */
- public void setJobName(String jobName) {
- this.jobName = jobName;
- }
-
- /**
- * @return the jobName
- */
- public String getJobName() {
- return jobName;
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSPMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPMessage.java (working copy)
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import org.apache.hadoop.io.Writable;
-
-/**
- * BSPMessage consists of the tag and the arbitrary amount of data to be
- * communicated.
- */
-public abstract class BSPMessage implements Messagable, Writable {
-
- /**
- * BSP messages are typically identified with tags. This allows to get the tag
- * of data.
- *
- * @return tag of data of BSP message
- */
- public abstract Object getTag();
-
- /**
- * @return data of BSP message
- */
- public abstract Object getData();
-
-}
Index: src/java/org/apache/hama/bsp/ID.java
===================================================================
--- src/java/org/apache/hama/bsp/ID.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/ID.java (working copy)
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.WritableComparable;
-
-/**
- * A general identifier, which internally stores the id as an integer. This is
- * the super class of {@link BSPJobID}, {@link TaskID} and {@link TaskAttemptID}
- * .
- */
-public abstract class ID implements WritableComparable {
- protected static final char SEPARATOR = '_';
- protected int id;
-
- public ID(int id) {
- this.id = id;
- }
-
- protected ID() {
- }
-
- public int getId() {
- return id;
- }
-
- @Override
- public String toString() {
- return String.valueOf(id);
- }
-
- @Override
- public int hashCode() {
- return Integer.valueOf(id).hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null)
- return false;
- if (o.getClass() == this.getClass()) {
- ID that = (ID) o;
- return this.id == that.id;
- } else
- return false;
- }
-
- public int compareTo(ID that) {
- return this.id - that.id;
- }
-
- public void readFields(DataInput in) throws IOException {
- this.id = in.readInt();
- }
-
- public void write(DataOutput out) throws IOException {
- out.writeInt(id);
- }
-}
Index: src/java/org/apache/hama/bsp/IntegerDoubleMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/IntegerDoubleMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/IntegerDoubleMessage.java (working copy)
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * A message that consists of a int tag and a double data.
- */
-public class IntegerDoubleMessage extends BSPMessage {
-
- int tag;
- double data;
-
- public IntegerDoubleMessage() {
- super();
- }
-
- public IntegerDoubleMessage(int tag, double data) {
- super();
- this.tag = tag;
- this.data = data;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(tag);
- out.writeDouble(data);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- tag = in.readInt();
- data = in.readDouble();
- }
-
- @Override
- public Integer getTag() {
- return tag;
- }
-
- @Override
- public Double getData() {
- return data;
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSPPeer.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPPeer.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPPeer.java (working copy)
@@ -1,423 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hama.Constants;
-import org.apache.hama.ipc.BSPPeerProtocol;
-import org.apache.hama.util.Bytes;
-import org.apache.hama.zookeeper.QuorumPeer;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.Stat;
-
-/**
- * This class represents a BSP peer.
- */
-public class BSPPeer implements Watcher, BSPPeerInterface {
-
- public static final Log LOG = LogFactory.getLog(BSPPeer.class);
-
- private Configuration conf;
- private BSPJob jobConf;
-
- private Server server = null;
- private ZooKeeper zk = null;
- private volatile Integer mutex = 0;
-
- private final String bspRoot;
- private final String quorumServers;
-
- private final Map peers = new ConcurrentHashMap();
- private final Map> outgoingQueues = new ConcurrentHashMap>();
- private ConcurrentLinkedQueue localQueue = new ConcurrentLinkedQueue();
- private ConcurrentLinkedQueue localQueueForNextIteration = new ConcurrentLinkedQueue();
- private final Map peerSocketCache = new ConcurrentHashMap();
-
- private SortedSet allPeerNames = new TreeSet();
- private InetSocketAddress peerAddress;
- private TaskStatus currentTaskStatus;
-
- private TaskAttemptID taskid;
- private BSPPeerProtocol umbilical;
-
- /**
- * Protected default constructor for LocalBSPRunner.
- */
- protected BSPPeer() {
- bspRoot = null;
- quorumServers = null;
- }
-
- /**
- * Constructor
- *
- * @param umbilical
- * @param taskid
- */
- public BSPPeer(Configuration conf, TaskAttemptID taskid,
- BSPPeerProtocol umbilical) throws IOException {
- this.conf = conf;
- this.taskid = taskid;
- this.umbilical = umbilical;
-
- String bindAddress = conf.get(Constants.PEER_HOST,
- Constants.DEFAULT_PEER_HOST);
- int bindPort = conf
- .getInt(Constants.PEER_PORT, Constants.DEFAULT_PEER_PORT);
- bspRoot = conf.get(Constants.ZOOKEEPER_ROOT,
- Constants.DEFAULT_ZOOKEEPER_ROOT);
- quorumServers = QuorumPeer.getZKQuorumServersString(conf);
- LOG.debug("Quorum " + quorumServers);
-
- // TODO: may require to dynamic reflect the underlying
- // network e.g. ip address, port.
- peerAddress = new InetSocketAddress(bindAddress, bindPort);
- reinitialize();
- }
-
- public void reinitialize() {
- try {
- LOG.debug("reinitialize(): " + getPeerName());
- server = RPC.getServer(this, peerAddress.getHostName(),
- peerAddress.getPort(), conf);
- server.start();
- LOG.info(" BSPPeer address:" + peerAddress.getHostName() + " port:"
- + peerAddress.getPort());
- } catch (IOException e) {
- LOG.error("Exception during reinitialization!", e);
- }
-
- try {
- zk = new ZooKeeper(quorumServers, conf.getInt(
- Constants.ZOOKEEPER_SESSION_TIMEOUT, 1200000), this);
- } catch (IOException e) {
- LOG.error("Exception during reinitialization!", e);
- }
-
- Stat s = null;
- if (zk != null) {
- try {
- s = zk.exists(Constants.DEFAULT_ZOOKEEPER_ROOT, false);
- } catch (Exception e) {
- LOG.error(s, e);
- }
-
- if (s == null) {
- try {
- zk.create(Constants.DEFAULT_ZOOKEEPER_ROOT, new byte[0],
- Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- } catch (KeeperException e) {
- LOG.error(e);
- } catch (InterruptedException e) {
- LOG.error(e);
- }
- }
- }
-
- }
-
- @Override
- public BSPMessage getCurrentMessage() throws IOException {
- return localQueue.poll();
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.BSPPeerInterface#send(java.net.InetSocketAddress,
- * org.apache.hadoop.io.Writable, org.apache.hadoop.io.Writable)
- */
- @Override
- public void send(String peerName, BSPMessage msg) throws IOException {
- if (peerName.equals(getPeerName())) {
- LOG.debug("Local send bytes (" + msg.getData().toString() + ")");
- localQueueForNextIteration.add(msg);
- } else {
- LOG.debug("Send bytes (" + msg.getData().toString() + ") to " + peerName);
- InetSocketAddress targetPeerAddress = null;
- // Get socket for target peer.
- if (peerSocketCache.containsKey(peerName)) {
- targetPeerAddress = peerSocketCache.get(peerName);
- } else {
- targetPeerAddress = getAddress(peerName);
- peerSocketCache.put(peerName, targetPeerAddress);
- }
- ConcurrentLinkedQueue queue = outgoingQueues
- .get(targetPeerAddress);
- if (queue == null) {
- queue = new ConcurrentLinkedQueue();
- }
- queue.add(msg);
- outgoingQueues.put(targetPeerAddress, queue);
- }
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.BSPPeerInterface#sync()
- */
- @Override
- public void sync() throws IOException, KeeperException, InterruptedException {
- enterBarrier();
- long startTime = System.currentTimeMillis();
- Iterator>> it = this.outgoingQueues
- .entrySet().iterator();
-
- while (it.hasNext()) {
- Entry> entry = it
- .next();
-
- BSPPeerInterface peer = peers.get(entry.getKey());
- if (peer == null) {
- peer = getBSPPeerConnection(entry.getKey());
- }
- Iterable messages = entry.getValue();
- BSPMessageBundle bundle = new BSPMessageBundle();
- for (BSPMessage message : messages) {
- bundle.addMessage(message);
- }
- peer.put(bundle);
- }
-
- if ((System.currentTimeMillis() - startTime) < 200) {
- Thread.sleep(200);
- }
-
- leaveBarrier();
- currentTaskStatus.incrementSuperstepCount();
- umbilical.incrementSuperstepCount(taskid);
-
- startTime = System.currentTimeMillis();
- // Clear outgoing queues.
- clearOutgoingQueues();
-
- // Add non-processed messages from this iteration for the next's queue.
- while (!localQueue.isEmpty()) {
- BSPMessage message = localQueue.poll();
- localQueueForNextIteration.add(message);
- }
- // Switch local queues.
- localQueue = localQueueForNextIteration;
- localQueueForNextIteration = new ConcurrentLinkedQueue();
-
- // TODO: This is a quite temporary solution of HAMA-387.
- // If zk.getChildren() response is slower than 200 milliseconds,
- // BSP system will be hanged.
-
- // We have to consider another way to avoid this problem.
- if ((System.currentTimeMillis() - startTime) < 200) {
- Thread.sleep(200); // at least wait
- }
- }
-
- protected boolean enterBarrier() throws KeeperException, InterruptedException {
- LOG.debug("[" + getPeerName() + "] enter the enterbarrier: "
- + this.getSuperstepCount());
- zk.create(bspRoot + "/" + getPeerName(),
- Bytes.toBytes(this.getSuperstepCount()), Ids.OPEN_ACL_UNSAFE,
- CreateMode.EPHEMERAL);
-
- while (true) {
- synchronized (mutex) {
- List list = zk.getChildren(bspRoot, true);
-
- if (list.size() < jobConf.getNumBspTask()) {
- mutex.wait();
- } else {
- return true;
- }
- }
- }
- }
-
- protected boolean leaveBarrier() throws KeeperException, InterruptedException {
- zk.delete(bspRoot + "/" + getPeerName(), 0);
- while (true) {
- synchronized (mutex) {
- List list = zk.getChildren(bspRoot, true);
-
- if (list.size() > 0) {
- mutex.wait();
- } else {
- return true;
- }
- }
- }
- }
-
- @Override
- public void process(WatchedEvent event) {
- synchronized (mutex) {
- mutex.notify();
- }
- }
-
- public void clear() {
- this.localQueue.clear();
- this.outgoingQueues.clear();
- }
-
- @Override
- public void close() throws IOException {
- server.stop();
- }
-
- @Override
- public void put(BSPMessage msg) throws IOException {
- this.localQueueForNextIteration.add(msg);
- }
-
- @Override
- public void put(BSPMessageBundle messages) throws IOException {
- for (BSPMessage message : messages.getMessages()) {
- this.localQueueForNextIteration.add(message);
- }
- }
-
- @Override
- public long getProtocolVersion(String arg0, long arg1) throws IOException {
- return BSPPeerInterface.versionID;
- }
-
- protected BSPPeerInterface getBSPPeerConnection(InetSocketAddress addr) {
- BSPPeerInterface peer;
- synchronized (this.peers) {
- peer = peers.get(addr);
-
- if (peer == null) {
- try {
- peer = (BSPPeerInterface) RPC.getProxy(BSPPeerInterface.class,
- BSPPeerInterface.versionID, addr, this.conf);
- } catch (IOException e) {
-
- }
- this.peers.put(addr, peer);
- }
- }
-
- return peer;
- }
-
- /**
- * @return the string as host:port of this Peer
- */
- public String getPeerName() {
- return peerAddress.getHostName() + ":" + peerAddress.getPort();
- }
-
- private InetSocketAddress getAddress(String peerName) {
- String[] peerAddrParts = peerName.split(":");
- return new InetSocketAddress(peerAddrParts[0],
- Integer.parseInt(peerAddrParts[1]));
- }
-
- @Override
- public String[] getAllPeerNames() {
- return allPeerNames.toArray(new String[0]);
- }
-
- /**
- * To be invoked by the Groom Server with a list of peers received from an
- * heartbeat response (BSPMaster).
- *
- * @param allPeers
- */
- void setAllPeerNames(Collection allPeerNames) {
- this.allPeerNames = new TreeSet(allPeerNames);
- }
-
- /**
- * @return the number of messages
- */
- public int getNumCurrentMessages() {
- return localQueue.size();
- }
-
- /**
- * Sets the current status
- *
- * @param currentTaskStatus
- */
- public void setCurrentTaskStatus(TaskStatus currentTaskStatus) {
- this.currentTaskStatus = currentTaskStatus;
- }
-
- /**
- * @return the count of current super-step
- */
- public long getSuperstepCount() {
- return currentTaskStatus.getSuperstepCount();
- }
-
- /**
- * Sets the job configuration
- *
- * @param jobConf
- */
- public void setJobConf(BSPJob jobConf) {
- this.jobConf = jobConf;
- }
-
- /**
- * @return the size of local queue
- */
- public int getLocalQueueSize() {
- return localQueue.size();
- }
-
- /**
- * @return the size of outgoing queue
- */
- public int getOutgoingQueueSize() {
- return outgoingQueues.size();
- }
-
- /**
- * Clears local queue
- */
- public void clearLocalQueue() {
- this.localQueue.clear();
- }
-
- /**
- * Clears outgoing queues
- */
- public void clearOutgoingQueues() {
- this.outgoingQueues.clear();
- }
-}
Index: src/java/org/apache/hama/bsp/BSPTaskRunner.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPTaskRunner.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPTaskRunner.java (working copy)
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * Base class that runs a task in a separate process.
- */
-public class BSPTaskRunner extends TaskRunner {
-
- public static final Log LOG = LogFactory.getLog(BSPTaskRunner.class);
-
- public BSPTaskRunner(BSPTask bspTask, GroomServer groom, BSPJob conf) {
- super(bspTask, groom, conf);
- }
-
-}
Index: src/java/org/apache/hama/bsp/IntegerMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/IntegerMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/IntegerMessage.java (working copy)
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * A message that consists of a string tag and a int data.
- */
-public class IntegerMessage extends BSPMessage {
-
- String tag;
- int data;
-
- public IntegerMessage() {
- super();
- }
-
- public IntegerMessage(String tag, int data) {
- super();
- this.tag = tag;
- this.data = data;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(tag);
- out.writeInt(data);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- tag = in.readUTF();
- data = in.readInt();
- }
-
- @Override
- public String getTag() {
- return tag;
- }
-
- @Override
- public Integer getData() {
- return data;
- }
-
-}
\ No newline at end of file
Index: src/java/org/apache/hama/bsp/Task.java
===================================================================
--- src/java/org/apache/hama/bsp/Task.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/Task.java (working copy)
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hama.ipc.BSPPeerProtocol;
-
-/**
- * Base class for tasks.
- */
-public abstract class Task implements Writable {
- public static final Log LOG = LogFactory.getLog(Task.class);
- // //////////////////////////////////////////
- // Fields
- // //////////////////////////////////////////
-
- protected BSPJobID jobId;
- protected String jobFile;
- protected TaskAttemptID taskId;
- protected int partition;
-
- protected LocalDirAllocator lDirAlloc;
-
- public Task() {
- jobId = new BSPJobID();
- taskId = new TaskAttemptID();
- }
-
- public Task(BSPJobID jobId, String jobFile, TaskAttemptID taskId,
- int partition) {
- this.jobId = jobId;
- this.jobFile = jobFile;
- this.taskId = taskId;
- this.partition = partition;
- }
-
- // //////////////////////////////////////////
- // Accessors
- // //////////////////////////////////////////
- public void setJobFile(String jobFile) {
- this.jobFile = jobFile;
- }
-
- public String getJobFile() {
- return jobFile;
- }
-
- public TaskAttemptID getTaskAttemptId() {
- return this.taskId;
- }
-
- public TaskAttemptID getTaskID() {
- return taskId;
- }
-
- /**
- * Get the job name for this task.
- *
- * @return the job name
- */
- public BSPJobID getJobID() {
- return jobId;
- }
-
- /**
- * Get the index of this task within the job.
- *
- * @return the integer part of the task id
- */
- public int getPartition() {
- return partition;
- }
-
- @Override
- public String toString() {
- return taskId.toString();
- }
-
- // //////////////////////////////////////////
- // Writable
- // //////////////////////////////////////////
- @Override
- public void write(DataOutput out) throws IOException {
- jobId.write(out);
- Text.writeString(out, jobFile);
- taskId.write(out);
- out.writeInt(partition);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- jobId.readFields(in);
- jobFile = Text.readString(in);
- taskId.readFields(in);
- partition = in.readInt();
- }
-
- /**
- * Run this task as a part of the named job. This method is executed in the
- * child process.
- *
- * @param bspPeer for communications
- * @param umbilical for communications with GroomServer
- */
- public abstract void run(BSPJob job, BSPPeer bspPeer, BSPPeerProtocol umbilical)
- throws IOException;
-
- public abstract BSPTaskRunner createRunner(GroomServer groom);
-
- public void done(BSPPeerProtocol umbilical) throws IOException {
- umbilical.done(getTaskID(), true);
- }
-
- public abstract BSPJob getConf();
- public abstract void setConf(BSPJob localJobConf);
-
-}
Index: src/java/org/apache/hama/bsp/DispatchTasksDirective.java
===================================================================
--- src/java/org/apache/hama/bsp/DispatchTasksDirective.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/DispatchTasksDirective.java (working copy)
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * Handles the tasks dispatching between the BSPMaster and the GroomServers.
- */
-public final class DispatchTasksDirective extends Directive implements Writable {
-
- public static final Log LOG = LogFactory.getLog(DispatchTasksDirective.class);
-
- private Map groomServerPeers;
- private GroomServerAction[] actions;
-
- public DispatchTasksDirective() {
- super();
- }
-
- public DispatchTasksDirective(Map groomServerPeers,
- GroomServerAction[] actions) {
- super(Directive.Type.Request);
- this.groomServerPeers = groomServerPeers;
- this.actions = actions;
- }
-
- public Map getGroomServerPeers() {
- return this.groomServerPeers;
- }
-
- public GroomServerAction[] getActions() {
- return this.actions;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- if (this.actions == null) {
- WritableUtils.writeVInt(out, 0);
- } else {
- WritableUtils.writeVInt(out, actions.length);
- for (GroomServerAction action : this.actions) {
- WritableUtils.writeEnum(out, action.getActionType());
- action.write(out);
- }
- }
- String[] groomServerNames = groomServerPeers.keySet()
- .toArray(new String[0]);
- WritableUtils.writeCompressedStringArray(out, groomServerNames);
-
- List groomServerAddresses = new ArrayList(
- groomServerNames.length);
- for (String groomName : groomServerNames) {
- groomServerAddresses.add(groomServerPeers.get(groomName));
- }
- WritableUtils.writeCompressedStringArray(out, groomServerAddresses
- .toArray(new String[0]));
-
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- int length = WritableUtils.readVInt(in);
- if (length > 0) {
- this.actions = new GroomServerAction[length];
- for (int i = 0; i < length; ++i) {
- GroomServerAction.ActionType actionType = WritableUtils.readEnum(in,
- GroomServerAction.ActionType.class);
- actions[i] = GroomServerAction.createAction(actionType);
- actions[i].readFields(in);
- }
- } else {
- this.actions = null;
- }
- String[] groomServerNames = WritableUtils.readCompressedStringArray(in);
- String[] groomServerAddresses = WritableUtils.readCompressedStringArray(in);
- groomServerPeers = new HashMap(groomServerNames.length);
-
- for (int i = 0; i < groomServerNames.length; i++) {
- groomServerPeers.put(groomServerNames[i], groomServerAddresses[i]);
- }
- }
-}
Index: src/java/org/apache/hama/bsp/GroomServerManager.java
===================================================================
--- src/java/org/apache/hama/bsp/GroomServerManager.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/GroomServerManager.java (working copy)
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.util.Collection;
-import java.util.Map;
-
-import org.apache.hama.ipc.GroomProtocol;
-
-/**
- * Manages information about the {@link GroomServer}s in the cluster
- * environment. This interface is not intended to be implemented by users.
- */
-interface GroomServerManager {
-
- /**
- * Get the current status of the cluster
- * @param detailed if true then report groom names as well
- * @return summary of the state of the cluster
- */
- ClusterStatus getClusterStatus(boolean detailed);
-
- /**
- * Find WorkerProtocol with corresponded groom server status
- *
- * @param groomId The identification value of GroomServer
- * @return GroomServerStatus
- */
- GroomProtocol findGroomServer(GroomServerStatus status);
-
- /**
- * Find the collection of groom servers.
- *
- * @return Collection of groom servers list.
- */
- Collection findGroomServers();
-
- /**
- * Collection of GroomServerStatus as the key set.
- *
- * @return Collection of GroomServerStatus.
- */
- Collection groomServerStatusKeySet();
-
- /**
- * Registers a JobInProgressListener to GroomServerManager. Therefore,
- * adding a JobInProgress will trigger the jobAdded function.
- * @param the JobInProgressListener listener to be added.
- */
- void addJobInProgressListener(JobInProgressListener listener);
-
- /**
- * Unregisters a JobInProgressListener to GroomServerManager. Therefore,
- * the remove of a JobInProgress will trigger the jobRemoved action.
- * @param the JobInProgressListener to be removed.
- */
- void removeJobInProgressListener(JobInProgressListener listener);
-
- /**
- * Current GroomServer Peers.
- * @return GroomName and PeerName(host:port) in pair.
- */
- Map currentGroomServerPeers();
-}
Index: src/java/org/apache/hama/bsp/GroomServerAction.java
===================================================================
--- src/java/org/apache/hama/bsp/GroomServerAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/GroomServerAction.java (working copy)
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * A generic directive from the {@link org.apache.hama.bsp.BSPMaster}
- * to the {@link org.apache.hama.bsp.GroomServer} to take some 'action'.
- */
-abstract class GroomServerAction implements Writable {
-
- /**
- * Ennumeration of various 'actions' that the {@link BSPMaster}
- * directs the {@link GroomServer} to perform periodically.
- *
- */
- public static enum ActionType {
- /** Launch a new task. */
- LAUNCH_TASK,
-
- /** Kill a task. */
- KILL_TASK,
-
- /** Kill any tasks of this job and cleanup. */
- KILL_JOB,
-
- /** Reinitialize the groomserver. */
- REINIT_GROOM,
-
- /** Ask a task to save its output. */
- COMMIT_TASK
- };
-
- /**
- * A factory-method to create objects of given {@link ActionType}.
- * @param actionType the {@link ActionType} of object to create.
- * @return an object of {@link ActionType}.
- */
- public static GroomServerAction createAction(ActionType actionType) {
- GroomServerAction action = null;
-
- switch (actionType) {
- case LAUNCH_TASK:
- {
- action = new LaunchTaskAction();
- }
- break;
- case KILL_TASK:
- {
- action = new KillTaskAction();
- }
- break;
- case KILL_JOB:
- {
- action = new KillJobAction();
- }
- break;
- case REINIT_GROOM:
- {
- action = new ReinitGroomAction();
- }
- break;
- case COMMIT_TASK:
- {
- action = new CommitTaskAction();
- }
- break;
- }
-
- return action;
- }
-
- private ActionType actionType;
-
- protected GroomServerAction(ActionType actionType) {
- this.actionType = actionType;
- }
-
- /**
- * Return the {@link ActionType}.
- * @return the {@link ActionType}.
- */
- ActionType getActionType() {
- return actionType;
- }
-
- public void write(DataOutput out) throws IOException {
- WritableUtils.writeEnum(out, actionType);
- }
-
- public void readFields(DataInput in) throws IOException {
- actionType = WritableUtils.readEnum(in, ActionType.class);
- }
-}
Index: src/java/org/apache/hama/bsp/ByteMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/ByteMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/ByteMessage.java (working copy)
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * A message that consists of a byte tag and a byte data.
- */
-public class ByteMessage extends BSPMessage {
-
- private byte[] tag;
- private byte[] data;
-
- public ByteMessage() {
- super();
- }
-
- public ByteMessage(byte[] tag, byte[] data) {
- super();
- this.tag = tag;
- this.data = data;
- }
-
- @Override
- public byte[] getTag() {
- return this.tag;
- }
-
- @Override
- public byte[] getData() {
- return this.data;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- this.tag = new byte[in.readInt()];
- in.readFully(tag, 0, this.tag.length);
- this.data = new byte[in.readInt()];
- in.readFully(data, 0, this.data.length);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(tag.length);
- out.write(tag);
- out.writeInt(data.length);
- out.write(data);
- }
-
-}
Index: src/java/org/apache/hama/bsp/CommitTaskAction.java
===================================================================
--- src/java/org/apache/hama/bsp/CommitTaskAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/CommitTaskAction.java (working copy)
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster}
- * to the {@link org.apache.hama.bsp.GroomServer} to commit the output
- * of the task.
- */
-class CommitTaskAction extends GroomServerAction {
- private TaskAttemptID taskId;
-
- public CommitTaskAction() {
- super(ActionType.COMMIT_TASK);
- taskId = new TaskAttemptID();
- }
-
- public CommitTaskAction(TaskAttemptID taskId) {
- super(ActionType.COMMIT_TASK);
- this.taskId = taskId;
- }
-
- public TaskAttemptID getTaskID() {
- return taskId;
- }
-
- public void write(DataOutput out) throws IOException {
- taskId.write(out);
- }
-
- public void readFields(DataInput in) throws IOException {
- taskId.readFields(in);
- }
-}
Index: src/java/org/apache/hama/bsp/RunningJob.java
===================================================================
--- src/java/org/apache/hama/bsp/RunningJob.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/RunningJob.java (working copy)
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-/**
- * RunningJob is the user-interface to query for details on a
- * running BSP job.
- *
- *
- * Clients can get hold of RunningJob via the {@link BSPJobClient}
- * and then query the running-job for details such as name, configuration,
- * progress etc.
- *
- *
- * @see BSPJobClient
- */
-public interface RunningJob {
- /**
- * Get the job identifier.
- *
- * @return the job identifier.
- */
- public BSPJobID getID();
-
- /**
- * Get the name of the job.
- *
- * @return the name of the job.
- */
- public String getJobName();
-
- /**
- * Get the path of the submitted job configuration.
- *
- * @return the path of the submitted job configuration.
- */
- public String getJobFile();
-
- /**
- * Get the progress of the job's tasks, as a float between 0.0 and 1.0.
- * When all bsp tasks have completed, the function returns 1.0.
- *
- * @return the progress of the job's tasks.
- * @throws IOException
- */
- public long progress() throws IOException;
-
- /**
- * Check if the job is finished or not. This is a non-blocking call.
- *
- * @return true if the job is complete, else false.
- * @throws IOException
- */
- public boolean isComplete() throws IOException;
-
- /**
- * Check if the job completed successfully.
- *
- * @return true if the job succeeded, else false.
- * @throws IOException
- */
- public boolean isSuccessful() throws IOException;
-
- /**
- * Blocks until the job is complete.
- *
- * @throws IOException
- */
- public void waitForCompletion() throws IOException;
-
- /**
- * Returns the current state of the Job. {@link JobStatus}
- *
- * @throws IOException
- */
- public int getJobState() throws IOException;
-
- /**
- * Kill the running job. Blocks until all job tasks have been killed as well.
- * If the job is no longer running, it simply returns.
- *
- * @throws IOException
- */
- public void killJob() throws IOException;
-
- /**
- * Kill indicated task attempt.
- *
- * @param taskId the id of the task to be terminated.
- * @param shouldFail if true the task is failed and added to failed tasks
- * list, otherwise it is just killed, w/o affecting job failure
- * status.
- * @throws IOException
- */
- public void killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException;
-
- public long getSuperstepCount() throws IOException;
-}
Index: src/java/org/apache/hama/bsp/Schedulable.java
===================================================================
--- src/java/org/apache/hama/bsp/Schedulable.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/Schedulable.java (working copy)
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-/**
- * This is the class that schedules commands to GroomServer(s)
- */
-public interface Schedulable{
-
- /**
- * Schedule job to designated GroomServer(s) immediately.
- * @param job to be scheduled.
- * @param statuses of GroomServer(s).
- * @throws IOException
- */
- void schedule(JobInProgress job, GroomServerStatus... statuses)
- throws IOException;
-}
Index: src/java/org/apache/hama/bsp/SimpleTaskScheduler.java
===================================================================
--- src/java/org/apache/hama/bsp/SimpleTaskScheduler.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/SimpleTaskScheduler.java (working copy)
@@ -1,183 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hama.ipc.GroomProtocol;
-
-/**
- * A simple task scheduler.
- */
-class SimpleTaskScheduler extends TaskScheduler {
-
- private static final Log LOG = LogFactory.getLog(SimpleTaskScheduler.class);
-
- public static final String WAIT_QUEUE = "waitQueue";
- public static final String PROCESSING_QUEUE = "processingQueue";
- public static final String FINISHED_QUEUE = "finishedQueue";
-
- private QueueManager queueManager;
- private volatile boolean initialized;
- private JobListener jobListener;
- private JobProcessor jobProcessor;
-
- private class JobListener extends JobInProgressListener {
- @Override
- public void jobAdded(JobInProgress job) throws IOException {
- queueManager.initJob(job); // init task
- queueManager.addJob(WAIT_QUEUE, job);
- }
-
- @Override
- public void jobRemoved(JobInProgress job) throws IOException {
- // queueManager.removeJob(WAIT_QUEUE, job);
- queueManager.moveJob(PROCESSING_QUEUE, FINISHED_QUEUE, job);
- }
- }
-
- private class JobProcessor extends Thread implements Schedulable {
- JobProcessor() {
- super("JobProcess");
- }
-
- /**
- * Main logic scheduling task to GroomServer(s). Also, it will move
- * JobInProgress from Wait Queue to Processing Queue.
- */
- public void run() {
- if (false == initialized) {
- throw new IllegalStateException("SimpleTaskScheduler initialization"
- + " is not yet finished!");
- }
- while (initialized) {
- Queue queue = queueManager.findQueue(WAIT_QUEUE);
- if (null == queue) {
- LOG.error(WAIT_QUEUE + " does not exist.");
- throw new NullPointerException(WAIT_QUEUE + " does not exist.");
- }
- // move a job from the wait queue to the processing queue
- JobInProgress j = queue.removeJob();
- queueManager.addJob(PROCESSING_QUEUE, j);
- // schedule
- Collection glist = groomServerManager
- .groomServerStatusKeySet();
- schedule(j, (GroomServerStatus[]) glist
- .toArray(new GroomServerStatus[glist.size()]));
- }
- }
-
- /**
- * Schedule job to designated GroomServer(s) immediately.
- *
- * @param Targeted GroomServer(s).
- * @param Job to be scheduled.
- */
- @Override
- public void schedule(JobInProgress job, GroomServerStatus... statuses) {
- ClusterStatus clusterStatus = groomServerManager.getClusterStatus(false);
- final int numGroomServers = clusterStatus.getGroomServers();
- final ScheduledExecutorService sched = Executors
- .newScheduledThreadPool(statuses.length + 5);
- for (GroomServerStatus status : statuses) {
- sched
- .schedule(new TaskWorker(status, numGroomServers, job), 0, SECONDS);
- }// for
- }
- }
-
- private class TaskWorker implements Runnable {
- private final GroomServerStatus stus;
- private final int groomNum;
- private final JobInProgress jip;
-
- TaskWorker(final GroomServerStatus stus, final int num,
- final JobInProgress jip) {
- this.stus = stus;
- this.groomNum = num;
- this.jip = jip;
- if (null == this.stus)
- throw new NullPointerException("Target groom server is not "
- + "specified.");
- if (-1 == this.groomNum)
- throw new IllegalArgumentException("Groom number is not specified.");
- if (null == this.jip)
- throw new NullPointerException("No job is specified.");
- }
-
- public void run() {
- // obtain tasks
- Task t = jip.obtainNewTask(this.stus, groomNum);
-
- // assembly into actions
- // List tasks = new ArrayList();
- if (jip.getStatus().getRunState() == JobStatus.RUNNING) {
- GroomProtocol worker = groomServerManager.findGroomServer(this.stus);
- try {
- // dispatch() to the groom server
- Directive d1 = new DispatchTasksDirective(groomServerManager
- .currentGroomServerPeers(), new GroomServerAction[] {
- new LaunchTaskAction(t)});
- worker.dispatch(d1);
- } catch (IOException ioe) {
- LOG.error("Fail to dispatch tasks to GroomServer "
- + this.stus.getGroomName(), ioe);
- }
- } else {
- LOG.warn("Currently master only shcedules job in running state. "
- + "This may be refined in the future. JobId:" + jip.getJobID());
- }
- }
- }
-
- public SimpleTaskScheduler() {
- this.jobListener = new JobListener();
- this.jobProcessor = new JobProcessor();
- }
-
- @Override
- public void start() {
- this.queueManager = new QueueManager(getConf()); // TODO: need factory?
- this.queueManager.createFCFSQueue(WAIT_QUEUE);
- this.queueManager.createFCFSQueue(PROCESSING_QUEUE);
- this.queueManager.createFCFSQueue(FINISHED_QUEUE);
- groomServerManager.addJobInProgressListener(this.jobListener);
- this.initialized = true;
- this.jobProcessor.start();
- }
-
- @Override
- public void terminate() {
- this.initialized = false;
- if (null != this.jobListener)
- groomServerManager.removeJobInProgressListener(this.jobListener);
- }
-
- @Override
- public Collection getJobs(String queue) {
- return (queueManager.findQueue(queue)).jobs();
- // return jobQueue;
- }
-}
Index: src/java/org/apache/hama/bsp/DirectiveHandler.java
===================================================================
--- src/java/org/apache/hama/bsp/DirectiveHandler.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/DirectiveHandler.java (working copy)
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * A DirectiveHandler interface.
- */
-public interface DirectiveHandler{
-
- /**
- * Handle directives on demand.
- * @param directive to be handled.
- */
- void handle(Directive directive) throws DirectiveException;
-}
Index: src/java/org/apache/hama/bsp/KillTaskAction.java
===================================================================
--- src/java/org/apache/hama/bsp/KillTaskAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/KillTaskAction.java (working copy)
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster}
- * to the {@link org.apache.hama.bsp.GroomServer} to kill a task.
- */
-class KillTaskAction extends GroomServerAction {
- TaskAttemptID taskId;
-
- public KillTaskAction() {
- super(ActionType.KILL_TASK);
- taskId = new TaskAttemptID();
- }
-
- public KillTaskAction(TaskAttemptID killTaskId) {
- super(ActionType.KILL_TASK);
- this.taskId = killTaskId;
- }
-
- public TaskAttemptID getTaskID() {
- return taskId;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- taskId.write(out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- taskId.readFields(in);
- }
-}
Index: src/java/org/apache/hama/bsp/GroomServerStatus.java
===================================================================
--- src/java/org/apache/hama/bsp/GroomServerStatus.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/GroomServerStatus.java (working copy)
@@ -1,227 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-
-/**
- * A GroomServerStatus is a BSP primitive. Keeps info on a BSPMaster. The
- * BSPMaster maintains a set of the most recent GroomServerStatus objects for
- * each unique GroomServer it knows about.
- */
-public class GroomServerStatus implements Writable {
- public static final Log LOG = LogFactory.getLog(GroomServerStatus.class);
-
- static {
- WritableFactories.setFactory(GroomServerStatus.class,
- new WritableFactory() {
- public Writable newInstance() {
- return new GroomServerStatus();
- }
- });
- }
-
- String groomName;
- String peerName;
- String rpcServer;
- int failures;
- List taskReports;
-
- volatile long lastSeen;
- private int maxTasks;
-
- public GroomServerStatus() {
- // taskReports = new ArrayList();
- taskReports = new CopyOnWriteArrayList();
- }
-
- public GroomServerStatus(String groomName, String peerName,
- List taskReports, int failures, int maxTasks) {
- this(groomName, peerName, taskReports, failures, maxTasks, "");
- }
-
- public GroomServerStatus(String groomName, String peerName,
- List taskReports, int failures, int maxTasks, String rpc) {
- this.groomName = groomName;
- this.peerName = peerName;
- this.taskReports = new ArrayList(taskReports);
- this.failures = failures;
- this.maxTasks = maxTasks;
- this.rpcServer = rpc;
- }
-
- public String getGroomName() {
- return groomName;
- }
-
- /**
- * The host (and port) from where the groom server can be reached.
- *
- * @return The groom server address in the form of "hostname:port"
- */
- public String getPeerName() {
- return peerName;
- }
-
- public String getRpcServer() {
- return rpcServer;
- }
-
- /**
- * Get the current tasks at the GroomServer. Tasks are tracked by a
- * {@link TaskStatus} object.
- *
- * @return a list of {@link TaskStatus} representing the current tasks at the
- * GroomServer.
- */
- public List getTaskReports() {
- return taskReports;
- }
-
- public int getFailures() {
- return failures;
- }
-
- public long getLastSeen() {
- return lastSeen;
- }
-
- public void setLastSeen(long lastSeen) {
- this.lastSeen = lastSeen;
- }
-
- public int getMaxTasks() {
- return maxTasks;
- }
-
- /**
- * Return the current MapTask count
- */
- public int countTasks() {
- int taskCount = 0;
- for (Iterator it = taskReports.iterator(); it.hasNext();) {
- TaskStatus ts = it.next();
- TaskStatus.State state = ts.getRunState();
- if (state == TaskStatus.State.RUNNING
- || state == TaskStatus.State.UNASSIGNED) {
- taskCount++;
- }
- }
-
- return taskCount;
- }
-
- /**
- * For BSPMaster to distinguish between different GroomServers, because
- * BSPMaster stores using GroomServerStatus as key.
- */
- @Override
- public int hashCode() {
- int result = 17;
- result = 37 * result + groomName.hashCode();
- result = 37 * result + peerName.hashCode();
- result = 37 * result + rpcServer.hashCode();
- /*
- * result = 37*result + (int)failures; result = 37*result +
- * taskReports.hashCode(); result = 37*result +
- * (int)(lastSeen^(lastSeen>>>32)); result = 37*result + (int)maxTasks;
- */
- return result;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this)
- return true;
- if (null == o)
- return false;
- if (getClass() != o.getClass())
- return false;
-
- GroomServerStatus s = (GroomServerStatus) o;
- if (!s.groomName.equals(groomName))
- return false;
- if (!s.peerName.equals(peerName))
- return false;
- if (!s.rpcServer.equals(rpcServer))
- return false;
- /*
- * if(s.failures != failures) return false; if(null == s.taskReports){
- * if(null != s.taskReports) return false; }else
- * if(!s.taskReports.equals(taskReports)){ return false; } if(s.lastSeen !=
- * lastSeen) return false; if(s.maxTasks != maxTasks) return false;
- */
- return true;
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- this.groomName = Text.readString(in);
- this.peerName = Text.readString(in);
- this.rpcServer = Text.readString(in);
- this.failures = in.readInt();
- this.maxTasks = in.readInt();
- taskReports.clear();
- int numTasks = in.readInt();
-
- TaskStatus status;
- for (int i = 0; i < numTasks; i++) {
- status = new TaskStatus();
- status.readFields(in);
- taskReports.add(status);
- }
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, groomName);
- Text.writeString(out, peerName);
- Text.writeString(out, rpcServer);
- out.writeInt(failures);
- out.writeInt(maxTasks);
- out.writeInt(taskReports.size());
- for (TaskStatus taskStatus : taskReports) {
- taskStatus.write(out);
- }
- }
-
- public Iterator taskReports() {
- return taskReports.iterator();
- }
-}
Index: src/java/org/apache/hama/bsp/BSPJobClient.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPJobClient.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPJobClient.java (working copy)
@@ -1,679 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Random;
-
-import javax.security.auth.login.LoginException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UnixUserGroupInformation;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.ipc.JobSubmissionProtocol;
-
-/**
- * BSPJobClient is the primary interface for the user-job to interact with the
- * BSPMaster.
- *
- * BSPJobClient provides facilities to submit jobs, track their progress, access
- * component-tasks' reports/logs, get the BSP cluster status information etc.
- */
-public class BSPJobClient extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(BSPJobClient.class);
-
- public static enum TaskStatusFilter {
- NONE, KILLED, FAILED, SUCCEEDED, ALL
- }
-
- private static final long MAX_JOBPROFILE_AGE = 1000 * 2;
-
- class NetworkedJob implements RunningJob {
- JobProfile profile;
- JobStatus status;
- long statustime;
-
- public NetworkedJob(JobStatus job) throws IOException {
- this.status = job;
- this.profile = jobSubmitClient.getJobProfile(job.getJobID());
- this.statustime = System.currentTimeMillis();
- }
-
- /**
- * Some methods rely on having a recent job profile object. Refresh it, if
- * necessary
- */
- synchronized void ensureFreshStatus() throws IOException {
- if (System.currentTimeMillis() - statustime > MAX_JOBPROFILE_AGE) {
- updateStatus();
- }
- }
-
- /**
- * Some methods need to update status immediately. So, refresh immediately
- *
- * @throws IOException
- */
- synchronized void updateStatus() throws IOException {
- this.status = jobSubmitClient.getJobStatus(profile.getJobID());
- this.statustime = System.currentTimeMillis();
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.RunningJob#getID()
- */
- @Override
- public BSPJobID getID() {
- return profile.getJobID();
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.RunningJob#getJobName()
- */
- @Override
- public String getJobName() {
- return profile.getJobName();
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.RunningJob#getJobFile()
- */
- @Override
- public String getJobFile() {
- return profile.getJobFile();
- }
-
- @Override
- public long progress() throws IOException {
- ensureFreshStatus();
- return status.progress();
- }
-
- /**
- * Returns immediately whether the whole job is done yet or not.
- */
- public synchronized boolean isComplete() throws IOException {
- updateStatus();
- return (status.getRunState() == JobStatus.SUCCEEDED
- || status.getRunState() == JobStatus.FAILED || status.getRunState() == JobStatus.KILLED);
- }
-
- /**
- * True if job completed successfully.
- */
- public synchronized boolean isSuccessful() throws IOException {
- updateStatus();
- return status.getRunState() == JobStatus.SUCCEEDED;
- }
-
- public synchronized long getSuperstepCount() throws IOException {
- ensureFreshStatus();
- return status.getSuperstepCount();
- }
-
- /**
- * Blocks until the job is finished
- */
- public void waitForCompletion() throws IOException {
- while (!isComplete()) {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException ie) {
- }
- }
- }
-
- /**
- * Tells the service to get the state of the current job.
- */
- public synchronized int getJobState() throws IOException {
- updateStatus();
- return status.getRunState();
- }
-
- /**
- * Tells the service to terminate the current job.
- */
- public synchronized void killJob() throws IOException {
- jobSubmitClient.killJob(getID());
- }
-
- @Override
- public void killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException {
- jobSubmitClient.killTask(taskId, shouldFail);
- }
- }
-
- private JobSubmissionProtocol jobSubmitClient = null;
- private Path sysDir = null;
- private FileSystem fs = null;
-
- // job files are world-wide readable and owner writable
- final private static FsPermission JOB_FILE_PERMISSION = FsPermission
- .createImmutable((short) 0644); // rw-r--r--
-
- // job submission directory is world readable/writable/executable
- final static FsPermission JOB_DIR_PERMISSION = FsPermission
- .createImmutable((short) 0777); // rwx-rwx-rwx
-
- public BSPJobClient(Configuration conf) throws IOException {
- setConf(conf);
- init(conf);
- }
-
- public BSPJobClient() {
- }
-
- public void init(Configuration conf) throws IOException {
- String masterAdress = conf.get("bsp.master.address");
- if (masterAdress != null && !masterAdress.equals("local")) {
- this.jobSubmitClient = (JobSubmissionProtocol) RPC.getProxy(
- JobSubmissionProtocol.class, JobSubmissionProtocol.versionID,
- BSPMaster.getAddress(conf), conf,
- NetUtils.getSocketFactory(conf, JobSubmissionProtocol.class));
- } else {
- LOG.debug("Using local BSP runner.");
- this.jobSubmitClient = new LocalBSPRunner(conf);
- }
- }
-
- /**
- * Close the JobClient.
- */
- public synchronized void close() throws IOException {
- RPC.stopProxy(jobSubmitClient);
- }
-
- /**
- * Get a filesystem handle. We need this to prepare jobs for submission to the
- * BSP system.
- *
- * @return the filesystem handle.
- */
- public synchronized FileSystem getFs() throws IOException {
- if (this.fs == null) {
- Path sysDir = getSystemDir();
- this.fs = sysDir.getFileSystem(getConf());
- }
- return fs;
- }
-
- /**
- * Gets the jobs that are submitted.
- *
- * @return array of {@link JobStatus} for the submitted jobs.
- * @throws IOException
- */
- public JobStatus[] getAllJobs() throws IOException {
- return jobSubmitClient.getAllJobs();
- }
-
- /**
- * Gets the jobs that are not completed and not failed.
- *
- * @return array of {@link JobStatus} for the running/to-be-run jobs.
- * @throws IOException
- */
- public JobStatus[] jobsToComplete() throws IOException {
- return jobSubmitClient.jobsToComplete();
- }
-
- private UnixUserGroupInformation getUGI(Configuration conf)
- throws IOException {
- UnixUserGroupInformation ugi = null;
- try {
- ugi = UnixUserGroupInformation.login(conf, true);
- } catch (LoginException e) {
- throw (IOException) (new IOException(
- "Failed to get the current user's information.").initCause(e));
- }
- return ugi;
- }
-
- /**
- * Submit a job to the BSP system. This returns a handle to the
- * {@link RunningJob} which can be used to track the running-job.
- *
- * @param job the job configuration.
- * @return a handle to the {@link RunningJob} which can be used to track the
- * running-job.
- * @throws FileNotFoundException
- * @throws IOException
- */
- public RunningJob submitJob(BSPJob job) throws FileNotFoundException,
- IOException {
- return submitJobInternal(job);
- }
-
- static Random r = new Random();
-
- public RunningJob submitJobInternal(BSPJob job) throws IOException {
- BSPJobID jobId = jobSubmitClient.getNewJobId();
-
- Path submitJobDir = new Path(getSystemDir(), "submit_"
- + Integer.toString(Math.abs(r.nextInt()), 36));
- Path submitJarFile = new Path(submitJobDir, "job.jar");
- Path submitJobFile = new Path(submitJobDir, "job.xml");
- LOG.debug("BSPJobClient.submitJobDir: " + submitJobDir);
-
- /*
- * set this user's id in job configuration, so later job files can be
- * accessed using this user's id
- */
- UnixUserGroupInformation ugi = getUGI(job.getConf());
-
- ClusterStatus clusterStatus = getClusterStatus(true);
-
- // check the number of BSP tasks
- int tasks = job.getNumBspTask();
- int maxTasks = clusterStatus.getMaxTasks();
-
- if (tasks <= 0 || tasks > maxTasks) {
- LOG.warn("The number of tasks you've entered was invalid. Using default value of "
- + maxTasks + "!");
- job.setNumBspTask(maxTasks);
- }
-
- // Create a number of filenames in the BSPMaster's fs namespace
- FileSystem fs = getFs();
- fs.delete(submitJobDir, true);
- submitJobDir = fs.makeQualified(submitJobDir);
- submitJobDir = new Path(submitJobDir.toUri().getPath());
- FsPermission bspSysPerms = new FsPermission(JOB_DIR_PERMISSION);
- FileSystem.mkdirs(fs, submitJobDir, bspSysPerms);
- fs.mkdirs(submitJobDir);
- short replication = (short) job.getInt("bsp.submit.replication", 10);
-
- String originalJarPath = job.getJar();
-
- if (originalJarPath != null) { // copy jar to BSPMaster's fs
- // use jar name if job is not named.
- if ("".equals(job.getJobName())) {
- job.setJobName(new Path(originalJarPath).getName());
- }
- job.setJar(submitJarFile.toString());
- fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile);
-
- fs.setReplication(submitJarFile, replication);
- fs.setPermission(submitJarFile, new FsPermission(JOB_FILE_PERMISSION));
- } else {
- LOG.warn("No job jar file set. User classes may not be found. "
- + "See BSPJob#setJar(String) or check Your jar file.");
- }
-
- // Set the user's name and working directory
- job.setUser(ugi.getUserName());
- if (ugi.getGroupNames().length > 0) {
- job.set("group.name", ugi.getGroupNames()[0]);
- }
- if (job.getWorkingDirectory() == null) {
- job.setWorkingDirectory(fs.getWorkingDirectory());
- }
-
- // Write job file to BSPMaster's fs
- FSDataOutputStream out = FileSystem.create(fs, submitJobFile,
- new FsPermission(JOB_FILE_PERMISSION));
-
- try {
- job.writeXml(out);
- } finally {
- out.close();
- }
-
- //
- // Now, actually submit the job (using the submit name)
- //
- JobStatus status = jobSubmitClient.submitJob(jobId,
- submitJobFile.toString());
- if (status != null) {
- return new NetworkedJob(status);
- } else {
- throw new IOException("Could not launch job");
- }
- }
-
- /**
- * Monitor a job and print status in real-time as progress is made and tasks
- * fail.
- *
- * @param job
- * @param info
- * @return true, if job is successful
- * @throws IOException
- * @throws InterruptedException
- */
- public boolean monitorAndPrintJob(BSPJob job, RunningJob info)
- throws IOException, InterruptedException {
-
- String lastReport = null;
- LOG.info("Running job: " + info.getID());
-
- while (!job.isComplete()) {
- Thread.sleep(3000);
- long step = job.progress();
- String report = "Current supersteps number: " + step;
-
- if (!report.equals(lastReport)) {
- LOG.info(report);
- lastReport = report;
- }
- }
-
- LOG.info("The total number of supersteps: " + info.getSuperstepCount());
- return job.isSuccessful();
- }
-
- /**
- * Grab the bspmaster system directory path where job-specific files are to be
- * placed.
- *
- * @return the system directory where job-specific files are to be placed.
- */
- public Path getSystemDir() {
- if (sysDir == null) {
- sysDir = new Path(jobSubmitClient.getSystemDir());
- }
- return sysDir;
- }
-
- public static void runJob(BSPJob job) throws FileNotFoundException,
- IOException {
- BSPJobClient jc = new BSPJobClient(job.getConf());
-
- if (job.getNumBspTask() == 0
- || job.getNumBspTask() > jc.getClusterStatus(false).getMaxTasks()) {
- job.setNumBspTask(jc.getClusterStatus(false).getMaxTasks());
- }
-
- RunningJob running = jc.submitJobInternal(job);
- BSPJobID jobId = running.getID();
- LOG.info("Running job: " + jobId.toString());
-
- while (true) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
-
- if (running.isComplete()) {
- break;
- }
-
- running = jc.getJob(jobId);
- }
-
- LOG.info("Job complete: " + jobId);
- LOG.info("The total number of supersteps: " + running.getSuperstepCount());
-
- // TODO if error found, kill job
- // running.killJob();
- jc.close();
- }
-
- /**
- * Get an RunningJob object to track an ongoing job. Returns null if the id
- * does not correspond to any known job.
- *
- * @throws IOException
- */
- private RunningJob getJob(BSPJobID jobId) throws IOException {
- JobStatus status = jobSubmitClient.getJobStatus(jobId);
- if (status != null) {
- return new NetworkedJob(status);
- } else {
- return null;
- }
- }
-
- /**
- * Get status information about the BSP cluster
- *
- * @param detailed if true then get a detailed status including the
- * groomserver names
- *
- * @return the status information about the BSP cluster as an object of
- * {@link ClusterStatus}.
- *
- * @throws IOException
- */
- public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
- return jobSubmitClient.getClusterStatus(detailed);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- int exitCode = -1;
- if (args.length < 1) {
- displayUsage("");
- return exitCode;
- }
-
- // process arguments
- String cmd = args[0];
- boolean listJobs = false;
- boolean listAllJobs = false;
- boolean listActiveGrooms = false;
- boolean killJob = false;
- boolean submitJob = false;
- boolean getStatus = false;
- String submitJobFile = null;
- String jobid = null;
-
- HamaConfiguration conf = new HamaConfiguration(getConf());
- init(conf);
-
- if ("-list".equals(cmd)) {
- if (args.length != 1 && !(args.length == 2 && "all".equals(args[1]))) {
- displayUsage(cmd);
- return exitCode;
- }
- if (args.length == 2 && "all".equals(args[1])) {
- listAllJobs = true;
- } else {
- listJobs = true;
- }
- } else if ("-list-active-grooms".equals(cmd)) {
- if (args.length != 1) {
- displayUsage(cmd);
- return exitCode;
- }
- listActiveGrooms = true;
- } else if ("-submit".equals(cmd)) {
- if (args.length == 1) {
- displayUsage(cmd);
- return exitCode;
- }
-
- submitJob = true;
- submitJobFile = args[1];
- } else if ("-kill".equals(cmd)) {
- if (args.length == 1) {
- displayUsage(cmd);
- return exitCode;
- }
- killJob = true;
- jobid = args[1];
-
- } else if ("-status".equals(cmd)) {
- if (args.length != 2) {
- displayUsage(cmd);
- return exitCode;
- }
- jobid = args[1];
- getStatus = true;
-
- // TODO Later, below functions should be implemented
- // with the Fault Tolerant mechanism.
- } else if ("-list-attempt-ids".equals(cmd)) {
- System.out.println("This function is not implemented yet.");
- return exitCode;
- } else if ("-kill-task".equals(cmd)) {
- System.out.println("This function is not implemented yet.");
- return exitCode;
- } else if ("-fail-task".equals(cmd)) {
- System.out.println("This function is not implemented yet.");
- return exitCode;
- }
-
- BSPJobClient jc = new BSPJobClient(new HamaConfiguration());
- if (listJobs) {
- listJobs();
- exitCode = 0;
- } else if (listAllJobs) {
- listAllJobs();
- exitCode = 0;
- } else if (listActiveGrooms) {
- listActiveGrooms();
- exitCode = 0;
- } else if (submitJob) {
- HamaConfiguration tConf = new HamaConfiguration(new Path(submitJobFile));
- RunningJob job = jc.submitJob(new BSPJob(tConf));
- System.out.println("Created job " + job.getID().toString());
- } else if (killJob) {
- RunningJob job = jc.getJob(BSPJobID.forName(jobid));
- if (job == null) {
- System.out.println("Could not find job " + jobid);
- } else {
- job.killJob();
- System.out.println("Killed job " + jobid);
- }
- exitCode = 0;
- } else if (getStatus) {
- RunningJob job = jc.getJob(BSPJobID.forName(jobid));
- if (job == null) {
- System.out.println("Could not find job " + jobid);
- } else {
- JobStatus jobStatus = jobSubmitClient.getJobStatus(job.getID());
- System.out.println("Job name: " + job.getJobName());
- System.out.printf("States are:\n\tRunning : 1\tSucceded : 2"
- + "\tFailed : 3\tPrep : 4\n");
- System.out.printf("%s\t%d\t%d\t%s\n", jobStatus.getJobID(),
- jobStatus.getRunState(), jobStatus.getStartTime(),
- jobStatus.getUsername());
-
- exitCode = 0;
- }
- }
-
- return 0;
- }
-
- /**
- * Display usage of the command-line tool and terminate execution
- */
- private void displayUsage(String cmd) {
- String prefix = "Usage: hama job ";
- String taskStates = "running, completed";
- if ("-submit".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " ]");
- } else if ("-status".equals(cmd) || "-kill".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " ]");
- } else if ("-list".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " [all]]");
- } else if ("-kill-task".equals(cmd) || "-fail-task".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " ]");
- } else if ("-list-active-grooms".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + "]");
- } else if ("-list-attempt-ids".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " ]. "
- + "Valid values for are " + taskStates);
- } else {
- System.err.printf(prefix + "\n");
- System.err.printf("\t[-submit ]\n");
- System.err.printf("\t[-status ]\n");
- System.err.printf("\t[-kill ]\n");
- System.err.printf("\t[-list [all]]\n");
- System.err.printf("\t[-list-active-grooms]\n");
- System.err.println("\t[-list-attempt-ids " + "]\n");
- System.err.printf("\t[-kill-task ]\n");
- System.err.printf("\t[-fail-task ]\n\n");
- }
- }
-
- /**
- * Dump a list of currently running jobs
- *
- * @throws IOException
- */
- private void listJobs() throws IOException {
- JobStatus[] jobs = jobsToComplete();
- if (jobs == null)
- jobs = new JobStatus[0];
-
- System.out.printf("%d jobs currently running\n", jobs.length);
- displayJobList(jobs);
- }
-
- /**
- * Dump a list of all jobs submitted.
- *
- * @throws IOException
- */
- private void listAllJobs() throws IOException {
- JobStatus[] jobs = getAllJobs();
- if (jobs == null)
- jobs = new JobStatus[0];
- System.out.printf("%d jobs submitted\n", jobs.length);
- System.out.printf("States are:\n\tRunning : 1\tSucceded : 2"
- + "\tFailed : 3\tPrep : 4\n");
- displayJobList(jobs);
- }
-
- void displayJobList(JobStatus[] jobs) {
- System.out.printf("JobId\tState\tStartTime\tUserName\n");
- for (JobStatus job : jobs) {
- System.out.printf("%s\t%d\t%d\t%s\n", job.getJobID(), job.getRunState(),
- job.getStartTime(), job.getUsername());
- }
- }
-
- /**
- * Display the list of active groom servers
- */
- private void listActiveGrooms() throws IOException {
- ClusterStatus c = jobSubmitClient.getClusterStatus(true);
- Map grooms = c.getActiveGroomNames();
- for (String groomName : grooms.keySet()) {
- System.out.println(groomName);
- }
- }
-
- /**
- */
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new BSPJobClient(), args);
- System.exit(res);
- }
-}
Index: src/java/org/apache/hama/bsp/TaskAttemptID.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskAttemptID.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskAttemptID.java (working copy)
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * TaskAttemptID is a unique identifier for a task attempt.
- */
-public class TaskAttemptID extends ID {
- protected static final String ATTEMPT = "attempt";
- private TaskID taskId;
-
- public TaskAttemptID(TaskID taskId, int id) {
- super(id);
- if (taskId == null) {
- throw new IllegalArgumentException("taskId cannot be null");
- }
- this.taskId = taskId;
- }
-
- public TaskAttemptID(String jtIdentifier, int jobId, int taskId, int id) {
- this(new TaskID(jtIdentifier, jobId, taskId), id);
- }
-
- public TaskAttemptID() {
- taskId = new TaskID();
- }
-
- public BSPJobID getJobID() {
- return taskId.getJobID();
- }
-
- public TaskID getTaskID() {
- return taskId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- TaskAttemptID that = (TaskAttemptID) o;
- return this.taskId.equals(that.taskId);
- }
-
- protected StringBuilder appendTo(StringBuilder builder) {
- return taskId.appendTo(builder).append(SEPARATOR).append(id);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- taskId.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- taskId.write(out);
- }
-
- @Override
- public int hashCode() {
- return taskId.hashCode() * 5 + id;
- }
-
- @Override
- public int compareTo(ID o) {
- TaskAttemptID that = (TaskAttemptID) o;
- int tipComp = this.taskId.compareTo(that.taskId);
- if (tipComp == 0) {
- return this.id - that.id;
- } else
- return tipComp;
- }
-
- @Override
- public String toString() {
- return appendTo(new StringBuilder(ATTEMPT)).toString();
- }
-
- public static TaskAttemptID forName(String str)
- throws IllegalArgumentException {
- if (str == null)
- return null;
- try {
- String[] parts = str.split(Character.toString(SEPARATOR));
- if (parts.length == 5) {
- if (parts[0].equals(ATTEMPT)) {
- return new TaskAttemptID(parts[1], Integer.parseInt(parts[2]),
- Integer.parseInt(parts[3]), Integer.parseInt(parts[4]));
- }
- }
- } catch (Exception ex) {
- // fall below
- }
- throw new IllegalArgumentException("TaskAttemptId string : " + str
- + " is not properly formed");
- }
-}
Index: src/java/org/apache/hama/bsp/package.html
===================================================================
--- src/java/org/apache/hama/bsp/package.html (revision 1152138)
+++ src/java/org/apache/hama/bsp/package.html (working copy)
@@ -1,23 +0,0 @@
-
-
-
-
-
-BSP computing framework.
-
-
Index: src/java/org/apache/hama/bsp/BSPJob.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPJob.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPJob.java (working copy)
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.Enumeration;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hama.HamaConfiguration;
-
-/**
- * A BSP job configuration.
- *
- * BSPJob is the primary interface for a user to describe a BSP job to the Hama
- * BSP framework for execution.
- */
-public class BSPJob extends BSPJobContext {
- public static enum JobState {
- DEFINE, RUNNING
- };
-
- private JobState state = JobState.DEFINE;
- private BSPJobClient jobClient;
- private RunningJob info;
-
- public BSPJob() throws IOException {
- this(new HamaConfiguration());
- }
-
- public BSPJob(HamaConfiguration conf) throws IOException {
- super(conf, null);
- jobClient = new BSPJobClient(conf);
- }
-
- public BSPJob(HamaConfiguration conf, String jobName) throws IOException {
- this(conf);
- setJobName(jobName);
- }
-
- public BSPJob(BSPJobID jobID, String jobFile) throws IOException {
- super(new Path(jobFile), jobID);
- }
-
- public BSPJob(HamaConfiguration conf, Class> exampleClass)
- throws IOException {
- this(conf);
- setJarByClass(exampleClass);
- }
-
- public BSPJob(HamaConfiguration conf, int numPeer) {
- super(conf, null);
- this.setNumBspTask(numPeer);
- }
-
- private void ensureState(JobState state) throws IllegalStateException {
- if (state != this.state) {
- throw new IllegalStateException("Job in state " + this.state
- + " instead of " + state);
- }
- }
-
- // /////////////////////////////////////
- // Setter for Job Submission
- // /////////////////////////////////////
- public void setWorkingDirectory(Path dir) throws IOException {
- ensureState(JobState.DEFINE);
- dir = new Path(getWorkingDirectory(), dir);
- conf.set(WORKING_DIR, dir.toString());
- }
-
- /**
- * Set the BSP algorithm class for the job.
- *
- * @param cls
- * @throws IllegalStateException
- */
- public void setBspClass(Class extends BSP> cls)
- throws IllegalStateException {
- ensureState(JobState.DEFINE);
- conf.setClass(WORK_CLASS_ATTR, cls, BSP.class);
- }
-
- @SuppressWarnings("unchecked")
- public Class extends BSP> getBspClass() {
- return (Class extends BSP>) conf.getClass(WORK_CLASS_ATTR, BSP.class);
- }
-
- public void setJar(String jar) {
- conf.set("bsp.jar", jar);
- }
-
- public void setJarByClass(Class> cls) {
- String jar = findContainingJar(cls);
- if (jar != null) {
- conf.set("bsp.jar", jar);
- }
- }
-
- private static String findContainingJar(Class> my_class) {
- ClassLoader loader = my_class.getClassLoader();
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
- try {
- for (Enumeration itr = loader.getResources(class_file); itr
- .hasMoreElements();) {
-
- URL url = itr.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- return toReturn.replaceAll("!.*$", "");
- }
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return null;
- }
-
- public void setJobName(String name) throws IllegalStateException {
- ensureState(JobState.DEFINE);
- conf.set("bsp.job.name", name);
- }
-
- public void setInputPath(HamaConfiguration conf, Path iNPUTPATH) {
-
- }
-
- public void setUser(String user) {
- conf.set("user.name", user);
- }
-
- // /////////////////////////////////////
- // Methods for Job Control
- // /////////////////////////////////////
- public long progress() throws IOException {
- ensureState(JobState.RUNNING);
- return info.progress();
- }
-
- public boolean isComplete() throws IOException {
- ensureState(JobState.RUNNING);
- return info.isComplete();
- }
-
- public boolean isSuccessful() throws IOException {
- ensureState(JobState.RUNNING);
- return info.isSuccessful();
- }
-
- public void killJob() throws IOException {
- ensureState(JobState.RUNNING);
- info.killJob();
- }
-
- public void killTask(TaskAttemptID taskId) throws IOException {
- ensureState(JobState.RUNNING);
- info.killTask(taskId, false);
- }
-
- public void failTask(TaskAttemptID taskId) throws IOException {
- ensureState(JobState.RUNNING);
- info.killTask(taskId, true);
- }
-
- public void submit() throws IOException, InterruptedException {
- ensureState(JobState.DEFINE);
- info = jobClient.submitJobInternal(this);
- state = JobState.RUNNING;
- }
-
- public boolean waitForCompletion(boolean verbose) throws IOException,
- InterruptedException, ClassNotFoundException {
- if (state == JobState.DEFINE) {
- submit();
- }
- if (verbose) {
- jobClient.monitorAndPrintJob(this, info);
- } else {
- info.waitForCompletion();
- }
- return isSuccessful();
- }
-
- public void set(String name, String value) {
- conf.set(name, value);
- }
-
- public void setNumBspTask(int tasks) {
- conf.setInt("bsp.peers.num", tasks);
- }
-
- public int getNumBspTask() {
- return conf.getInt("bsp.peers.num", 0);
- }
-}
Index: src/java/org/apache/hama/bsp/PeerNames.java
===================================================================
--- src/java/org/apache/hama/bsp/PeerNames.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/PeerNames.java (working copy)
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-
-/**
- *
- */
-public class PeerNames implements Writable {
- Collection allPeers;
-
- public PeerNames() {
- this.allPeers = new ArrayList();
- }
-
- public PeerNames(Collection allPeers) {
- this.allPeers = allPeers;
- }
-
- public Collection getAllPeerNames() {
- return allPeers;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(allPeers.size());
- for (String peerName : allPeers) {
- Text.writeString(out, peerName);
- }
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- int peersNum = in.readInt();
- for (int i = 0; i < peersNum; i++) {
- allPeers.add(Text.readString(in));
- }
- }
-
-}
Index: src/java/org/apache/hama/bsp/TaskID.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskID.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskID.java (working copy)
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-/**
- * TaskID represents the immutable and unique identifier for a BSP Task.
- */
-public class TaskID extends ID {
- protected static final String TASK = "task";
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(6);
- }
-
- private BSPJobID jobId;
-
- public TaskID(BSPJobID jobId, int id) {
- super(id);
- if (jobId == null) {
- throw new IllegalArgumentException("jobId cannot be null");
- }
- this.jobId = jobId;
- }
-
- public TaskID(String jtIdentifier, int jobId, int id) {
- this(new BSPJobID(jtIdentifier, jobId), id);
- }
-
- public TaskID() {
- jobId = new BSPJobID();
- }
-
- /** Returns the {@link BSPJobID} object that this tip belongs to */
- public BSPJobID getJobID() {
- return jobId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- TaskID that = (TaskID) o;
- return this.jobId.equals(that.jobId);
- }
-
- @Override
- public int compareTo(ID o) {
- TaskID that = (TaskID) o;
- int jobComp = this.jobId.compareTo(that.jobId);
- if (jobComp == 0) {
- return this.id - that.id;
- } else {
- return jobComp;
- }
- }
-
- @Override
- public String toString() {
- return appendTo(new StringBuilder(TASK)).toString();
- }
-
- protected StringBuilder appendTo(StringBuilder builder) {
- return jobId.appendTo(builder).append(SEPARATOR)
- .append(idFormat.format(id));
- }
-
- @Override
- public int hashCode() {
- return jobId.hashCode() * 524287 + id;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- jobId.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- jobId.write(out);
- }
-
- public static TaskID forName(String str) throws IllegalArgumentException {
- if (str == null)
- return null;
- try {
- String[] parts = str.split("_");
- if (parts.length == 5) {
- if (parts[0].equals(TASK)) {
- return new TaskID(parts[1], Integer.parseInt(parts[2]), Integer
- .parseInt(parts[4]));
- }
- }
- } catch (Exception ex) {
- }
- throw new IllegalArgumentException("TaskId string : " + str
- + " is not properly formed");
- }
-}
Index: src/java/org/apache/hama/bsp/BSPPeerInterface.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPPeerInterface.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPPeerInterface.java (working copy)
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.hama.Constants;
-import org.apache.hama.ipc.HamaRPCProtocolVersion;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * BSP communication interface.
- */
-public interface BSPPeerInterface extends HamaRPCProtocolVersion, Closeable,
- Constants {
-
- /**
- * Send a data with a tag to another BSPSlave corresponding to hostname.
- * Messages sent by this method are not guaranteed to be received in a sent
- * order.
- *
- * @param peerName
- * @param msg
- * @throws IOException
- */
- public void send(String peerName, BSPMessage msg) throws IOException;
-
- /**
- * Puts a message to local queue.
- *
- * @param msg
- * @throws IOException
- */
- public void put(BSPMessage msg) throws IOException;
-
- /**
- * Puts a bundle of messages to local queue.
- *
- * @param messages
- * @throws IOException
- */
- public void put(BSPMessageBundle messages) throws IOException;
-
- /**
- * @return A message from the peer's received messages queue (a FIFO).
- * @throws IOException
- */
- public BSPMessage getCurrentMessage() throws IOException;
-
- /**
- * @return The number of messages in the peer's received messages queue.
- */
- public int getNumCurrentMessages();
-
- /**
- * Barrier Synchronization.
- *
- * Sends all the messages in the outgoing message queues to the corresponding
- * remote peers.
- *
- * @throws InterruptedException
- * @throws KeeperException
- */
- public void sync() throws IOException, KeeperException, InterruptedException;
-
- /**
- * @return the count of current super-step
- */
- public long getSuperstepCount();
-
- /**
- * @return The name of this peer in the format "hostname:port".
- */
- public String getPeerName();
-
- /**
- * @return The names of all the peers executing tasks from the same job
- * (including this peer).
- */
- public String[] getAllPeerNames();
-
- /**
- * Clears all queues entries.
- */
- public void clear();
-}
Index: src/java/org/apache/hama/bsp/JobChangeEvent.java
===================================================================
--- src/java/org/apache/hama/bsp/JobChangeEvent.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobChangeEvent.java (working copy)
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * {@link JobChangeEvent} is used to capture state changes in a job. A job can
- * change its state w.r.t priority, progress, run-state etc.
- */
-abstract class JobChangeEvent {
- private JobInProgress jip;
-
- JobChangeEvent(JobInProgress jip) {
- this.jip = jip;
- }
-
- /**
- * Get the job object for which the change is reported
- */
- JobInProgress getJobInProgress() {
- return jip;
- }
-}
Index: src/java/org/apache/hama/bsp/BSPMessageBundle.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPMessageBundle.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPMessageBundle.java (working copy)
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * BSPMessageBundle stores a group of BSPMessages so that they can be sent in
- * batch rather than individually.
- *
- */
-public class BSPMessageBundle implements Writable {
-
- public static final Log LOG = LogFactory.getLog(BSPMessageBundle.class);
-
- private HashMap> messages = new HashMap>();
- private HashMap> classCache = new HashMap>();
-
- public BSPMessageBundle() {
- }
-
- /**
- * Add message to this bundle.
- *
- * @param message BSPMessage to add.
- */
- public void addMessage(BSPMessage message) {
- String className = message.getClass().getName();
- if (!messages.containsKey(className)) {
- // use linked list because we're just iterating over them
- LinkedList list = new LinkedList();
- list.add(message);
- messages.put(className, list);
- } else {
- messages.get(className).add(message);
- }
- }
-
- public List getMessages() {
- // here we use an arraylist, because we know the size and outside may need
- // random access
- List mergeList = new ArrayList(messages.size());
- for (LinkedList c : messages.values()) {
- mergeList.addAll(c);
- }
- return mergeList;
- }
-
- public void write(DataOutput out) throws IOException {
- // writes the k/v mapping size
- out.writeInt(messages.size());
- if (messages.size() > 0) {
- for (Entry> entry : messages.entrySet()) {
- out.writeUTF(entry.getKey());
- LinkedList messageList = entry.getValue();
- out.writeInt(messageList.size());
- for (BSPMessage msg : messageList) {
- msg.write(out);
- }
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- public void readFields(DataInput in) throws IOException {
- if (messages == null) {
- messages = new HashMap>();
- }
- int numMessages = in.readInt();
- if (numMessages > 0) {
- for (int entries = 0; entries < numMessages; entries++) {
- String className = in.readUTF();
- int size = in.readInt();
- LinkedList msgList = new LinkedList();
- messages.put(className, msgList);
-
- Class extends BSPMessage> clazz = null;
- if ((clazz = classCache.get(className)) == null) {
- try {
- clazz = (Class extends BSPMessage>) Class.forName(className);
- classCache.put(className, clazz);
- } catch (ClassNotFoundException e) {
- LOG.error("Class was not found.",e);
- }
- }
-
- for (int i = 0; i < size; i++) {
- BSPMessage msg = ReflectionUtils.newInstance(clazz, null);
- msg.readFields(in);
- msgList.add(msg);
- }
-
- }
- }
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSP.java
===================================================================
--- src/java/org/apache/hama/bsp/BSP.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSP.java (working copy)
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * This class provides an abstract implementation of the BSP interface.
- */
-public abstract class BSP implements BSPInterface {
-}
Index: src/java/org/apache/hama/bsp/KillJobAction.java
===================================================================
--- src/java/org/apache/hama/bsp/KillJobAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/KillJobAction.java (working copy)
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the
- * {@link org.apache.hama.bsp.GroomServer} to kill the task of a job and cleanup
- * resources.
- */
-class KillJobAction extends GroomServerAction {
- String jobId;
-
- public KillJobAction() {
- super(ActionType.KILL_JOB);
- jobId = new String();
- }
-
- public KillJobAction(String killJobId) {
- super(ActionType.KILL_JOB);
- this.jobId = killJobId;
- }
-
- public String getJobID() {
- return jobId;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, jobId);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- jobId = Text.readString(in);
- }
-
-}
Index: src/java/org/apache/hama/bsp/Queue.java
===================================================================
--- src/java/org/apache/hama/bsp/Queue.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/Queue.java (working copy)
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.util.Collection;
-
-/**
- * Job Queue interface.
- *
- * @param
- */
-public interface Queue{
-
- /**
- * The queue name.
- * @return the name of current queue.
- */
- String getName();
-
- /**
- * Add a job to a queue.
- * @param job to be added to the queue.
- */
- void addJob(T job);
-
- /**
- * Remove a job from the queue.
- * @param job to be removed from the queue.
- */
- void removeJob(T job);
-
- /**
- * Get a job
- * @return job that is removed from the queue.
- */
- T removeJob();
-
- /**
- * Return all data stored in this queue.
- * @return Collection of jobs.
- */
- public Collection jobs();
-
-}
Index: src/java/org/apache/hama/bsp/LocalBSPRunner.java
===================================================================
--- src/java/org/apache/hama/bsp/LocalBSPRunner.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/LocalBSPRunner.java (working copy)
@@ -1,356 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadPoolExecutor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hama.bsp.BSPMaster.State;
-import org.apache.hama.ipc.JobSubmissionProtocol;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * A multithreaded local BSP runner that can be used for debugging BSP's. It
- * uses the working directory "/user/hama/bsp/" and starts runners based on the
- * number of the machines core.
- *
- */
-public class LocalBSPRunner implements JobSubmissionProtocol {
- public static final Log LOG = LogFactory.getLog(LocalBSPRunner.class);
-
- private static final String IDENTIFIER = "localrunner";
- private static String WORKING_DIR = "/user/hama/bsp/";
- protected static volatile ThreadPoolExecutor threadPool;
- protected static int threadPoolSize;
- protected static final LinkedList> futureList = new LinkedList>();
- protected static CyclicBarrier barrier;
-
- static {
- threadPoolSize = Runtime.getRuntime().availableProcessors();
- barrier = new CyclicBarrier(threadPoolSize);
- threadPool = (ThreadPoolExecutor) Executors
- .newFixedThreadPool(threadPoolSize);
- }
-
- protected HashMap localGrooms = new HashMap();
- protected String jobFile;
- protected String jobName;
-
- protected JobStatus currentJobStatus;
-
- protected Configuration conf;
- protected FileSystem fs;
-
- public LocalBSPRunner(Configuration conf) throws IOException {
- super();
- this.conf = conf;
- this.fs = FileSystem.get(conf);
- String path = conf.get("bsp.local.dir");
- if (path != null && !path.isEmpty())
- WORKING_DIR = path;
-
- threadPoolSize = conf.getInt("bsp.local.tasks.maximum", 20);
- threadPool = (ThreadPoolExecutor) Executors
- .newFixedThreadPool(threadPoolSize);
- barrier = new CyclicBarrier(threadPoolSize);
-
- for (int i = 0; i < threadPoolSize; i++) {
- String name = IDENTIFIER + " " + i;
- localGrooms.put(name, new LocalGroom(name));
- }
-
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return 3;
- }
-
- @Override
- public BSPJobID getNewJobId() throws IOException {
- return new BSPJobID(IDENTIFIER, 1);
- }
-
- @Override
- public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException {
- this.jobFile = jobFile;
- BSPJob job = new BSPJob(jobID, jobFile);
- job.setNumBspTask(threadPoolSize);
- this.jobName = job.getJobName();
- currentJobStatus = new JobStatus(jobID, System.getProperty("user.name"), 0,
- JobStatus.RUNNING);
- for (int i = 0; i < threadPoolSize; i++) {
- String name = IDENTIFIER + " " + i;
- LocalGroom localGroom = new LocalGroom(name);
- localGrooms.put(name, localGroom);
- futureList.add(threadPool.submit(new BSPRunner(conf, job, ReflectionUtils
- .newInstance(job.getBspClass(), conf), localGroom)));
- }
- new Thread(new ThreadObserver(currentJobStatus)).start();
- return currentJobStatus;
- }
-
- @Override
- public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
- Map map = new HashMap();
- for (Entry entry : localGrooms.entrySet()) {
- map.put(entry.getKey(), entry.getValue().getPeerName());
- }
- return new ClusterStatus(map, threadPoolSize, threadPoolSize, State.RUNNING);
- }
-
- @Override
- public JobProfile getJobProfile(BSPJobID jobid) throws IOException {
- return new JobProfile(System.getProperty("user.name"), jobid, jobFile,
- jobName);
- }
-
- @Override
- public JobStatus getJobStatus(BSPJobID jobid) throws IOException {
- if (currentJobStatus == null) {
- currentJobStatus = new JobStatus(jobid, System.getProperty("user.name"),
- 0L, JobStatus.RUNNING);
- }
- return currentJobStatus;
- }
-
- @Override
- public String getFilesystemName() throws IOException {
- return fs.getUri().toString();
- }
-
- @Override
- public JobStatus[] jobsToComplete() throws IOException {
- return null;
- }
-
- @Override
- public JobStatus[] getAllJobs() throws IOException {
- return null;
- }
-
- @Override
- public String getSystemDir() {
- return WORKING_DIR;
- }
-
- @Override
- public void killJob(BSPJobID jobid) throws IOException {
- return;
- }
-
- @Override
- public boolean killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException {
- return false;
- }
-
- // this class will spawn a new thread and executes the BSP
- class BSPRunner implements Callable {
-
- Configuration conf;
- BSPJob job;
- BSP bsp;
- LocalGroom groom;
-
- public BSPRunner(Configuration conf, BSPJob job, BSP bsp, LocalGroom groom) {
- super();
- this.conf = conf;
- this.job = job;
- this.bsp = bsp;
- this.groom = groom;
- }
-
- public void run() {
- bsp.setConf(conf);
- try {
- bsp.bsp(groom);
- } catch (Exception e) {
- LOG.error("Exception during BSP execution!", e);
- }
- }
-
- @Override
- public BSP call() throws Exception {
- run();
- return bsp;
- }
- }
-
- // this thread observes the status of the runners.
- class ThreadObserver implements Runnable {
-
- JobStatus status;
-
- public ThreadObserver(JobStatus currentJobStatus) {
- this.status = currentJobStatus;
- }
-
- @Override
- public void run() {
- boolean success = true;
- for (Future future : futureList) {
- try {
- future.get();
- } catch (InterruptedException e) {
- LOG.error("Exception during BSP execution!", e);
- success = false;
- } catch (ExecutionException e) {
- LOG.error("Exception during BSP execution!", e);
- success = false;
- }
- }
- if (success) {
- currentJobStatus.setState(JobStatus.State.SUCCEEDED);
- currentJobStatus.setRunState(JobStatus.SUCCEEDED);
- } else {
- currentJobStatus.setState(JobStatus.State.FAILED);
- currentJobStatus.setRunState(JobStatus.FAILED);
- }
- threadPool.shutdownNow();
- }
-
- }
-
- class LocalGroom extends BSPPeer {
- private long superStepCount = 0;
- private final ConcurrentLinkedQueue localMessageQueue = new ConcurrentLinkedQueue();
- // outgoing queue
- private final Map> outgoingQueues = new ConcurrentHashMap>();
- private final String peerName;
-
- public LocalGroom(String peerName) throws IOException {
- this.peerName = peerName;
- }
-
- @Override
- public void send(String peerName, BSPMessage msg) throws IOException {
- if (this.peerName.equals(peerName)) {
- put(msg);
- } else {
- // put this into a outgoing queue
- if (outgoingQueues.get(peerName) == null) {
- outgoingQueues.put(peerName, new ConcurrentLinkedQueue());
- }
- outgoingQueues.get(peerName).add(msg);
- }
- }
-
- @Override
- public void put(BSPMessage msg) throws IOException {
- localMessageQueue.add(msg);
- }
-
- @Override
- public BSPMessage getCurrentMessage() throws IOException {
- return localMessageQueue.poll();
- }
-
- @Override
- public int getNumCurrentMessages() {
- return localMessageQueue.size();
- }
-
- @Override
- public void sync() throws IOException, KeeperException,
- InterruptedException {
- // wait until all threads reach this barrier
- barrierSync();
- // send the messages
- for (Entry> entry : outgoingQueues
- .entrySet()) {
- String peerName = entry.getKey();
- for (BSPMessage msg : entry.getValue())
- localGrooms.get(peerName).put(msg);
- }
- // clear the local outgoing queue
- outgoingQueues.clear();
- // sync again to avoid data inconsistency
- barrierSync();
- incrementSuperSteps();
- }
-
- private void barrierSync() throws InterruptedException {
- try {
- barrier.await();
- } catch (BrokenBarrierException e) {
- throw new InterruptedException("Barrier has been broken!" + e);
- }
- }
-
- private void incrementSuperSteps() {
- currentJobStatus.setprogress(superStepCount++);
- currentJobStatus.setSuperstepCount(currentJobStatus.progress());
- }
-
- @Override
- public long getSuperstepCount() {
- return superStepCount;
- }
-
- @Override
- public String getPeerName() {
- return peerName;
- }
-
- @Override
- public String[] getAllPeerNames() {
- return localGrooms.keySet().toArray(
- new String[localGrooms.keySet().size()]);
- }
-
- @Override
- public void clear() {
- localMessageQueue.clear();
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return 3;
- }
-
- @Override
- public void close() throws IOException {
-
- }
-
- @Override
- public void put(BSPMessageBundle messages) throws IOException {
- }
-
- }
-}
Index: src/java/org/apache/hama/bsp/JobProfile.java
===================================================================
--- src/java/org/apache/hama/bsp/JobProfile.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobProfile.java (working copy)
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-
-/**
- * A JobProfile tracks job's status.
- */
-public class JobProfile implements Writable {
-
- static { // register actor
- WritableFactories.setFactory(JobProfile.class, new WritableFactory() {
- public Writable newInstance() {
- return new JobProfile();
- }
- });
- }
-
- String user;
- final BSPJobID jobid;
- String jobFile;
- String name;
-
- /**
- * Construct an empty {@link JobProfile}.
- */
- public JobProfile() {
- jobid = new BSPJobID();
- }
-
- /**
- * Construct a {@link JobProfile} the userid, jobid, job config-file,
- * job-details url and job name.
- *
- * @param user userid of the person who submitted the job.
- * @param jobid id of the job.
- * @param jobFile job configuration file.
- * @param name user-specified job name.
- */
- public JobProfile(String user, BSPJobID jobid, String jobFile, String name) {
- this.user = user;
- this.jobid = jobid;
- this.jobFile = jobFile;
- this.name = name;
- }
-
- /**
- * Get the user id.
- */
- public String getUser() {
- return user;
- }
-
- /**
- * Get the job id.
- */
- public BSPJobID getJobID() {
- return jobid;
- }
-
- /**
- * Get the configuration file for the job.
- */
- public String getJobFile() {
- return jobFile;
- }
-
- /**
- * Get the user-specified job name.
- */
- public String getJobName() {
- return name;
- }
-
- public void write(DataOutput out) throws IOException {
- jobid.write(out);
- Text.writeString(out, jobFile);
- Text.writeString(out, user);
- Text.writeString(out, name);
- }
-
- public void readFields(DataInput in) throws IOException {
- jobid.readFields(in);
- this.jobFile = Text.readString(in);
- this.user = Text.readString(in);
- this.name = Text.readString(in);
- }
-}
Index: src/java/org/apache/hama/bsp/LaunchTaskAction.java
===================================================================
--- src/java/org/apache/hama/bsp/LaunchTaskAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/LaunchTaskAction.java (working copy)
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the
- * {@link org.apache.hama.bsp.GroomServer} to launch a new task.
- */
-class LaunchTaskAction extends GroomServerAction {
- private Task task;
-
- public LaunchTaskAction() {
- super(ActionType.LAUNCH_TASK);
- }
-
- public LaunchTaskAction(Task task) {
- super(ActionType.LAUNCH_TASK);
- this.task = task;
- }
-
- public Task getTask() {
- return task;
- }
-
- public void write(DataOutput out) throws IOException {
- task.write(out);
- }
-
- public void readFields(DataInput in) throws IOException {
- task = new BSPTask();
- task.readFields(in);
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSPJobID.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPJobID.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPJobID.java (working copy)
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-import org.apache.hadoop.io.Text;
-
-/**
- * BSPJobID represents the immutable and unique identifier for the job.
- */
-public class BSPJobID extends ID implements Comparable {
- protected static final String JOB = "job";
- private final Text jtIdentifier;
-
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(4);
- }
-
- public BSPJobID(String jtIdentifier, int id) {
- super(id);
- this.jtIdentifier = new Text(jtIdentifier);
- }
-
- public BSPJobID() {
- jtIdentifier = new Text();
- }
-
- public String getJtIdentifier() {
- return jtIdentifier.toString();
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- BSPJobID that = (BSPJobID) o;
- return this.jtIdentifier.equals(that.jtIdentifier);
- }
-
- @Override
- public int compareTo(ID o) {
- BSPJobID that = (BSPJobID) o;
- int jtComp = this.jtIdentifier.compareTo(that.jtIdentifier);
- if (jtComp == 0) {
- return this.id - that.id;
- } else
- return jtComp;
- }
-
- public StringBuilder appendTo(StringBuilder builder) {
- builder.append(SEPARATOR);
- builder.append(jtIdentifier);
- builder.append(SEPARATOR);
- builder.append(idFormat.format(id));
- return builder;
- }
-
- @Override
- public int hashCode() {
- return jtIdentifier.hashCode() + id;
- }
-
- @Override
- public String toString() {
- return appendTo(new StringBuilder(JOB)).toString();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- this.jtIdentifier.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- jtIdentifier.write(out);
- }
-
- public static BSPJobID forName(String str) throws IllegalArgumentException {
- if (str == null)
- return null;
- try {
- String[] parts = str.split("_");
- if (parts.length == 3) {
- if (parts[0].equals(JOB)) {
- return new BSPJobID(parts[1], Integer.parseInt(parts[2]));
- }
- }
- } catch (Exception ex) {
- }
- throw new IllegalArgumentException("JobId string : " + str
- + " is not properly formed");
- }
-}
Index: src/java/org/apache/hama/bsp/JobInProgressListener.java
===================================================================
--- src/java/org/apache/hama/bsp/JobInProgressListener.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobInProgressListener.java (working copy)
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-/**
- * A listener for changes in a {@link JobInProgress job}'s lifecycle in the
- * {@link BSPMaster}.
- */
-abstract class JobInProgressListener {
-
- /**
- * Invoked when a new job has been added to the {@link BSPMaster}.
- * @param job The job to be added.
- * @throws IOException
- */
- public abstract void jobAdded(JobInProgress job) throws IOException;
-
- /**
- * Invoked when a job has been removed from the {@link BSPMaster}.
- * @param job The job to be removed .
- * @throws IOException
- */
- public abstract void jobRemoved(JobInProgress job) throws IOException;
-
-}
Index: src/java/org/apache/hama/bsp/ReinitGroomAction.java
===================================================================
--- src/java/org/apache/hama/bsp/ReinitGroomAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/ReinitGroomAction.java (working copy)
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the
- * {@link org.apache.hama.bsp.GroomServer} to reinitialize itself.
- */
-class ReinitGroomAction extends GroomServerAction {
-
- public ReinitGroomAction() {
- super(ActionType.REINIT_GROOM);
- }
-
- public void write(DataOutput out) throws IOException {
- }
-
- public void readFields(DataInput in) throws IOException {
- }
-
-}
Index: src/java/org/apache/hama/bsp/TaskRunner.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskRunner.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskRunner.java (working copy)
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.InetSocketAddress;
-import java.util.Vector;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.RunJar;
-
-/**
- * Base class that runs a task in a separate process.
- */
-public class TaskRunner extends Thread {
-
- public static final Log LOG = LogFactory.getLog(TaskRunner.class);
-
- boolean killed = false;
- private Process process;
- private Task task;
- private BSPJob conf;
- private GroomServer groomServer;
-
- public TaskRunner(BSPTask bspTask, GroomServer groom, BSPJob conf) {
- this.task = bspTask;
- this.conf = conf;
- this.groomServer = groom;
- }
-
- public Task getTask() {
- return task;
- }
-
- /**
- * Called to assemble this task's input. This method is run in the parent
- * process before the child is spawned. It should not execute user code, only
- * system code.
- */
- public boolean prepare() throws IOException {
- return true;
- }
-
- public void run() {
- try {
- String sep = System.getProperty("path.separator");
- File workDir = new File(new File(task.getJobFile()).getParent(), "work");
- boolean isCreated = workDir.mkdirs();
- if(!isCreated) {
- LOG.debug("TaskRunner.workDir : " + workDir);
- }
-
- StringBuffer classPath = new StringBuffer();
- // start with same classpath as parent process
- classPath.append(System.getProperty("java.class.path"));
- classPath.append(sep);
-
- String jar = conf.getJar();
- if (jar != null) { // if jar exists, it into workDir
- RunJar.unJar(new File(jar), workDir);
- File[] libs = new File(workDir, "lib").listFiles();
- if (libs != null) {
- for (int i = 0; i < libs.length; i++) {
- classPath.append(sep); // add libs from jar to classpath
- classPath.append(libs[i]);
- }
- }
- classPath.append(sep);
- classPath.append(new File(workDir, "classes"));
- classPath.append(sep);
- classPath.append(workDir);
- }
-
- // Build exec child jmv args.
- Vector vargs = new Vector();
- File jvm = // use same jvm as parent
- new File(new File(System.getProperty("java.home"), "bin"), "java");
- vargs.add(jvm.toString());
-
- // bsp.child.java.opts
- String javaOpts = conf.getConf().get("bsp.child.java.opts", "-Xmx200m");
- javaOpts = javaOpts.replace("@taskid@", task.getTaskID().toString());
-
- String[] javaOptsSplit = javaOpts.split(" ");
- for (int i = 0; i < javaOptsSplit.length; i++) {
- vargs.add(javaOptsSplit[i]);
- }
-
- // Add classpath.
- vargs.add("-classpath");
- vargs.add(classPath.toString());
- // Add main class and its arguments
- vargs.add(GroomServer.Child.class.getName()); // main of Child
-
- InetSocketAddress addr = groomServer.getTaskTrackerReportAddress();
- vargs.add(addr.getHostName());
- vargs.add(Integer.toString(addr.getPort()));
- vargs.add(task.getTaskID().toString());
- vargs.add(groomServer.groomHostName);
-
- // Run java
- runChild((String[]) vargs.toArray(new String[0]), workDir);
- } catch (IOException e) {
- LOG.error(e);
- }
- }
-
- /**
- * Run the child process
- */
- private void runChild(String[] args, File dir) throws IOException {
- this.process = Runtime.getRuntime().exec(args, null, dir);
- try {
- new Thread() {
- public void run() {
- logStream(process.getErrorStream()); // copy log output
- }
- }.start();
-
- logStream(process.getInputStream()); // normally empty
-
- int exit_code = process.waitFor();
- if (!killed && exit_code != 0) {
- throw new IOException("Task process exit with nonzero status of "
- + exit_code + ".");
- }
-
- } catch (InterruptedException e) {
- throw new IOException(e.toString());
- } finally {
- kill();
- }
- }
-
- /**
- * Kill the child process
- */
- public void kill() {
- if (process != null) {
- process.destroy();
- }
- killed = true;
- }
-
- /**
- */
- private void logStream(InputStream output) {
- try {
- BufferedReader in = new BufferedReader(new InputStreamReader(output));
- String line;
- while ((line = in.readLine()) != null) {
- LOG.info(task.getTaskID() + " " + line);
- }
- } catch (IOException e) {
- LOG.warn(task.getTaskID() + " Error reading child output", e);
- } finally {
- try {
- output.close();
- } catch (IOException e) {
- LOG.warn(task.getTaskID() + " Error closing child output", e);
- }
- }
- }
-
-}
Index: src/java/org/apache/hama/bsp/DirectiveException.java
===================================================================
--- src/java/org/apache/hama/bsp/DirectiveException.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/DirectiveException.java (working copy)
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * A custom exception class for Directive.
- */
-public class DirectiveException extends RuntimeException{
- private static final long serialVersionUID = -8052582046894492822L;
-
- public DirectiveException(){
- super();
- }
-
- public DirectiveException(String message){
- super(message);
- }
-
- public DirectiveException(String message, Throwable t){
- super(message, t);
- }
-}
Index: src/java/org/apache/hama/BSPMasterRunner.java
===================================================================
--- src/java/org/apache/hama/BSPMasterRunner.java (revision 1152138)
+++ src/java/org/apache/hama/BSPMasterRunner.java (working copy)
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hama.bsp.BSPMaster;
-
-/**
- * This class starts and runs the BSPMaster.
- */
-public class BSPMasterRunner extends Configured implements Tool {
-
- public static final Log LOG = LogFactory.getLog(BSPMasterRunner.class);
-
- @Override
- public int run(String[] args) throws Exception {
- StringUtils.startupShutdownMessage(BSPMaster.class, args, LOG);
-
- if (args.length != 0) {
- System.out.println("usage: BSPMasterRunner");
- System.exit(-1);
- }
-
- try {
- HamaConfiguration conf = new HamaConfiguration(getConf());
- BSPMaster master = BSPMaster.startMaster(conf);
- master.offerService();
- } catch (Throwable e) {
- LOG.fatal(StringUtils.stringifyException(e));
- return -1;
- }
- return 0;
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new BSPMasterRunner(), args);
- System.exit(exitCode);
- }
-
-}
Index: src/java/org/apache/hama/ipc/BSPPeerProtocol.java
===================================================================
--- src/java/org/apache/hama/ipc/BSPPeerProtocol.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/BSPPeerProtocol.java (working copy)
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.hama.Constants;
-import org.apache.hama.bsp.PeerNames;
-import org.apache.hama.bsp.Task;
-import org.apache.hama.bsp.TaskAttemptID;
-
-/**
- * Protocol that task child process uses to contact its parent process.
- */
-public interface BSPPeerProtocol extends HamaRPCProtocolVersion, Closeable,
- Constants {
-
- /** Called when a child task process starts, to get its task. */
- Task getTask(TaskAttemptID taskid) throws IOException;
-
- /**
- * Periodically called by child to check if parent is still alive.
- *
- * @return True if the task is known
- */
- boolean ping(TaskAttemptID taskid) throws IOException;
-
- /**
- * Report that the task is successfully completed. Failure is assumed if the
- * task process exits without calling this.
- *
- * @param taskid task's id
- * @param shouldBePromoted whether to promote the task's output or not
- */
- void done(TaskAttemptID taskid, boolean shouldBePromoted) throws IOException;
-
- /** Report that the task encounted a local filesystem error. */
- void fsError(TaskAttemptID taskId, String message) throws IOException;
-
- void incrementSuperstepCount(TaskAttemptID taskid) throws IOException;
-
- /**
- * @return the all BSPPeer names.
- */
- PeerNames getAllPeerNames();
-
-}
Index: src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java
===================================================================
--- src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java (working copy)
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import org.apache.hadoop.ipc.VersionedProtocol;
-
-/**
- * There is one version id for all the RPC interfaces. If any interface is
- * changed, the versionID must be changed here.
- */
-public interface HamaRPCProtocolVersion extends VersionedProtocol {
- public static final long versionID = 1L;
-}
Index: src/java/org/apache/hama/ipc/GroomProtocol.java
===================================================================
--- src/java/org/apache/hama/ipc/GroomProtocol.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/GroomProtocol.java (working copy)
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-
-import org.apache.hama.bsp.Directive;
-
-/**
- * A protocol for BSPMaster talks to GroomServer.
- */
-public interface GroomProtocol extends HamaRPCProtocolVersion {
-
- /**
- * Instruct GroomServer performaning tasks.
- *
- * @param directive instructs a GroomServer performing necessary
- * execution.
- * @throws IOException
- */
- void dispatch(Directive directive) throws IOException;
-
-}
Index: src/java/org/apache/hama/ipc/JobSubmissionProtocol.java
===================================================================
--- src/java/org/apache/hama/ipc/JobSubmissionProtocol.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/JobSubmissionProtocol.java (working copy)
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-
-import org.apache.hama.bsp.ClusterStatus;
-import org.apache.hama.bsp.BSPJobID;
-import org.apache.hama.bsp.JobProfile;
-import org.apache.hama.bsp.JobStatus;
-import org.apache.hama.bsp.TaskAttemptID;
-
-/**
- * Protocol that a groom server and the central BSP Master use to communicate.
- * This interface will contains several methods: submitJob, killJob, and
- * killTask.
- */
-public interface JobSubmissionProtocol extends HamaRPCProtocolVersion {
-
- /**
- * Allocate a new id for the job.
- *
- * @return job id
- * @throws IOException
- */
- public BSPJobID getNewJobId() throws IOException;
-
- /**
- * Submit a Job for execution. Returns the latest profile for that job. The
- * job files should be submitted in system-dir/jobName.
- *
- * @param jobID
- * @param jobFile
- * @return jobStatus
- * @throws IOException
- */
- // public JobStatus submitJob(BSPJobID jobName) throws IOException;
-
- public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException;
-
- /**
- * Get the current status of the cluster
- *
- * @param detailed if true then report groom names as well
- * @return summary of the state of the cluster
- */
- public ClusterStatus getClusterStatus(boolean detailed) throws IOException;
-
- /**
- * Grab a handle to a job that is already known to the BSPMaster.
- *
- * @return Profile of the job, or null if not found.
- */
- public JobProfile getJobProfile(BSPJobID jobid) throws IOException;
-
- /**
- * Grab a handle to a job that is already known to the BSPMaster.
- *
- * @return Status of the job, or null if not found.
- */
- public JobStatus getJobStatus(BSPJobID jobid) throws IOException;
-
- /**
- * A BSP system always operates on a single filesystem. This function returns
- * the fs name. ('local' if the localfs; 'addr:port' if dfs). The client can
- * then copy files into the right locations prior to submitting the job.
- */
- public String getFilesystemName() throws IOException;
-
- /**
- * Get the jobs that are not completed and not failed
- *
- * @return array of JobStatus for the running/to-be-run jobs.
- */
- public JobStatus[] jobsToComplete() throws IOException;
-
- /**
- * Get all the jobs submitted.
- *
- * @return array of JobStatus for the submitted jobs
- */
- public JobStatus[] getAllJobs() throws IOException;
-
- /**
- * Grab the bspmaster system directory path where job-specific files are to be
- * placed.
- *
- * @return the system directory where job-specific files are to be placed.
- */
- public String getSystemDir();
-
- /**
- * Kill the indicated job
- */
- public void killJob(BSPJobID jobid) throws IOException;
-
- /**
- * Kill indicated task attempt.
- *
- * @param taskId the id of the task to kill.
- * @param shouldFail if true the task is failed and added to failed tasks
- * list, otherwise it is just killed, w/o affecting job failure
- * status.
- */
- public boolean killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException;
-
-}
Index: src/java/org/apache/hama/ipc/MasterProtocol.java
===================================================================
--- src/java/org/apache/hama/ipc/MasterProtocol.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/MasterProtocol.java (working copy)
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-
-import org.apache.hama.bsp.GroomServerStatus;
-import org.apache.hama.bsp.Directive;
-
-/**
- * A new protocol for GroomServers communicate with BSPMaster. This
- * protocol paired with WorkerProtocl, let GroomServers enrol with
- * BSPMaster, so that BSPMaster can dispatch tasks to GroomServers.
- */
-public interface MasterProtocol extends HamaRPCProtocolVersion {
-
- /**
- * A GroomServer register with its status to BSPMaster, which will update
- * GroomServers cache.
- *
- * @param status to be updated in cache.
- * @return true if successfully register with BSPMaster; false if fail.
- */
- boolean register(GroomServerStatus status) throws IOException;
-
- /**
- * A GroomServer (periodically) reports task statuses back to the BSPMaster.
- * @param directive
- */
- boolean report(Directive directive) throws IOException;
-
- public String getSystemDir();
-
-}
Index: src/java/org/apache/hama/ipc/package.html
===================================================================
--- src/java/org/apache/hama/ipc/package.html (revision 1152138)
+++ src/java/org/apache/hama/ipc/package.html (working copy)
@@ -1,23 +0,0 @@
-
-
-
-
-
-Tools to help define network clients and servers.
-
-
Index: src/java/org/apache/hama/Constants.java
===================================================================
--- src/java/org/apache/hama/Constants.java (revision 1152138)
+++ src/java/org/apache/hama/Constants.java (working copy)
@@ -1,95 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-/**
- * Some constants used in the Hama.
- */
-public interface Constants {
-
- public static final String GROOM_RPC_HOST = "bsp.groom.rpc.hostname";
-
- public static final String DEFAULT_GROOM_RPC_HOST = "0.0.0.0";
-
- public static final String GROOM_RPC_PORT = "bsp.groom.rpc.port";
-
- /** Default port region rpc server listens on. */
- public static final int DEFAULT_GROOM_RPC_PORT = 50000;
-
-
- ///////////////////////////////////////
- // Constants for BSP Package
- ///////////////////////////////////////
- /** default host address */
- public static final String PEER_HOST = "bsp.peer.hostname";
- /** default host address */
- public static final String DEFAULT_PEER_HOST = "0.0.0.0";
-
- public static final String PEER_PORT = "bsp.peer.port";
- /** Default port region server listens on. */
- public static final int DEFAULT_PEER_PORT = 61000;
-
- public static final String PEER_ID = "bsp.peer.id";
-
- /** Parameter name for what groom server implementation to use. */
- public static final String GROOM_SERVER_IMPL= "hama.groomserver.impl";
-
- /** When we encode strings, we always specify UTF8 encoding */
- static final String UTF8_ENCODING = "UTF-8";
-
- ///////////////////////////////////////
- // Constants for ZooKeeper
- ///////////////////////////////////////
- /** zookeeper root */
- public static final String ZOOKEEPER_ROOT = "bsp.zookeeper.root";
- /** zookeeper default root */
- public static final String DEFAULT_ZOOKEEPER_ROOT = "/bsp";
-
- /** zookeeper server address */
- public static final String ZOOKEEPER_SERVER_ADDRS = "zookeeper.server";
- /** zookeeper default server address */
- static final String DEFAULT_ZOOKEEPER_SERVER_ADDR = "localhost:21810";
- /** Parameter name for number of times to retry writes to ZooKeeper. */
- public static final String ZOOKEEPER_RETRIES = "zookeeper.retries";
- /** Default number of times to retry writes to ZooKeeper. */
- public static final int DEFAULT_ZOOKEEPER_RETRIES = 5;
- /** Parameter name for ZooKeeper pause between retries. In milliseconds. */
- public static final String ZOOKEEPER_PAUSE = "zookeeper.pause";
- /** Default ZooKeeper pause value. In milliseconds. */
- public static final int DEFAULT_ZOOKEEPER_PAUSE = 2 * 1000;
-
- static final String ZOOKEEPER_CONFIG_NAME = "zoo.cfg";
- static final String ZOOKEEPER_CLIENT_PORT = "hama.zookeeper.property.clientPort";
- static final String ZOOKEEPER_SESSION_TIMEOUT = "hama.zookeeper.session.timeout";
- static final int DEFAULT_ZOOKEEPER_CLIENT_PORT = 21810;
- static final String ZOOKEEPER_QUORUM = "hama.zookeeper.quorum";
- /** Cluster is in distributed mode or not */
- static final String CLUSTER_DISTRIBUTED = "hama.cluster.distributed";
- /** Cluster is fully-distributed */
- static final String CLUSTER_IS_DISTRIBUTED = "true";
-
-
- // Other constants
-
- /**
- * An empty instance.
- */
- static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
-}
Index: src/java/org/apache/hama/ZooKeeperRunner.java
===================================================================
--- src/java/org/apache/hama/ZooKeeperRunner.java (revision 1152138)
+++ src/java/org/apache/hama/ZooKeeperRunner.java (working copy)
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hama.zookeeper.QuorumPeer;
-
-/**
- * This class starts and runs the ZooKeeperServer.
- */
-public class ZooKeeperRunner extends Configured implements Tool {
-
- public static final Log LOG = LogFactory.getLog(ZooKeeperRunner.class);
-
- @Override
- public int run(String[] args) throws Exception {
- QuorumPeer.run(getConf());
- return 0;
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new ZooKeeperRunner(), args);
- System.exit(exitCode);
- }
-
-}
Index: src/java/org/apache/hama/util/ClusterUtil.java
===================================================================
--- src/java/org/apache/hama/util/ClusterUtil.java (revision 1152138)
+++ src/java/org/apache/hama/util/ClusterUtil.java (working copy)
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.bsp.BSPMaster;
-import org.apache.hama.bsp.GroomServer;
-
-public class ClusterUtil {
- private static final Log LOG = LogFactory.getLog(ClusterUtil.class);
-
- /**
- * Data Structure to hold GroomServer Thread and GroomServer instance
- */
- public static class GroomServerThread extends Thread {
- private final GroomServer groomServer;
-
- public GroomServerThread(final GroomServer r, final int index) {
- super(r, "GroomServer:" + index);
- this.groomServer = r;
- }
-
- /** @return the groom server */
- public GroomServer getGroomServer() {
- return this.groomServer;
- }
-
- /**
- * Block until the groom server has come online, indicating it is ready
- * to be used.
- */
- public void waitForServerOnline() {
- while (!groomServer.isRunning()) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // continue waiting
- }
- }
- }
- }
-
- /**
- * Creates a {@link GroomServerThread}.
- * Call 'start' on the returned thread to make it run.
- * @param c Configuration to use.
- * @param hrsc Class to create.
- * @param index Used distingushing the object returned.
- * @throws IOException
- * @return Groom server added.
- */
- public static ClusterUtil.GroomServerThread createGroomServerThread(final Configuration c,
- final Class extends GroomServer> hrsc, final int index)
- throws IOException {
- GroomServer server;
- try {
- server = hrsc.getConstructor(Configuration.class).newInstance(c);
- } catch (Exception e) {
- IOException ioe = new IOException();
- ioe.initCause(e);
- throw ioe;
- }
- return new ClusterUtil.GroomServerThread(server, index);
- }
-
- /**
- * Start the cluster.
- * @param m
- * @param conf
- * @param groomservers
- * @return Address to use contacting master.
- * @throws InterruptedException
- * @throws IOException
- */
- public static String startup(final BSPMaster m,
- final List groomservers, Configuration conf) throws IOException, InterruptedException {
- if (m != null) {
- BSPMaster.startMaster((HamaConfiguration) conf);
- }
-
- if (groomservers != null) {
- for (ClusterUtil.GroomServerThread t: groomservers) {
- t.start();
- }
- }
-
- return m == null? null: BSPMaster.getAddress(conf).getHostName();
- }
-
- public static void shutdown(BSPMaster master,
- List groomThreads, Configuration conf) {
- LOG.debug("Shutting down HAMA Cluster");
- // TODO:
- }
-}
Index: src/java/org/apache/hama/util/RunJar.java
===================================================================
--- src/java/org/apache/hama/util/RunJar.java (revision 1152138)
+++ src/java/org/apache/hama/util/RunJar.java (working copy)
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.lang.reflect.Array;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-import java.util.jar.Manifest;
-
-import org.apache.hadoop.fs.FileUtil;
-
-/**
- * Run a Hama job jar.
- */
-public class RunJar {
-
- /** Unpack a jar file into a directory. */
- public static void unJar(File jarFile, File toDir) throws IOException {
- JarFile jar = new JarFile(jarFile);
- try {
- Enumeration entries = jar.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = (JarEntry) entries.nextElement();
- if (!entry.isDirectory()) {
- InputStream in = jar.getInputStream(entry);
- try {
- File file = new File(toDir, entry.getName());
- file.getParentFile().mkdirs();
- OutputStream out = new FileOutputStream(file);
- try {
- byte[] buffer = new byte[8192];
- int i;
- while ((i = in.read(buffer)) != -1) {
- out.write(buffer, 0, i);
- }
- } finally {
- out.close();
- }
- } finally {
- in.close();
- }
- }
- }
- } finally {
- jar.close();
- }
- }
-
- /**
- * Run a Hama job jar. If the main class is not in the jar's manifest, then
- * it must be provided on the command line.
- */
- public static void main(String[] args) throws Throwable {
- String usage = "Usage: hama jar [mainClass] args...";
-
- if (args.length < 1) {
- System.err.println(usage);
- System.exit(-1);
- }
-
- int firstArg = 0;
- String fileName = args[firstArg++];
- File file = new File(fileName);
- String mainClassName = null;
-
- JarFile jarFile = new JarFile(fileName);
- Manifest manifest = jarFile.getManifest();
- if (manifest != null) {
- mainClassName = manifest.getMainAttributes().getValue("Main-Class");
- }
- jarFile.close();
-
- if (mainClassName == null) {
- if (args.length < 2) {
- System.err.println(usage);
- System.exit(-1);
- }
- mainClassName = args[firstArg++];
- }
- mainClassName = mainClassName.replaceAll("/", ".");
-
- final File workDir = File.createTempFile("hama-unjar", "");
- workDir.delete();
- workDir.mkdirs();
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- public void run() {
- try {
- FileUtil.fullyDelete(workDir);
- } catch (IOException e) {
- }
- }
- });
-
- unJar(file, workDir);
-
- List classPath = new ArrayList();
- classPath.add(new File(workDir + "/").toURI().toURL());
- classPath.add(file.toURI().toURL());
- classPath.add(new File(workDir, "classes/").toURI().toURL());
- File[] libs = new File(workDir, "lib").listFiles();
- if (libs != null) {
- for (int i = 0; i < libs.length; i++) {
- classPath.add(libs[i].toURI().toURL());
- }
- }
- ClassLoader loader = new URLClassLoader((URL[]) classPath
- .toArray(new URL[0]));
-
- Thread.currentThread().setContextClassLoader(loader);
- Class> mainClass = loader.loadClass(mainClassName);
- Method main = mainClass.getMethod("main", new Class[] { Array.newInstance(
- String.class, 0).getClass() });
- String[] newArgs = (String[]) Arrays.asList(args).subList(firstArg,
- args.length).toArray(new String[0]);
- try {
- main.invoke(null, new Object[] { newArgs });
- } catch (InvocationTargetException e) {
- throw e.getTargetException();
- }
- }
-
-}
Index: src/java/org/apache/hama/util/VersionInfo.java
===================================================================
--- src/java/org/apache/hama/util/VersionInfo.java (revision 1152138)
+++ src/java/org/apache/hama/util/VersionInfo.java (working copy)
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-/**
- * A version information class.
- */
-public class VersionInfo {
-
- public static void main(String[] args) {
- System.out.println("Apache Hama - 0.4");
- }
-}
Index: src/java/org/apache/hama/util/Bytes.java
===================================================================
--- src/java/org/apache/hama/util/Bytes.java (revision 1152138)
+++ src/java/org/apache/hama/util/Bytes.java (working copy)
@@ -1,1226 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.Comparator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.hama.Constants;
-
-/**
- * Utility class that handles byte arrays, conversions to/from other types,
- * comparisons, hash code generation, manufacturing keys for HashMaps or
- * HashSets, etc.
- */
-public class Bytes {
-
- private static final Log LOG = LogFactory.getLog(Bytes.class);
-
- /**
- * Size of boolean in bytes
- */
- public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
-
- /**
- * Size of byte in bytes
- */
- public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
-
- /**
- * Size of char in bytes
- */
- public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
-
- /**
- * Size of double in bytes
- */
- public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
-
- /**
- * Size of float in bytes
- */
- public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
-
- /**
- * Size of int in bytes
- */
- public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
-
- /**
- * Size of long in bytes
- */
- public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
-
- /**
- * Size of short in bytes
- */
- public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
-
- /**
- * Estimate of size cost to pay beyond payload in jvm for instance of byte [].
- * Estimate based on study of jhat and jprofiler numbers.
- */
- // JHat says BU is 56 bytes.
- // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
- public static final int ESTIMATED_HEAP_TAX = 16;
-
- /**
- * Byte array comparator class.
- */
- public static class ByteArrayComparator implements RawComparator {
- /**
- * Constructor
- */
- public ByteArrayComparator() {
- super();
- }
-
- public int compare(byte[] left, byte[] right) {
- return compareTo(left, right);
- }
-
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- return compareTo(b1, s1, l1, b2, s2, l2);
- }
- }
-
- /**
- * Pass this to TreeMaps where byte [] are keys.
- */
- public static Comparator BYTES_COMPARATOR = new ByteArrayComparator();
-
- /**
- * Use comparing byte arrays, byte-by-byte
- */
- public static RawComparator BYTES_RAWCOMPARATOR = new ByteArrayComparator();
-
- /**
- * Read byte-array written with a WritableableUtils.vint prefix.
- *
- * @param in Input to read from.
- * @return byte array read off in
- * @throws IOException e
- */
- public static byte[] readByteArray(final DataInput in) throws IOException {
- int len = WritableUtils.readVInt(in);
- if (len < 0) {
- throw new NegativeArraySizeException(Integer.toString(len));
- }
- byte[] result = new byte[len];
- in.readFully(result, 0, len);
- return result;
- }
-
- /**
- * Read byte-array written with a WritableableUtils.vint prefix. IOException
- * is converted to a RuntimeException.
- *
- * @param in Input to read from.
- * @return byte array read off in
- */
- public static byte[] readByteArrayThrowsRuntime(final DataInput in) {
- try {
- return readByteArray(in);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Write byte-array with a WritableableUtils.vint prefix.
- *
- * @param out output stream to be written to
- * @param b array to write
- * @throws IOException e
- */
- public static void writeByteArray(final DataOutput out, final byte[] b)
- throws IOException {
- if (b == null) {
- WritableUtils.writeVInt(out, 0);
- } else {
- writeByteArray(out, b, 0, b.length);
- }
- }
-
- /**
- * Write byte-array to out with a vint length prefix.
- *
- * @param out output stream
- * @param b array
- * @param offset offset into array
- * @param length length past offset
- * @throws IOException e
- */
- public static void writeByteArray(final DataOutput out, final byte[] b,
- final int offset, final int length) throws IOException {
- WritableUtils.writeVInt(out, length);
- out.write(b, offset, length);
- }
-
- /**
- * Write byte-array from src to tgt with a vint length prefix.
- *
- * @param tgt target array
- * @param tgtOffset offset into target array
- * @param src source array
- * @param srcOffset source offset
- * @param srcLength source length
- * @return New offset in src array.
- */
- public static int writeByteArray(final byte[] tgt, final int tgtOffset,
- final byte[] src, final int srcOffset, final int srcLength) {
- byte[] vint = vintToBytes(srcLength);
- System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
- int offset = tgtOffset + vint.length;
- System.arraycopy(src, srcOffset, tgt, offset, srcLength);
- return offset + srcLength;
- }
-
- /**
- * Put bytes at the specified byte array position.
- *
- * @param tgtBytes the byte array
- * @param tgtOffset position in the array
- * @param srcBytes array to write out
- * @param srcOffset source offset
- * @param srcLength source length
- * @return incremented offset
- */
- public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
- int srcOffset, int srcLength) {
- System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
- return tgtOffset + srcLength;
- }
-
- /**
- * Write a single byte out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param b byte to write out
- * @return incremented offset
- */
- public static int putByte(byte[] bytes, int offset, byte b) {
- bytes[offset] = b;
- return offset + 1;
- }
-
- /**
- * Returns a new byte array, copied from the passed ByteBuffer.
- *
- * @param bb A ByteBuffer
- * @return the byte array
- */
- public static byte[] toBytes(ByteBuffer bb) {
- int length = bb.limit();
- byte[] result = new byte[length];
- System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length);
- return result;
- }
-
- /**
- * @param b Presumed UTF-8 encoded byte array.
- * @return String made from b
- */
- public static String toString(final byte[] b) {
- if (b == null) {
- return null;
- }
- return toString(b, 0, b.length);
- }
-
- /**
- * Joins two byte arrays together using a separator.
- *
- * @param b1 The first byte array.
- * @param sep The separator to use.
- * @param b2 The second byte array.
- */
- public static String toString(final byte[] b1, String sep, final byte[] b2) {
- return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
- }
-
- /**
- * This method will convert utf8 encoded bytes into a string. If an
- * UnsupportedEncodingException occurs, this method will eat it and return
- * null instead.
- *
- * @param b Presumed UTF-8 encoded byte array.
- * @param off offset into array
- * @param len length of utf-8 sequence
- * @return String made from b or null
- */
- public static String toString(final byte[] b, int off, int len) {
- if (b == null) {
- return null;
- }
- if (len == 0) {
- return "";
- }
- try {
- return new String(b, off, len, Constants.UTF8_ENCODING);
- } catch (UnsupportedEncodingException e) {
- LOG.error("UTF-8 not supported?", e);
- return null;
- }
- }
-
- /**
- * Write a printable representation of a byte array.
- *
- * @param b byte array
- * @return string
- * @see #toStringBinary(byte[], int, int)
- */
- public static String toStringBinary(final byte[] b) {
- return toStringBinary(b, 0, b.length);
- }
-
- /**
- * Write a printable representation of a byte array. Non-printable characters
- * are hex escaped in the format \\x%02X, eg: \x00 \x05 etc
- *
- * @param b array to write out
- * @param off offset to start at
- * @param len length to write
- * @return string output
- */
- public static String toStringBinary(final byte[] b, int off, int len) {
- StringBuilder result = new StringBuilder();
- try {
- String first = new String(b, off, len, "ISO-8859-1");
- for (int i = 0; i < first.length(); ++i) {
- int ch = first.charAt(i) & 0xFF;
- if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z')
- || (ch >= 'a' && ch <= 'z') || ch == ',' || ch == '_' || ch == '-'
- || ch == ':' || ch == ' ' || ch == '<' || ch == '>' || ch == '='
- || ch == '/' || ch == '.') {
- result.append(first.charAt(i));
- } else {
- result.append(String.format("\\x%02X", ch));
- }
- }
- } catch (UnsupportedEncodingException e) {
- LOG.error("ISO-8859-1 not supported?", e);
- }
- return result.toString();
- }
-
- private static boolean isHexDigit(char c) {
- return (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9');
- }
-
- /**
- * Takes a ASCII digit in the range A-F0-9 and returns the corresponding
- * integer/ordinal value.
- *
- * @param ch The hex digit.
- * @return The converted hex value as a byte.
- */
- public static byte toBinaryFromHex(byte ch) {
- if (ch >= 'A' && ch <= 'F')
- return (byte) ((byte) 10 + (byte) (ch - 'A'));
- // else
- return (byte) (ch - '0');
- }
-
- public static byte[] toBytesBinary(String in) {
- // this may be bigger than we need, but lets be safe.
- byte[] b = new byte[in.length()];
- int size = 0;
- for (int i = 0; i < in.length(); ++i) {
- char ch = in.charAt(i);
- if (ch == '\\') {
- // begin hex escape:
- char next = in.charAt(i + 1);
- if (next != 'x') {
- // invalid escape sequence, ignore this one.
- b[size++] = (byte) ch;
- continue;
- }
- // ok, take next 2 hex digits.
- char hd1 = in.charAt(i + 2);
- char hd2 = in.charAt(i + 3);
-
- // they need to be A-F0-9:
- if (!isHexDigit(hd1) || !isHexDigit(hd2)) {
- // bogus escape code, ignore:
- continue;
- }
- // turn hex ASCII digit -> number
- byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2));
-
- b[size++] = d;
- i += 3; // skip 3
- } else {
- b[size++] = (byte) ch;
- }
- }
- // resize:
- byte[] b2 = new byte[size];
- System.arraycopy(b, 0, b2, 0, size);
- return b2;
- }
-
- /**
- * Converts a string to a UTF-8 byte array.
- *
- * @param s string
- * @return the byte array
- */
- public static byte[] toBytes(String s) {
- try {
- return s.getBytes(Constants.UTF8_ENCODING);
- } catch (UnsupportedEncodingException e) {
- LOG.error("UTF-8 not supported?", e);
- return null;
- }
- }
-
- /**
- * Convert a boolean to a byte array. True becomes -1 and false becomes 0.
- *
- * @param b value
- * @return b encoded in a byte array.
- */
- public static byte[] toBytes(final boolean b) {
- return new byte[] { b ? (byte) -1 : (byte) 0 };
- }
-
- /**
- * Reverses {@link #toBytes(boolean)}
- *
- * @param b array
- * @return True or false.
- */
- public static boolean toBoolean(final byte[] b) {
- if (b.length != 1) {
- throw new IllegalArgumentException("Array has wrong size: " + b.length);
- }
- return b[0] != (byte) 0;
- }
-
- /**
- * Convert a long value to a byte array using big-endian.
- *
- * @param val value to convert
- * @return the byte array
- */
- public static byte[] toBytes(long val) {
- byte[] b = new byte[8];
- for (int i = 7; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to a long value. Reverses {@link #toBytes(long)}
- *
- * @param bytes array
- * @return the long value
- */
- public static long toLong(byte[] bytes) {
- return toLong(bytes, 0, SIZEOF_LONG);
- }
-
- /**
- * Converts a byte array to a long value. Assumes there will be
- * {@link #SIZEOF_LONG} bytes available.
- *
- * @param bytes bytes
- * @param offset offset
- * @return the long value
- */
- public static long toLong(byte[] bytes, int offset) {
- return toLong(bytes, offset, SIZEOF_LONG);
- }
-
- /**
- * Converts a byte array to a long value.
- *
- * @param bytes array of bytes
- * @param offset offset into array
- * @param length length of data (must be {@link #SIZEOF_LONG})
- * @return the long value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or
- * if there's not enough room in the array at the offset indicated.
- */
- public static long toLong(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_LONG || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
- }
- long l = 0;
- for (int i = offset; i < offset + length; i++) {
- l <<= 8;
- l ^= bytes[i] & 0xFF;
- }
- return l;
- }
-
- private static IllegalArgumentException explainWrongLengthOrOffset(
- final byte[] bytes, final int offset, final int length,
- final int expectedLength) {
- String reason;
- if (length != expectedLength) {
- reason = "Wrong length: " + length + ", expected " + expectedLength;
- } else {
- reason = "offset (" + offset + ") + length (" + length + ") exceed the"
- + " capacity of the array: " + bytes.length;
- }
- return new IllegalArgumentException(reason);
- }
-
- /**
- * Put a long value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val long to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putLong(byte[] bytes, int offset, long val) {
- if (bytes.length - offset < SIZEOF_LONG) {
- throw new IllegalArgumentException("Not enough room to put a long at"
- + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- for (int i = offset + 7; i > offset; i--) {
- bytes[i] = (byte) val;
- val >>>= 8;
- }
- bytes[offset] = (byte) val;
- return offset + SIZEOF_LONG;
- }
-
- /**
- * Presumes float encoded as IEEE 754 floating-point "single format"
- *
- * @param bytes byte array
- * @return Float made from passed byte array.
- */
- public static float toFloat(byte[] bytes) {
- return toFloat(bytes, 0);
- }
-
- /**
- * Presumes float encoded as IEEE 754 floating-point "single format"
- *
- * @param bytes array to convert
- * @param offset offset into array
- * @return Float made from passed byte array.
- */
- public static float toFloat(byte[] bytes, int offset) {
- return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
- }
-
- /**
- * @param bytes byte array
- * @param offset offset to write to
- * @param f float value
- * @return New offset in bytes
- */
- public static int putFloat(byte[] bytes, int offset, float f) {
- return putInt(bytes, offset, Float.floatToRawIntBits(f));
- }
-
- /**
- * @param f float value
- * @return the float represented as byte []
- */
- public static byte[] toBytes(final float f) {
- // Encode it as int
- return Bytes.toBytes(Float.floatToRawIntBits(f));
- }
-
- /**
- * @param bytes byte array
- * @return Return double made from passed bytes.
- */
- public static double toDouble(final byte[] bytes) {
- return toDouble(bytes, 0);
- }
-
- /**
- * @param bytes byte array
- * @param offset offset where double is
- * @return Return double made from passed bytes.
- */
- public static double toDouble(final byte[] bytes, final int offset) {
- return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
- }
-
- /**
- * @param bytes byte array
- * @param offset offset to write to
- * @param d value
- * @return New offset into array bytes
- */
- public static int putDouble(byte[] bytes, int offset, double d) {
- return putLong(bytes, offset, Double.doubleToLongBits(d));
- }
-
- /**
- * Serialize a double as the IEEE 754 double format output. The resultant
- * array will be 8 bytes long.
- *
- * @param d value
- * @return the double represented as byte []
- */
- public static byte[] toBytes(final double d) {
- // Encode it as a long
- return Bytes.toBytes(Double.doubleToRawLongBits(d));
- }
-
- /**
- * Convert an int value to a byte array
- *
- * @param val value
- * @return the byte array
- */
- public static byte[] toBytes(int val) {
- byte[] b = new byte[4];
- for (int i = 3; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @return the int value
- */
- public static int toInt(byte[] bytes) {
- return toInt(bytes, 0, SIZEOF_INT);
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the int value
- */
- public static int toInt(byte[] bytes, int offset) {
- return toInt(bytes, offset, SIZEOF_INT);
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @param length length of int (has to be {@link #SIZEOF_INT})
- * @return the int value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or if
- * there's not enough room in the array at the offset indicated.
- */
- public static int toInt(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_INT || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
- }
- int n = 0;
- for (int i = offset; i < (offset + length); i++) {
- n <<= 8;
- n ^= bytes[i] & 0xFF;
- }
- return n;
- }
-
- /**
- * Put an int value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val int to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putInt(byte[] bytes, int offset, int val) {
- if (bytes.length - offset < SIZEOF_INT) {
- throw new IllegalArgumentException("Not enough room to put an int at"
- + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- for (int i = offset + 3; i > offset; i--) {
- bytes[i] = (byte) val;
- val >>>= 8;
- }
- bytes[offset] = (byte) val;
- return offset + SIZEOF_INT;
- }
-
- /**
- * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long.
- *
- * @param val value
- * @return the byte array
- */
- public static byte[] toBytes(short val) {
- byte[] b = new byte[SIZEOF_SHORT];
- b[1] = (byte) val;
- val >>= 8;
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @return the short value
- */
- public static short toShort(byte[] bytes) {
- return toShort(bytes, 0, SIZEOF_SHORT);
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the short value
- */
- public static short toShort(byte[] bytes, int offset) {
- return toShort(bytes, offset, SIZEOF_SHORT);
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @param length length, has to be {@link #SIZEOF_SHORT}
- * @return the short value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} or
- * if there's not enough room in the array at the offset indicated.
- */
- public static short toShort(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_SHORT || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
- }
- short n = 0;
- n ^= bytes[offset] & 0xFF;
- n <<= 8;
- n ^= bytes[offset + 1] & 0xFF;
- return n;
- }
-
- /**
- * Put a short value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val short to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putShort(byte[] bytes, int offset, short val) {
- if (bytes.length - offset < SIZEOF_SHORT) {
- throw new IllegalArgumentException("Not enough room to put a short at"
- + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- bytes[offset + 1] = (byte) val;
- val >>= 8;
- bytes[offset] = (byte) val;
- return offset + SIZEOF_SHORT;
- }
-
- /**
- * @param vint Integer to make a vint of.
- * @return Vint as bytes array.
- */
- public static byte[] vintToBytes(final long vint) {
- long i = vint;
- int size = WritableUtils.getVIntSize(i);
- byte[] result = new byte[size];
- int offset = 0;
- if (i >= -112 && i <= 127) {
- result[offset] = (byte) i;
- return result;
- }
-
- int len = -112;
- if (i < 0) {
- i ^= -1L; // take one's complement'
- len = -120;
- }
-
- long tmp = i;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- result[offset++] = (byte) len;
-
- len = (len < -120) ? -(len + 120) : -(len + 112);
-
- for (int idx = len; idx != 0; idx--) {
- int shiftbits = (idx - 1) * 8;
- long mask = 0xFFL << shiftbits;
- result[offset++] = (byte) ((i & mask) >> shiftbits);
- }
- return result;
- }
-
- /**
- * @param buffer buffer to convert
- * @return vint bytes as an integer.
- */
- public static long bytesToVint(final byte[] buffer) {
- int offset = 0;
- byte firstByte = buffer[offset++];
- int len = WritableUtils.decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = buffer[offset++];
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
- }
-
- /**
- * Reads a zero-compressed encoded long from input stream and returns it.
- *
- * @param buffer Binary array
- * @param offset Offset into array at which vint begins.
- * @throws java.io.IOException e
- * @return deserialized long from stream.
- */
- public static long readVLong(final byte[] buffer, final int offset)
- throws IOException {
- byte firstByte = buffer[offset];
- int len = WritableUtils.decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = buffer[offset + 1 + idx];
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
- }
-
- /**
- * @param left left operand
- * @param right right operand
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- public static int compareTo(final byte[] left, final byte[] right) {
- return compareTo(left, 0, left.length, right, 0, right.length);
- }
-
- /**
- * Lexographically compare two arrays.
- *
- * @param b1 left operand
- * @param b2 right operand
- * @param s1 Where to start comparing in the left buffer
- * @param s2 Where to start comparing in the right buffer
- * @param l1 How much to compare from the left buffer
- * @param l2 How much to compare from the right buffer
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- public static int compareTo(byte[] b1, int s1, int l1, byte[] b2, int s2,
- int l2) {
- // Bring WritableComparator code local
- int end1 = s1 + l1;
- int end2 = s2 + l2;
- for (int i = s1, j = s2; i < end1 && j < end2; i++, j++) {
- int a = (b1[i] & 0xff);
- int b = (b2[j] & 0xff);
- if (a != b) {
- return a - b;
- }
- }
- return l1 - l2;
- }
-
- /**
- * @param left left operand
- * @param right right operand
- * @return True if equal
- */
- public static boolean equals(final byte[] left, final byte[] right) {
- // Could use Arrays.equals?
- // noinspection SimplifiableConditionalExpression
- if (left == null && right == null) {
- return true;
- }
- return (left == null || right == null || (left.length != right.length) ? false
- : compareTo(left, right) == 0);
- }
-
- /**
- * @param b bytes to hash
- * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
- * passed in array. This method is what
- * {@link org.apache.hadoop.io.Text} use calculating hash code.
- */
- public static int hashCode(final byte[] b) {
- return hashCode(b, b.length);
- }
-
- /**
- * @param b value
- * @param length length of the value
- * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
- * passed in array. This method is what
- * {@link org.apache.hadoop.io.Text} use calculating hash code.
- */
- public static int hashCode(final byte[] b, final int length) {
- return WritableComparator.hashBytes(b, length);
- }
-
- /**
- * @param b bytes to hash
- * @return A hash of b as an Integer that can be used as key in
- * Maps.
- */
- public static Integer mapKey(final byte[] b) {
- return hashCode(b);
- }
-
- /**
- * @param b bytes to hash
- * @param length length to hash
- * @return A hash of b as an Integer that can be used as key in
- * Maps.
- */
- public static Integer mapKey(final byte[] b, final int length) {
- return hashCode(b, length);
- }
-
- /**
- * @param a lower half
- * @param b upper half
- * @return New array that has a in lower half and b in upper half.
- */
- public static byte[] add(final byte[] a, final byte[] b) {
- return add(a, b, Constants.EMPTY_BYTE_ARRAY);
- }
-
- /**
- * @param a first third
- * @param b second third
- * @param c third third
- * @return New array made from a, b and c
- */
- public static byte[] add(final byte[] a, final byte[] b, final byte[] c) {
- byte[] result = new byte[a.length + b.length + c.length];
- System.arraycopy(a, 0, result, 0, a.length);
- System.arraycopy(b, 0, result, a.length, b.length);
- System.arraycopy(c, 0, result, a.length + b.length, c.length);
- return result;
- }
-
- /**
- * @param a array
- * @param length amount of bytes to grab
- * @return First length bytes from a
- */
- public static byte[] head(final byte[] a, final int length) {
- if (a.length < length) {
- return null;
- }
- byte[] result = new byte[length];
- System.arraycopy(a, 0, result, 0, length);
- return result;
- }
-
- /**
- * @param a array
- * @param length amount of bytes to snarf
- * @return Last length bytes from a
- */
- public static byte[] tail(final byte[] a, final int length) {
- if (a.length < length) {
- return null;
- }
- byte[] result = new byte[length];
- System.arraycopy(a, a.length - length, result, 0, length);
- return result;
- }
-
- /**
- * @param a array
- * @param length new array size
- * @return Value in a plus length prepended 0 bytes
- */
- public static byte[] padHead(final byte[] a, final int length) {
- byte[] padding = new byte[length];
- for (int i = 0; i < length; i++) {
- padding[i] = 0;
- }
- return add(padding, a);
- }
-
- /**
- * @param a array
- * @param length new array size
- * @return Value in a plus length appended 0 bytes
- */
- public static byte[] padTail(final byte[] a, final int length) {
- byte[] padding = new byte[length];
- for (int i = 0; i < length; i++) {
- padding[i] = 0;
- }
- return add(a, padding);
- }
-
- /**
- * Split passed range. Expensive operation relatively. Uses BigInteger math.
- * Useful splitting ranges for MapReduce jobs.
- *
- * @param a Beginning of range
- * @param b End of range
- * @param num Number of times to split range. Pass 1 if you want to split the
- * range in two; i.e. one split.
- * @return Array of dividing values
- */
- public static byte[][] split(final byte[] a, final byte[] b, final int num) {
- byte[] aPadded;
- byte[] bPadded;
- if (a.length < b.length) {
- aPadded = padTail(a, b.length - a.length);
- bPadded = b;
- } else if (b.length < a.length) {
- aPadded = a;
- bPadded = padTail(b, a.length - b.length);
- } else {
- aPadded = a;
- bPadded = b;
- }
- if (compareTo(aPadded, bPadded) >= 0) {
- throw new IllegalArgumentException("b <= a");
- }
- if (num <= 0) {
- throw new IllegalArgumentException("num cannot be < 0");
- }
- byte[] prependHeader = { 1, 0 };
- BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
- BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
- BigInteger diffBI = stopBI.subtract(startBI);
- BigInteger splitsBI = BigInteger.valueOf(num + 1);
- if (diffBI.compareTo(splitsBI) < 0) {
- return null;
- }
- BigInteger intervalBI;
- try {
- intervalBI = diffBI.divide(splitsBI);
- } catch (Exception e) {
- LOG.error("Exception caught during division", e);
- return null;
- }
-
- byte[][] result = new byte[num + 2][];
- result[0] = a;
-
- for (int i = 1; i <= num; i++) {
- BigInteger curBI = startBI
- .add(intervalBI.multiply(BigInteger.valueOf(i)));
- byte[] padded = curBI.toByteArray();
- if (padded[1] == 0)
- padded = tail(padded, padded.length - 2);
- else
- padded = tail(padded, padded.length - 1);
- result[i] = padded;
- }
- result[num + 1] = b;
- return result;
- }
-
- /**
- * @param t operands
- * @return Array of byte arrays made from passed array of Text
- */
- public static byte[][] toByteArrays(final String[] t) {
- byte[][] result = new byte[t.length][];
- for (int i = 0; i < t.length; i++) {
- result[i] = Bytes.toBytes(t[i]);
- }
- return result;
- }
-
- /**
- * @param column operand
- * @return A byte array of a byte array where first and only entry is
- * column
- */
- public static byte[][] toByteArrays(final String column) {
- return toByteArrays(toBytes(column));
- }
-
- /**
- * @param column operand
- * @return A byte array of a byte array where first and only entry is
- * column
- */
- public static byte[][] toByteArrays(final byte[] column) {
- byte[][] result = new byte[1][];
- result[0] = column;
- return result;
- }
-
- /**
- * Binary search for keys in indexes.
- *
- * @param arr array of byte arrays to search for
- * @param key the key you want to find
- * @param offset the offset in the key you want to find
- * @param length the length of the key
- * @param comparator a comparator to compare.
- * @return index of key
- */
- public static int binarySearch(byte[][] arr, byte[] key, int offset,
- int length, RawComparator comparator) {
- int low = 0;
- int high = arr.length - 1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- // we have to compare in this order, because the comparator order
- // has special logic when the 'left side' is a special key.
- int cmp = comparator.compare(key, offset, length, arr[mid], 0,
- arr[mid].length);
- // key lives above the midpoint
- if (cmp > 0)
- low = mid + 1;
- // key lives below the midpoint
- else if (cmp < 0)
- high = mid - 1;
- // BAM. how often does this really happen?
- else
- return mid;
- }
- return -(low + 1);
- }
-
- /**
- * Bytewise binary increment/deincrement of long contained in byte array on
- * given amount.
- *
- * @param value - array of bytes containing long (length <= SIZEOF_LONG)
- * @param amount value will be incremented on (deincremented if negative)
- * @return array of bytes containing incremented long (length == SIZEOF_LONG)
- * @throws IOException - if value.length > SIZEOF_LONG
- */
- public static byte[] incrementBytes(byte[] value, long amount)
- throws IOException {
- byte[] val = value;
- if (val.length < SIZEOF_LONG) {
- // Hopefully this doesn't happen too often.
- byte[] newvalue;
- if (val[0] < 0) {
- newvalue = new byte[] { -1, -1, -1, -1, -1, -1, -1, -1 };
- } else {
- newvalue = new byte[SIZEOF_LONG];
- }
- System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
- val.length);
- val = newvalue;
- } else if (val.length > SIZEOF_LONG) {
- throw new IllegalArgumentException("Increment Bytes - value too big: "
- + val.length);
- }
- if (amount == 0)
- return val;
- if (val[0] < 0) {
- return binaryIncrementNeg(val, amount);
- }
- return binaryIncrementPos(val, amount);
- }
-
- /* increment/deincrement for positive value */
- private static byte[] binaryIncrementPos(byte[] value, long amount) {
- long amo = amount;
- int sign = 1;
- if (amount < 0) {
- amo = -amount;
- sign = -1;
- }
- for (int i = 0; i < value.length; i++) {
- int cur = ((int) amo % 256) * sign;
- amo = (amo >> 8);
- int val = value[value.length - i - 1] & 0x0ff;
- int total = val + cur;
- if (total > 255) {
- amo += sign;
- total %= 256;
- } else if (total < 0) {
- amo -= sign;
- }
- value[value.length - i - 1] = (byte) total;
- if (amo == 0)
- return value;
- }
- return value;
- }
-
- /* increment/deincrement for negative value */
- private static byte[] binaryIncrementNeg(byte[] value, long amount) {
- long amo = amount;
- int sign = 1;
- if (amount < 0) {
- amo = -amount;
- sign = -1;
- }
- for (int i = 0; i < value.length; i++) {
- int cur = ((int) amo % 256) * sign;
- amo = (amo >> 8);
- int val = ((~value[value.length - i - 1]) & 0x0ff) + 1;
- int total = cur - val;
- if (total >= 0) {
- amo += sign;
- } else if (total < -256) {
- amo -= sign;
- total %= 256;
- }
- value[value.length - i - 1] = (byte) total;
- if (amo == 0)
- return value;
- }
- return value;
- }
-
-}
Index: src/java/org/apache/hama/util/RandomVariable.java
===================================================================
--- src/java/org/apache/hama/util/RandomVariable.java (revision 1152138)
+++ src/java/org/apache/hama/util/RandomVariable.java (working copy)
@@ -1,227 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-/**
- * The RandomVaraibale Class provides static methods for generating random
- * numbers.
- */
-public class RandomVariable {
-
- /**
- * Generate a random number between 0 and 1.
- *
- * @return a double between 0 and 1.
- */
- public static double rand() {
- double x = Math.random();
- return x;
- }
-
- /**
- * Generate a random integer.
- *
- * @param i0 min of the random variable.
- * @param i1 max of the random variable.
- * @return an int between i0 and i1.
- */
- public static int randInt(int i0, int i1) {
- double x = rand();
- int i = i0 + (int) Math.floor((i1 - i0 + 1) * x);
- return i;
- }
-
- /**
- * Generate a random string using the specified prefix and a fixed length.
- * @param prefix
- * the specified string prefix.
- * @param length
- * the length of the string to be appended.
- * @return random string.
- */
- public static String randString(String prefix, int length) {
- StringBuilder result = new StringBuilder(prefix);
- for (int i = 0; i < length; i++) {
- char ch = (char) ((Math.random() * 26) + 97);
- result.append(ch);
- }
-
- return result.toString();
- }
-
- /**
- * Generate a random number from a uniform random variable.
- *
- * @param min min of the random variable.
- * @param max max of the random variable.
- * @return a double.
- */
- public static double uniform(double min, double max) {
- double x = min + (max - min) * rand();
- return x;
- }
-
- /**
- * Generate a random number from a discrete random variable.
- *
- * @param values discrete values.
- * @param prob probability of each value.
- * @return a double.
- */
- public static double dirac(double[] values, double[] prob) {
- double[] prob_cumul = new double[values.length];
- prob_cumul[0] = prob[0];
- for (int i = 1; i < values.length; i++) {
- prob_cumul[i] = prob_cumul[i - 1] + prob[i];
- }
- double y = rand();
- double x = 0;
- for (int i = 0; i < values.length - 1; i++) {
- if ((y > prob_cumul[i]) && (y < prob_cumul[i + 1])) {
- x = values[i];
- }
- }
- return x;
- }
-
- /**
- * Generate a random number from a Gaussian (Normal) random variable.
- *
- * @param mu mean of the random variable.
- * @param sigma standard deviation of the random variable.
- * @return a double.
- */
- public static double normal(double mu, double sigma) {
- double x = mu + sigma * Math.cos(2 * Math.PI * rand())
- * Math.sqrt(-2 * Math.log(rand()));
- return x;
- }
-
- /**
- * Generate a random number from a Chi-2 random variable.
- *
- * @param n degrees of freedom of the chi2 random variable.
- * @return a double.
- */
- public static double chi2(int n) {
- double x = 0;
- for (int i = 0; i < n; i++) {
- double norm = normal(0, 1);
- x += norm * norm;
- }
- return x;
- }
-
- /**
- * Generate a random number from a LogNormal random variable.
- *
- * @param mu mean of the Normal random variable.
- * @param sigma standard deviation of the Normal random variable.
- * @return a double.
- */
- public static double logNormal(double mu, double sigma) {
- double x = mu + sigma * Math.cos(2 * Math.PI * rand())
- * Math.sqrt(-2 * Math.log(rand()));
- return x;
- }
-
- /**
- * Generate a random number from an exponantial random variable (Mean =
- * 1/lambda, variance = 1/lambda^2).
- *
- * @param lambda parmaeter of the exponential random variable.
- * @return a double.
- */
- public static double exponential(double lambda) {
- double x = -1 / lambda * Math.log(rand());
- return x;
- }
-
- /**
- * Generate a random number from a symetric triangular random variable.
- *
- * @param min min of the random variable.
- * @param max max of the random variable.
- * @return a double.
- */
- public static double triangular(double min, double max) {
- double x = min / 2 + (max - min) * rand() / 2 + min / 2 + (max - min)
- * rand() / 2;
- return x;
- }
-
- /**
- * Generate a random number from a non-symetric triangular random variable.
- *
- * @param min min of the random variable.
- * @param med value of the random variable with max density.
- * @param max max of the random variable.
- * @return a double.
- */
- public static double triangular(double min, double med, double max) {
- double y = rand();
- double x = (y < ((med - min) / (max - min))) ? (min + Math.sqrt(y
- * (max - min) * (med - min))) : (max - Math.sqrt((1 - y) * (max - min)
- * (max - med)));
- return x;
- }
-
- /**
- * Generate a random number from a beta random variable.
- *
- * @param a first parameter of the Beta random variable.
- * @param b second parameter of the Beta random variable.
- * @return a double.
- */
- public static double beta(double a, double b) {
- double try_x;
- double try_y;
- do {
- try_x = Math.pow(rand(), 1 / a);
- try_y = Math.pow(rand(), 1 / b);
- } while ((try_x + try_y) > 1);
- return try_x / (try_x + try_y);
- }
-
- /**
- * Generate a random number from a Cauchy random variable (Mean = Inf, and
- * Variance = Inf).
- *
- * @param mu median of the Weibull random variable
- * @param sigma second parameter of the Cauchy random variable.
- * @return a double.
- */
- public static double cauchy(double mu, double sigma) {
- double x = sigma * Math.tan(Math.PI * (rand() - 0.5)) + mu;
- return x;
- }
-
- /**
- * Generate a random number from a Weibull random variable.
- *
- * @param lambda first parameter of the Weibull random variable.
- * @param c second parameter of the Weibull random variable.
- * @return a double.
- */
- public static double weibull(double lambda, double c) {
- double x = Math.pow(-Math.log(1 - rand()), 1 / c) / lambda;
- return x;
- }
-}
Index: src/java/org/apache/hama/util/BSPServletUtil.java
===================================================================
--- src/java/org/apache/hama/util/BSPServletUtil.java (revision 1152138)
+++ src/java/org/apache/hama/util/BSPServletUtil.java (working copy)
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.Calendar;
-
-import org.apache.hadoop.util.ServletUtil;
-import org.apache.hama.bsp.JobStatus;
-
-public class BSPServletUtil extends ServletUtil {
-
- public static final String HTML_TAIL = "\n"
- + "Hama, "
- + Calendar.getInstance().get(Calendar.YEAR) + ".\n" + "