Property changes on: .
___________________________________________________________________
Modified: svn:ignore
- .project
.classpath
build
.fbprefs
src-gen
hama-0.1.0-dev.jar
hama-0.1.0-dev-examples.jar
hama-0.1.0-dev-test.jar
logs
.settings
+ .project
.classpath
build
.fbprefs
src-gen
hama-0.1.0-dev.jar
hama-0.1.0-dev-examples.jar
hama-0.1.0-dev-test.jar
logs
.settings
lib
target
*.ipr
*.iml
Property changes on: site
___________________________________________________________________
Added: svn:ignore
+ *.iml
target
Index: site/src/docs/forrest.properties
===================================================================
--- site/src/docs/forrest.properties (revision 0)
+++ site/src/docs/forrest.properties (revision 0)
@@ -0,0 +1,104 @@
+# Copyright 2002-2004 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##############
+# Properties used by forrest.build.xml for building the website
+# These are the defaults, un-comment them if you need to change them.
+##############
+
+# Prints out a summary of Forrest settings for this project
+#forrest.echo=true
+
+# Project name (used to name .war file)
+#project.name=my-project
+
+# Specifies name of Forrest skin to use
+#project.skin=tigris
+#project.skin=pelt
+
+# comma separated list, file:// is supported
+#forrest.skins.descriptors=http://forrest.apache.org/skins/skins.xml,file:///c:/myskins/skins.xml
+
+##############
+# behavioural properties
+#project.menu-scheme=tab_attributes
+#project.menu-scheme=directories
+
+##############
+# layout properties
+
+# Properties that can be set to override the default locations
+#
+# Parent properties must be set. This usually means uncommenting
+# project.content-dir if any other property using it is uncommented
+
+#project.status=status.xml
+#project.content-dir=src/documentation
+#project.raw-content-dir=${project.content-dir}/content
+#project.conf-dir=${project.content-dir}/conf
+#project.sitemap-dir=${project.content-dir}
+#project.xdocs-dir=${project.content-dir}/content/xdocs
+#project.resources-dir=${project.content-dir}/resources
+#project.stylesheets-dir=${project.resources-dir}/stylesheets
+#project.images-dir=${project.resources-dir}/images
+#project.schema-dir=${project.resources-dir}/schema
+#project.skins-dir=${project.content-dir}/skins
+#project.skinconf=${project.content-dir}/skinconf.xml
+#project.lib-dir=${project.content-dir}/lib
+#project.classes-dir=${project.content-dir}/classes
+#project.translations-dir=${project.content-dir}/translations
+
+##############
+# validation properties
+
+# This set of properties determine if validation is performed
+# Values are inherited unless overridden.
+# e.g. if forrest.validate=false then all others are false unless set to true.
+forrest.validate=false
+#forrest.validate.xdocs=${forrest.validate}
+#forrest.validate.skinconf=${forrest.validate}
+#forrest.validate.sitemap=${forrest.validate}
+#forrest.validate.stylesheets=${forrest.validate}
+#forrest.validate.skins=${forrest.validate}
+#forrest.validate.skins.stylesheets=${forrest.validate.skins}
+
+# *.failonerror=(true|false) - stop when an XML file is invalid
+#forrest.validate.failonerror=true
+
+# *.excludes=(pattern) - comma-separated list of path patterns to not validate
+# e.g.
+#forrest.validate.xdocs.excludes=samples/subdir/**, samples/faq.xml
+#forrest.validate.xdocs.excludes=
+
+
+##############
+# General Forrest properties
+
+# The URL to start crawling from
+#project.start-uri=linkmap.html
+# Set logging level for messages printed to the console
+# (DEBUG, INFO, WARN, ERROR, FATAL_ERROR)
+#project.debuglevel=ERROR
+# Max memory to allocate to Java
+#forrest.maxmemory=64m
+# Any other arguments to pass to the JVM. For example, to run on an X-less
+# server, set to -Djava.awt.headless=true
+#forrest.jvmargs=
+# The bugtracking URL - the issue number will be appended
+#project.bugtracking-url=http://issues.apache.org/bugzilla/show_bug.cgi?id=
+#project.bugtracking-url=http://issues.apache.org/jira/browse/
+# The issues list as rss
+#project.issues-rss-url=
+#I18n Property only works for the "forrest run" target.
+#project.i18n=true
Index: site/src/docs/status.xml
===================================================================
--- site/src/docs/status.xml (revision 0)
+++ site/src/docs/status.xml (revision 0)
@@ -0,0 +1,74 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Initial Import
+
+
+
+
+
+
+
+
+ Customize this template project with your project's details. This
+ TODO list is generated from 'status.xml'.
+
+
+ Add lots of content. XML content goes in
+ src/documentation/content/xdocs, or wherever the
+ ${project.xdocs-dir} property (set in
+ forrest.properties) points.
+
+
+ Mail forrest-dev@xml.apache.org
+ with feedback.
+
+
+
+
+
+
Index: site/src/docs/src/documentation/skinconf.xml
===================================================================
--- site/src/docs/src/documentation/skinconf.xml (revision 0)
+++ site/src/docs/src/documentation/skinconf.xml (revision 0)
@@ -0,0 +1,354 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+ false
+
+ true
+
+ true
+
+
+ true
+
+
+ true
+
+
+ true
+
+
+ false
+
+
+ true
+
+
+ Apache Hama
+
+ index.html
+ images/hama_paint_logo_120x120.png
+
+
+ Apache Incubator
+
+ http://incubator.apache.org/
+ images/apache-incubator-logo.png
+
+
+
+
+
+
+ images/favicon.ico
+
+
+ 2008
+ The Apache Software Foundation.
+ http://www.apache.org/licenses/
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ #footer a { color: black; }
+ img.logo {
+ float: left;
+ margin-right: 0.5em;
+ }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1in
+ 1in
+ 1.25in
+ 1in
+
+
+
+ false
+
+
+ false
+
+
+
+
+
+ Built with Apache Forrest
+ http://forrest.apache.org/
+ images/built-with-forrest-button.png
+ 88
+ 31
+
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/site.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/site.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/site.xml (revision 0)
@@ -0,0 +1,84 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/irc_channel.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/irc_channel.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/irc_channel.xml (revision 0)
@@ -0,0 +1,46 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ IRC Channel
+
+ There is an IRC channel dedicated to Hama at irc.freenode.org.
+ The name of the channel is #hama.
+
+ The IRC channel can be used for online discussion about hama related stuff,
+ but developers should be careful to transfer all the official decisions
+ or useful discussions to the issue tracking
+ system.
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/index.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/index.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/index.xml (revision 0)
@@ -0,0 +1,91 @@
+
+
+
+
+
+
+
+
+ Welcome to Hama project
+
+
+
+ Overview
+
+ Hama is a distributed computing framework based on
+ BSP (Bulk Synchronous Parallel) computing techniques for massive scientific computations, such as Google's Pregel,
+ Currently being incubated as one of the incubator project by the Apache Software Foundation.
+
+
+
+
+ Getting Started
+
+ Start by installing Hama on a Hadoop cluster.
+
+
+
+
+
+
+ Getting Involved
+
+ Hama is an open source volunteer project under the Apache Software Foundation. We encourage you to learn about the project and contribute your expertise.
+ Here are some starter links:
+
+
+
+
+
+ Recent News
+
+ July 28, 2011: release 0.3.0 available
+
+ Added LockBSPRunner
+ Added web UI for BSP cluster and job monitoring
+ Added more practical examples e.g., Shortest Path Problem, PageRank
+ Performance has improved by more than 25% with BSPMessageBundle
+ Switched from Ant to Maven
+
+
+
+ June 2, 2011: release 0.2.0 available
+
+ BSP computing framework and its examples
+ Pi Estimator and Serialize Printing
+ CLI-based managing and monitoring tool of BSP job
+
+
+
+ Apr 30, 2010: Introduced BSP Worldwide
+ We're introduced in the BSP Worldwide .
+
+
+ May 20, 2008: Accept Hama to be a Apache incubator project
+ The Incubator PMC have voted to accept Hama to be a Apache incubator project.
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/developers.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/developers.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/developers.xml (revision 0)
@@ -0,0 +1,68 @@
+
+
+
+
+
+
+
+
+
+
+
+ Contribution
+
+ We are always looking for contributions. See the
+ Wiki for details on contributing.
+
+
+
+
+ Code Style
+
+ The following sections show how to set up the code style for Hama in Eclipse.
+ It is strongly preferred that patches use this style before they are supplied.
+
+
+ Download hama-eclipse-formatter.xml .
+
+ After this, select Window > Preferences, and open up the configuration for
+ Java > Code Style > Code Formatter. Click on the button labeled Import...
+ and select the file you downloaded. Give the style a name, and click OK.
+
+
+
+ Issues
+
+ All bugs, improvements, patches, etc. should be logged in
+ JIRA
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml (revision 0)
@@ -0,0 +1,37 @@
+
+
+
+
+
+
+
+
+ Welcome to Hama project
+
+
+
+
+
+
+
\ No newline at end of file
Index: site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/ApacheHama-0.2_UserGuide.pdf
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/ApacheHama-0.2_UserGuide.pdf
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml (revision 0)
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
+
+ Welcome to Hama project
+
+
+
+
+
+
+
\ No newline at end of file
Index: site/src/docs/src/documentation/content/xdocs/tabs.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/tabs.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/tabs.xml (revision 0)
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/downloads.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/downloads.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/downloads.xml (revision 0)
@@ -0,0 +1,64 @@
+
+
+
+
+
+
+
+
+ Apache Hama Downloads
+
+
+
+
+ Download
+ Use the links below to download Apache Hama releases from one of our mirrors.
+ You should verify the integrity of the files using the signatures and checksums available from this page.
+
+ Apache Hama releases are available under the
+ Apache License, Version 2.0 .
+ See the NOTICE.txt file contained in each release artifact for applicable copyright attribution notices.
+
+ Download a release now!
+
+
+
+
+ PGP Signatures
+ All of the release distribution package have been digitally signed (using PGP or GPG) by
+ the Apache Group members that constructed them. There will be an accompanying distribution.asc file
+ in the same directory as the distribution.
+ The PGP keys can be found in the MIT key repository (hkp://pgp.mit.edu),
+ the OpenPGP Public Key repository (hkp://subkeys.pgp.net), and within this project's KEYS file,
+ which is located within each download directory.
+
+ Always use the signature files to verify the authenticity of the distribution, e.g.,
+
% pgpk -a KEYS
+ % pgpv hama-x.x.x.tar.gz.asc
+ or,
+ % gpg --import KEYS
+ % gpg --verify hama-x.x.x.tar.gz.asc
+ We offer MD5 hashes as an alternative to validate the integrity of the downloaded files.
+ A unix program called md5 or md5sum is included in many unix distributions.
+ It is also available as part of GNU Core Utilities package .
+
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/issue_tracking.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/issue_tracking.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/issue_tracking.xml (revision 0)
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ Issue Tracking
+
+ Hama tracks both bugs and enhancement requests
+ here using Apache JIRA.
+ We welcome input, however, before filing a request,
+ please make sure you do the following:
+
+
+ Search the JIRA database.
+ Check the user mailing list ,
+ both by searching the archives and by asking questions.
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/privacy_policy.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/privacy_policy.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/privacy_policy.xml (revision 0)
@@ -0,0 +1,61 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ Privacy Policy
+
+ Information about your use of this website is collected using
+ server access logs and a tracking cookie.
+ The collected information consists of the following:
+
+
+ The IP address from which you access the website;
+ The type of browser and operating system you use to access our site;
+ The date and time you access our site;
+ The pages you visit; and
+ The addresses of pages from where you followed a link to our site.
+
+
+ Part of this information is gathered using a tracking cookie set by the
+ Google Analytics service
+ and handled by Google as described in their
+ privacy policy .
+ See your browser documentation for instructions on how to disable
+ the cookie if you prefer not to share this data with Google.
+ We use the gathered information to help us make our site more
+ useful to visitors and to better understand how and when
+ our site is used. We do not track or collect personally
+ identifiable information or associate gathered data
+ with any personally identifying information from other sources.
+ By using this website, you consent to the collection of this data
+ in the manner and for the purpose described above.
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/mailing_lists.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/mailing_lists.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/mailing_lists.xml (revision 0)
@@ -0,0 +1,70 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ Hama User List
+
+ If you use Hama, please subscribe to the Hama user mailing list.
+ The Hama user mailing list is :
+ hama-user@incubator.apache.org .
+
+
+
+
+
+ Hama Developer List
+
+ If you'd like to contribute to Hama, please subscribe to the Hama developer mailing list.
+ The Hama developer mailing list is :
+ hama-dev@incubator.apache.org .
+
+
+
+
+
+ Hama Commit List
+
+ If you'd like to see changes made in Hama's version control system then subscribe
+ to the Hama commit mailing list.
+ The Hama commit mailing list is :
+ hama-commits@incubator.apache.org .
+
+
+
+
+
+
Index: site/src/docs/src/documentation/content/xdocs/who_we_are.xml
===================================================================
--- site/src/docs/src/documentation/content/xdocs/who_we_are.xml (revision 0)
+++ site/src/docs/src/documentation/content/xdocs/who_we_are.xml (revision 0)
@@ -0,0 +1,80 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ Active Committers
+
+ Hama's active committers are:
+
+
+
+ username
+ name
+ organization
+ roles
+ timezone
+
+
+ edwardyoon
+ Edward J. Yoon
+
+
+ +9
+
+
+ chanwit
+ Chanwit Kaewkasi
+ The University of Manchester
+
+ +1
+
+
+
+
+
+ Mentors
+ Mentors while in incubation include:
+
+ Ian Holsman
+ Jeff Eastman
+ Tommaso Toefili
+ Steve Loughran
+
+
+
+
+ Contributors
+ Contributors are:
+
+ Soo-kyung Kim, a contributor of Hama logo sketch.
+ Morakot, a contributor of Hama logo paint.
+
+
+
+
+
Index: site/src/docs/src/documentation/resources/images/hama_art__arthur_300x220.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/hama_art__arthur_300x220.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/hama_paint_logo_120x120.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/hama_paint_logo_120x120.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/hama_art_arthur.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/hama_art_arthur.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/hama_paint_logo.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/hama_paint_logo.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/apache-incubator-logo.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/apache-incubator-logo.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/MapReduce.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/MapReduce.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/hama_logo.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/hama_logo.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/apache-incubator-hama-white.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/apache-incubator-hama-white.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/apache-incubator-hama.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/apache-incubator-hama.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/headerlogo.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/headerlogo.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/block.png
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Property changes on: site/src/docs/src/documentation/resources/images/block.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Index: site/src/docs/src/documentation/resources/images/favicon.ico
===================================================================
Index: site/src/docs/src/documentation/classes/CatalogManager.properties
===================================================================
--- site/src/docs/src/documentation/classes/CatalogManager.properties (revision 0)
+++ site/src/docs/src/documentation/classes/CatalogManager.properties (revision 0)
@@ -0,0 +1,37 @@
+# Copyright 2002-2004 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#=======================================================================
+# CatalogManager.properties
+#
+# This is the default properties file for Apache Forrest.
+# This facilitates local configuration of application-specific catalogs.
+#
+# See the Apache Forrest documentation:
+# http://forrest.apache.org/docs/your-project.html
+# http://forrest.apache.org/docs/validation.html
+
+# verbosity ... level of messages for status/debug
+# See forrest/src/core/context/WEB-INF/cocoon.xconf
+
+# catalogs ... list of additional catalogs to load
+# (Note that Apache Forrest will automatically load its own default catalog
+# from src/core/context/resources/schema/catalog.xcat)
+# use full pathnames
+# pathname separator is always semi-colon (;) regardless of operating system
+# directory separator is always slash (/) regardless of operating system
+#
+#catalogs=/home/me/forrest/my-site/src/documentation/resources/schema/catalog.xcat
+catalogs=
+
Index: site/pom.xml
===================================================================
--- site/pom.xml (revision 0)
+++ site/pom.xml (revision 0)
@@ -0,0 +1,61 @@
+
+
+
+
+
+ org.apache.hama
+ hama-parent
+ 0.4.0-incubating-SNAPSHOT
+
+
+ 4.0.0
+ org.apache.hama
+ hama-site
+ 0.4.0-incubating-SNAPSHOT
+ Apache Hama Site
+ pom
+
+
+
+
+ maven-antrun-plugin
+
+
+ package
+
+
+
+
+
+
+
+
+
+
+
+ run
+
+
+
+
+
+
+
+
Index: conf/hama-env.sh.template
===================================================================
--- conf/hama-env.sh.template (revision 1152138)
+++ conf/hama-env.sh.template (working copy)
@@ -1,37 +0,0 @@
-#
-#/**
-# * Copyright 2007 The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-# Set environment variables here.
-
-# The java implementation to use. Required.
-# export JAVA_HOME=/usr/lib/jvm/java-6-sun
-
-# Where log files are stored. $HAMA_HOME/logs by default.
-# export HAMA_LOG_DIR=${HAMA_HOME}/logs
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-# export HAMA_HEAPSIZE=1000
-
-# Extra ssh options. Empty by default.
-# export HAMA_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HAMA_CONF_DIR"
-
-# Tell Hama whether it should manage it's own instance of Zookeeper or not.
-# export HAMA_MANAGES_ZK=true
Index: conf/hama-site.xml.template
===================================================================
--- conf/hama-site.xml.template (revision 1152138)
+++ conf/hama-site.xml.template (working copy)
@@ -1,25 +0,0 @@
-
-
-
-
-
Index: conf/groomservers.template
===================================================================
--- conf/groomservers.template (revision 1152138)
+++ conf/groomservers.template (working copy)
@@ -1 +0,0 @@
-localhost
\ No newline at end of file
Index: conf/hama-default.xml
===================================================================
--- conf/hama-default.xml (revision 1152138)
+++ conf/hama-default.xml (working copy)
@@ -1,165 +0,0 @@
-
-
-
-
-
- bsp.master.address
- local
- The address of the bsp master server. Either the
- literal string "local" or a host[:port] (where host is a name or
- IP address) for distributed mode.
-
-
-
- bsp.master.port
- 40000
- The port master should bind to.
-
-
- bsp.peer.port
- 61000
- The port an groom server binds to.
-
-
- bsp.groom.rpc.port
- 50000
- The port an groom rpc binds to.
-
-
- bsp.http.infoserver.port
- 40013
- The port where the web-interface can be seen.
-
-
-
- bsp.groom.report.address
- 127.0.0.1:0
- The interface and port that groom server listens on.
- Since it is only connected to by the tasks, it uses the local interface.
- EXPERT ONLY. Should only be changed if your host does not have the loopback
- interface.
-
-
- bsp.system.dir
- ${hadoop.tmp.dir}/bsp/system
- The shared directory where BSP stores control files.
-
-
-
- bsp.local.dir
- ${hadoop.tmp.dir}/bsp/local
- local directory for temporal store.
-
-
- hama.tmp.dir
- /tmp/hama-${user.name}
- Temporary directory on the local filesystem.
-
-
- bsp.child.java.opts
- -Xmx1024m
- Java opts for the groom server child processes.
- The following symbol, if present, will be interpolated: @taskid@ is replaced
- by current TaskID. Any other occurrences of '@' will go unchanged.
- For example, to enable verbose gc logging to a file named for the taskid in
- /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
- -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
- The configuration variable bsp.child.ulimit can be used to control the
- maximum virtual memory of the child processes.
-
-
-
- bsp.local.tasks.maximum
- 20
- Number of tasks that run in parallel when in local mode.
-
-
-
-
- hama.zookeeper.property.initLimit
- 10
- Property from ZooKeeper's config zoo.cfg.
- The number of ticks that the initial synchronization phase can take.
-
-
-
- hama.zookeeper.property.syncLimit
- 5
- Property from ZooKeeper's config zoo.cfg.
- The number of ticks that can pass between sending a request and getting an
- acknowledgment.
-
-
-
- hama.zookeeper.property.dataDir
- ${hama.tmp.dir}/zookeeper
- Property from ZooKeeper's config zoo.cfg.
- The directory where the snapshot is stored.
-
-
-
- hama.zookeeper.property.clientPort
- 21810
- Property from ZooKeeper's config zoo.cfg.
- The port at which the clients will connect.
-
-
-
- hama.zookeeper.property.tickTime
- 3000
-
-
-
- hama.zookeeper.property.maxClientCnxns
- 30
- Property from ZooKeeper's config zoo.cfg.
- Limit on number of concurrent connections (at the socket level) that a
- single client, identified by IP address, may make to a single member of
- the ZooKeeper ensemble. Set high to avoid zk connection issues running
- standalone and pseudo-distributed.
-
-
-
-
- hama.zookeeper.quorum
- localhost
- Comma separated list of servers in the ZooKeeper quorum.
- For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
- By default this is set to localhost for local and pseudo-distributed modes
- of operation. For a fully-distributed setup, this should be set to a full
- list of ZooKeeper quorum servers. If HAMA_MANAGES_ZK is set in hama-env.sh
- this is the list of servers which we will start/stop ZooKeeper on.
-
-
-
-
-
Index: conf/log4j.properties
===================================================================
--- conf/log4j.properties (revision 1152138)
+++ conf/log4j.properties (working copy)
@@ -1,81 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Define some default values that can be overridden by system properties
-hama.root.logger=INFO,console
-hama.log.dir=.
-hama.log.file=hama.log
-
-# Define the root logger to the system property "hama.root.logger".
-log4j.rootLogger=${hama.root.logger}
-
-# Logging Threshold
-log4j.threshhold=ALL
-
-#
-# Daily Rolling File Appender
-#
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hama.log.dir}/${hama.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-#
-# TaskLog Appender
-#
-
-#Default values
-hama.tasklog.taskid=null
-hama.tasklog.noKeepSplits=4
-hama.tasklog.totalLogFileSize=100
-hama.tasklog.purgeLogSplits=true
-hama.tasklog.logsRetainHours=12
-
-log4j.appender.TLA=org.apache.hama.bsp.TaskLogAppender
-log4j.appender.TLA.taskId=${hama.tasklog.taskid}
-log4j.appender.TLA.totalLogFileSize=${hama.tasklog.totalLogFileSize}
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-# Custom Logging levels
-
-#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
-#log4j.logger.org.apache.hadoop.dfs=DEBUG
-#log4j.logger.org.apache.hama=DEBUG
Index: src/test/org/apache/hama/HamaTestCase.java
===================================================================
--- src/test/org/apache/hama/HamaTestCase.java (revision 1152138)
+++ src/test/org/apache/hama/HamaTestCase.java (working copy)
@@ -1,179 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.File;
-import java.io.IOException;
-
-import junit.framework.AssertionFailedError;
-import junit.framework.TestCase;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hama.util.Bytes;
-
-public abstract class HamaTestCase extends TestCase {
- private static Log LOG = LogFactory.getLog(HamaTestCase.class);
-
- /** configuration parameter name for test directory */
- public static final String TEST_DIRECTORY_KEY = "test.build.data";
-
- private boolean localfs = false;
- protected Path testDir = null;
- protected FileSystem fs = null;
-
- static {
- initialize();
- }
-
- public volatile HamaConfiguration conf;
-
- /** constructor */
- public HamaTestCase() {
- super();
- init();
- }
-
- /**
- * @param name
- */
- public HamaTestCase(String name) {
- super(name);
- init();
- }
-
- private void init() {
- conf = new HamaConfiguration();
- conf.setStrings("bsp.local.dir", "/tmp/hama-test");
- conf.set("bsp.master.address", "localhost");
- conf.set("bsp.groom.report.address", "127.0.0.1:0");
- }
-
- /**
- * Note that this method must be called after the mini hdfs cluster has
- * started or we end up with a local file system.
- */
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- localfs =
- (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
-
- if (fs == null) {
- this.fs = FileSystem.get(conf);
- }
- try {
- if (localfs) {
- this.testDir = getUnitTestdir(getName());
- if (fs.exists(testDir)) {
- fs.delete(testDir, true);
- }
- } else {
- this.testDir =
- this.fs.makeQualified(new Path("/tmp/hama-test"));
- }
- } catch (Exception e) {
- LOG.fatal("error during setup", e);
- throw e;
- }
- }
-
- @Override
- protected void tearDown() throws Exception {
- try {
- if (localfs) {
- if (this.fs.exists(testDir)) {
- this.fs.delete(testDir, true);
- }
- }
- } catch (Exception e) {
- LOG.fatal("error during tear down", e);
- }
- super.tearDown();
- }
-
- protected Path getUnitTestdir(String testName) {
- return new Path(
- conf.get(TEST_DIRECTORY_KEY, "/tmp/hama-test/build/data"), testName);
- }
-
- /**
- * Initializes parameters used in the test environment:
- *
- * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set.
- * Sets the boolean debugging if "DEBUGGING" is set in the environment.
- * If debugging is enabled, reconfigures logging so that the root log level is
- * set to WARN and the logging level for the package is set to DEBUG.
- */
- public static void initialize() {
- if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
- System.setProperty(TEST_DIRECTORY_KEY, new File(
- "build/hama/test").getAbsolutePath());
- }
- }
-
- /**
- * Common method to close down a MiniDFSCluster and the associated file system
- *
- * @param cluster
- */
- public static void shutdownDfs(MiniDFSCluster cluster) {
- if (cluster != null) {
- LOG.info("Shutting down Mini DFS ");
- try {
- cluster.shutdown();
- } catch (Exception e) {
- /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
- // here because of an InterruptedException. Don't let exceptions in
- // here be cause of test failure.
- }
- try {
- FileSystem fs = cluster.getFileSystem();
- if (fs != null) {
- LOG.info("Shutting down FileSystem");
- fs.close();
- }
- FileSystem.closeAll();
- } catch (IOException e) {
- LOG.error("error closing file system", e);
- }
- }
- }
-
- public void assertByteEquals(byte[] expected,
- byte[] actual) {
- if (Bytes.compareTo(expected, actual) != 0) {
- throw new AssertionFailedError("expected:<" +
- Bytes.toString(expected) + "> but was:<" +
- Bytes.toString(actual) + ">");
- }
- }
-
- public static void assertEquals(byte[] expected,
- byte[] actual) {
- if (Bytes.compareTo(expected, actual) != 0) {
- throw new AssertionFailedError("expected:<" +
- Bytes.toStringBinary(expected) + "> but was:<" +
- Bytes.toStringBinary(actual) + ">");
- }
- }
-
-}
Index: src/test/org/apache/hama/MiniBSPCluster.java
===================================================================
--- src/test/org/apache/hama/MiniBSPCluster.java (revision 1152138)
+++ src/test/org/apache/hama/MiniBSPCluster.java (working copy)
@@ -1,252 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.IOException;
-
-import java.net.ServerSocket;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import static java.util.concurrent.TimeUnit.*;
-
-import static junit.framework.Assert.*;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hama.bsp.BSPMaster;
-import org.apache.hama.bsp.GroomServer;
-import org.apache.hama.HamaConfiguration;
-
-
-public class MiniBSPCluster {
-
- public static final Log LOG = LogFactory.getLog(MiniBSPCluster.class);
-
- private ScheduledExecutorService scheduler;
-
- private HamaConfiguration configuration;
- private BSPMasterRunner master;
- private List groomServerList =
- new CopyOnWriteArrayList();
- private int grooms;
-
- public class BSPMasterRunner implements Runnable{
- BSPMaster bspm;
- HamaConfiguration conf;
-
- public BSPMasterRunner(HamaConfiguration conf){
- this.conf = conf;
- if(null == this.conf)
- throw new NullPointerException("No Configuration for BSPMaster.");
- }
-
- public void run(){
- try{
- LOG.info("Starting BSP Master.");
- this.bspm = BSPMaster.startMaster(this.conf);
- this.bspm.offerService();
- }catch(IOException ioe){
- LOG.error("Fail to startup BSP Master.", ioe);
- }catch(InterruptedException ie){
- LOG.error("BSP Master fails in offerService().", ie);
- Thread.currentThread().interrupt();
- }
- }
-
- public void shutdown(){
- if(null != this.bspm) this.bspm.shutdown();
- }
-
- public boolean isRunning(){
- if(null == this.bspm) return false;
-
- if(this.bspm.currentState().equals(BSPMaster.State.RUNNING)){
- return true;
- }
- return false;
- }
-
- public BSPMaster getMaster(){
- return this.bspm;
- }
- }
-
- public class GroomServerRunner implements Runnable{
- GroomServer gs;
- HamaConfiguration conf;
-
- public GroomServerRunner(HamaConfiguration conf){
- this.conf = conf;
- }
-
- public void run(){
- try{
- this.gs = GroomServer.constructGroomServer(GroomServer.class, conf);
- GroomServer.startGroomServer(this.gs).join();
- }catch(InterruptedException ie){
- LOG.error("Fail to start GroomServer. ", ie);
- Thread.currentThread().interrupt();
- }
- }
-
- public void shutdown(){
- try{
- if(null != this.gs) this.gs.shutdown();
- }catch(IOException ioe){
- LOG.info("Fail to shutdown GroomServer.", ioe);
- }
- }
-
- public boolean isRunning(){
- if(null == this.gs) return false;
- return this.gs.isRunning();
- }
-
- public GroomServer getGroomServer(){
- return this.gs;
- }
- }
-
- public MiniBSPCluster(HamaConfiguration conf, int groomServers) {
- this.configuration = conf;
- this.grooms = groomServers;
- if(1 > this.grooms) {
- this.grooms = 2;
- }
- LOG.info("Groom server number "+this.grooms);
- int threadpool = conf.getInt("bsp.test.threadpool", 10);
- LOG.info("Thread pool value "+threadpool);
- scheduler = Executors.newScheduledThreadPool(threadpool);
- }
-
- public void startBSPCluster(){
- startMaster();
- startGroomServers();
- }
-
- public void shutdownBSPCluster(){
- if(null != this.master && this.master.isRunning())
- this.master.shutdown();
- if(0 < groomServerList.size()){
- for(GroomServerRunner groom: groomServerList){
- if(groom.isRunning()) groom.shutdown();
- }
- }
- }
-
-
- public void startMaster(){
- if(null == this.scheduler)
- throw new NullPointerException("No ScheduledExecutorService exists.");
- this.master = new BSPMasterRunner(this.configuration);
- scheduler.schedule(this.master, 0, SECONDS);
- }
-
- public void startGroomServers(){
- if(null == this.scheduler)
- throw new NullPointerException("No ScheduledExecutorService exists.");
- if(null == this.master)
- throw new NullPointerException("No BSPMaster exists.");
- int cnt=0;
- while(!this.master.isRunning()){
- LOG.info("Waiting BSPMaster up.");
- try{
- Thread.sleep(1000);
- cnt++;
- if(100 < cnt){
- fail("Fail to launch BSPMaster.");
- }
- }catch(InterruptedException ie){
- LOG.error("Fail to check BSP Master's state.", ie);
- Thread.currentThread().interrupt();
- }
- }
- for(int i=0; i < this.grooms; i++){
- HamaConfiguration c = new HamaConfiguration(this.configuration);
- randomPort(c);
- GroomServerRunner gsr = new GroomServerRunner(c);
- groomServerList.add(gsr);
- scheduler.schedule(gsr, 0, SECONDS);
- cnt = 0;
- while(!gsr.isRunning()){
- LOG.info("Waitin for GroomServer up.");
- try{
- Thread.sleep(1000);
- cnt++;
- if(10 < cnt){
- fail("Fail to launch groom server.");
- }
- }catch(InterruptedException ie){
- LOG.error("Fail to check Groom Server's state.", ie);
- Thread.currentThread().interrupt();
- }
- }
- }
-
- }
-
- private void randomPort(HamaConfiguration conf){
- try{
- ServerSocket skt = new ServerSocket(0);
- int p = skt.getLocalPort();
- skt.close();
- conf.set(Constants.PEER_PORT, new Integer(p).toString());
- conf.setInt(Constants.GROOM_RPC_PORT, p+100);
- }catch(IOException ioe){
- LOG.error("Can not find a free port for BSPPeer.", ioe);
- }
- }
-
- public void shutdown() {
- shutdownBSPCluster();
- scheduler.shutdown();
- }
-
- public List getGroomServerThreads() {
- List list = new ArrayList();
- for(GroomServerRunner gsr: groomServerList){
- list.add(new Thread(gsr));
- }
- return list;
- }
-
- public Thread getMaster() {
- return new Thread(this.master);
- }
-
- public List getGroomServers(){
- List list = new ArrayList();
- for(GroomServerRunner gsr: groomServerList){
- list.add(gsr.getGroomServer());
- }
- return list;
- }
-
- public BSPMaster getBSPMaster(){
- if(null != this.master)
- return this.master.getMaster();
- return null;
- }
-
- public ScheduledExecutorService getScheduler(){
- return this.scheduler;
- }
-}
Index: src/test/org/apache/hama/zookeeper/TestZKTools.java
===================================================================
--- src/test/org/apache/hama/zookeeper/TestZKTools.java (revision 1152138)
+++ src/test/org/apache/hama/zookeeper/TestZKTools.java (working copy)
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.zookeeper;
-
-import org.apache.hama.Constants;
-import org.apache.hama.HamaConfiguration;
-
-import junit.framework.TestCase;
-
-public class TestZKTools extends TestCase {
-
- public void testZKProps() {
- HamaConfiguration conf = new HamaConfiguration();
- conf.set(Constants.ZOOKEEPER_QUORUM, "test.com:123");
- conf.set(Constants.ZOOKEEPER_CLIENT_PORT, "2222");
-
- assertEquals("test.com:2222", QuorumPeer.getZKQuorumServersString(conf));
- }
-}
Index: src/test/org/apache/hama/HamaClusterTestCase.java
===================================================================
--- src/test/org/apache/hama/HamaClusterTestCase.java (revision 1152138)
+++ src/test/org/apache/hama/HamaClusterTestCase.java (working copy)
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.File;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-
-public abstract class HamaClusterTestCase extends HamaTestCase {
- public static final Log LOG = LogFactory.getLog(HamaClusterTestCase.class);
- protected MiniDFSCluster dfsCluster;
- protected MiniBSPCluster bspCluster;
- protected MiniZooKeeperCluster zooKeeperCluster;
- protected boolean startDfs;
- protected int numOfGroom = 2;
-
- /** default constructor */
- public HamaClusterTestCase() {
- this(false);
- }
-
- public HamaClusterTestCase(boolean startDfs) {
- super();
- this.startDfs = startDfs;
- }
-
- /**
- * Actually start the MiniBSP instance.
- */
- protected void hamaClusterSetup() throws Exception {
- File testDir = new File(getUnitTestdir(getName()).toString());
-
- // Note that this is done before we create the MiniHamaCluster because we
- // need to edit the config to add the ZooKeeper servers.
- this.zooKeeperCluster = new MiniZooKeeperCluster();
- int clientPort = this.zooKeeperCluster.startup(testDir);
- conf.set("hama.zookeeper.property.clientPort", Integer.toString(clientPort));
- bspCluster = new MiniBSPCluster(this.conf, numOfGroom);
- bspCluster.startBSPCluster();
- }
-
- @Override
- protected void setUp() throws Exception {
- try {
- if (this.startDfs) {
- // This spews a bunch of warnings about missing scheme. TODO: fix.
- this.dfsCluster = new MiniDFSCluster(0, this.conf, 2, true, true, true,
- null, null, null, null);
-
- // mangle the conf so that the fs parameter points to the minidfs we
- // just started up
- FileSystem filesystem = dfsCluster.getFileSystem();
- conf.set("fs.defaultFS", filesystem.getUri().toString());
- Path parentdir = filesystem.getHomeDirectory();
-
- filesystem.mkdirs(parentdir);
- }
-
- // do the super setup now. if we had done it first, then we would have
- // gotten our conf all mangled and a local fs started up.
- super.setUp();
-
- // start the instance
- hamaClusterSetup();
- } catch (Exception e) {
- if (zooKeeperCluster != null) {
- zooKeeperCluster.shutdown();
- }
- if (dfsCluster != null) {
- shutdownDfs(dfsCluster);
- }
- throw e;
- }
- }
-
- @Override
- protected void tearDown() throws Exception {
- super.tearDown();
- try {
- if (startDfs) {
- shutdownDfs(dfsCluster);
- }
- bspCluster.shutdown();
- } catch (Exception e) {
- LOG.error(e);
- }
- }
-}
Index: src/test/org/apache/hama/bsp/TestBSPMessageBundle.java
===================================================================
--- src/test/org/apache/hama/bsp/TestBSPMessageBundle.java (revision 1152138)
+++ src/test/org/apache/hama/bsp/TestBSPMessageBundle.java (working copy)
@@ -1,81 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import junit.framework.TestCase;
-
-public class TestBSPMessageBundle extends TestCase {
-
- public void testEmpty() throws IOException {
- BSPMessageBundle bundle = new BSPMessageBundle();
- // Serialize it.
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- bundle.write(new DataOutputStream(baos));
- baos.close();
- // Deserialize it.
- BSPMessageBundle readBundle = new BSPMessageBundle();
- readBundle.readFields(new DataInputStream(new ByteArrayInputStream(baos
- .toByteArray())));
- assertEquals(0, readBundle.getMessages().size());
- }
-
- public void testSerializationDeserialization() throws IOException {
- BSPMessageBundle bundle = new BSPMessageBundle();
- ByteMessage[] testMessages = new ByteMessage[16];
- for (int i = 0; i < testMessages.length; ++i) {
- // Create a one byte tag containing the number of the message.
- byte[] tag = new byte[1];
- tag[0] = (byte) i;
- // Create a four bytes data part containing serialized number of the
- // message.
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- baos.write(i);
- baos.close();
- byte[] data = baos.toByteArray();
- testMessages[i] = new ByteMessage(tag, data);
- bundle.addMessage(testMessages[i]);
- }
- // Serialize it.
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- bundle.write(new DataOutputStream(baos));
- baos.close();
- // Deserialize it.
- BSPMessageBundle readBundle = new BSPMessageBundle();
- readBundle.readFields(new DataInputStream(new ByteArrayInputStream(baos
- .toByteArray())));
- // Check contents.
- int messageNumber = 0;
- for (BSPMessage message : readBundle.getMessages()) {
- ByteMessage byteMessage = (ByteMessage) message;
- assertTrue(Arrays.equals(testMessages[messageNumber].getTag(),
- byteMessage.getTag()));
- assertTrue(Arrays.equals(testMessages[messageNumber].getData(),
- byteMessage.getData()));
- ++messageNumber;
- }
- assertEquals(testMessages.length, messageNumber);
- }
-}
Index: src/test/org/apache/hama/bsp/TestClusterStatus.java
===================================================================
--- src/test/org/apache/hama/bsp/TestClusterStatus.java (revision 1152138)
+++ src/test/org/apache/hama/bsp/TestClusterStatus.java (working copy)
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hama.bsp.BSPMaster;
-import org.apache.hama.bsp.ClusterStatus;
-
-public class TestClusterStatus extends TestCase {
- Random rnd = new Random();
-
- protected void setUp() throws Exception {
- super.setUp();
- }
-
- public final void testWriteAndReadFields() throws IOException {
- DataOutputBuffer out = new DataOutputBuffer();
- DataInputBuffer in = new DataInputBuffer();
-
- ClusterStatus status1;
- Map grooms = new HashMap();
-
- for (int i = 0; i < 10; i++) {
- int num = rnd.nextInt();
- String groomName = "groom_" + num;
- String peerName = "peerhost:" + num;
- grooms.put(groomName, peerName);
- }
-
- int tasks = rnd.nextInt(100);
- int maxTasks = rnd.nextInt(100);
- BSPMaster.State state = BSPMaster.State.RUNNING;
-
- status1 = new ClusterStatus(grooms, tasks, maxTasks, state);
- status1.write(out);
-
- in.reset(out.getData(), out.getLength());
-
- ClusterStatus status2 = new ClusterStatus();
- status2.readFields(in);
-
- Map grooms_s = new HashMap(status1
- .getActiveGroomNames());
- Map grooms_o = new HashMap(status2
- .getActiveGroomNames());
-
- assertEquals(status1.getGroomServers(), status2.getGroomServers());
-
- assertTrue(grooms_s.entrySet().containsAll(grooms_o.entrySet()));
- assertTrue(grooms_o.entrySet().containsAll(grooms_s.entrySet()));
-
- assertEquals(status1.getTasks(), status2.getTasks());
- assertEquals(status1.getMaxTasks(), status2.getMaxTasks());
- }
-}
Index: src/test/org/apache/hama/bsp/TestMessages.java
===================================================================
--- src/test/org/apache/hama/bsp/TestMessages.java (revision 1152138)
+++ src/test/org/apache/hama/bsp/TestMessages.java (working copy)
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import junit.framework.TestCase;
-
-import org.apache.hama.util.Bytes;
-
-public class TestMessages extends TestCase {
-
- public void testByteMessage() {
- int dataSize = (int) (Runtime.getRuntime().maxMemory() * 0.60);
- ByteMessage msg = new ByteMessage(Bytes.toBytes("tag"), new byte[dataSize]);
- assertEquals(msg.getData().length, dataSize);
- msg = null;
-
- byte[] dummyData = new byte[1024];
- ByteMessage msg2 = new ByteMessage(Bytes.tail(dummyData, 128), dummyData);
- assertEquals(
- Bytes.compareTo(msg2.getTag(), 0, 128, msg2.getData(),
- msg2.getData().length - 128, 128), 0);
- }
-}
Index: src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java
===================================================================
--- src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java (revision 1152138)
+++ src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java (working copy)
@@ -1,94 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hama.Constants;
-import org.apache.hama.HamaCluster;
-import org.apache.hama.HamaConfiguration;
-
-public class TestBSPMasterGroomServer extends HamaCluster {
-
- private static Log LOG = LogFactory.getLog(TestBSPMasterGroomServer.class);
- private static String TMP_OUTPUT = "/tmp/test-example/";
- private HamaConfiguration configuration;
- private String TEST_JOB = "src/test/testjar/testjob.jar";
-
- public TestBSPMasterGroomServer() {
- configuration = new HamaConfiguration();
- configuration.set("bsp.master.address", "localhost");
- assertEquals("Make sure master addr is set to localhost:", "localhost",
- configuration.get("bsp.master.address"));
- configuration.setStrings("bsp.local.dir", "/tmp/hama-test");
- configuration.set(Constants.ZOOKEEPER_QUORUM, "localhost");
- configuration.setInt(Constants.ZOOKEEPER_CLIENT_PORT, 21810);
- }
-
- public void setUp() throws Exception {
- super.setUp();
- }
-
- public void testSubmitJob() throws Exception {
- BSPJob bsp = new BSPJob(configuration);
- bsp.setJobName("Test Serialize Printing");
- bsp.setBspClass(testjar.ClassSerializePrinting.HelloBSP.class);
- bsp.setJar(TEST_JOB);
-
- // Set the task size as a number of GroomServer
- BSPJobClient jobClient = new BSPJobClient(configuration);
- configuration.setInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 600);
- ClusterStatus cluster = jobClient.getClusterStatus(false);
- assertEquals(this.numOfGroom, cluster.getMaxTasks());
-
- // TODO test the multi-tasks
- bsp.setNumBspTask(1);
-
- FileSystem fileSys = FileSystem.get(conf);
-
- if (bsp.waitForCompletion(true)) {
- checkOutput(fileSys, cluster, conf);
- }
- LOG.info("Client finishes execution job.");
- }
-
- private static void checkOutput(FileSystem fileSys, ClusterStatus cluster,
- HamaConfiguration conf) throws Exception {
- for (int i = 0; i < 1; i++) { // TODO test the multi-tasks
- SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, new Path(
- TMP_OUTPUT + i), conf);
- LongWritable timestamp = new LongWritable();
- Text message = new Text();
- reader.next(timestamp, message);
- assertTrue("Check if `Hello BSP' gets printed.", message.toString()
- .indexOf("Hello BSP from") >= 0);
- reader.close();
- }
- }
-
- public void tearDown() throws Exception {
- super.tearDown();
- }
-}
Index: src/test/org/apache/hama/HamaCluster.java
===================================================================
--- src/test/org/apache/hama/HamaCluster.java (revision 1152138)
+++ src/test/org/apache/hama/HamaCluster.java (working copy)
@@ -1,47 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * Forming up the miniDfs and miniZooKeeper
- */
-public abstract class HamaCluster extends HamaClusterTestCase {
- public static final Log LOG = LogFactory.getLog(HamaCluster.class);
- protected final static HamaConfiguration conf = new HamaConfiguration();
-
- public HamaCluster(){
- super();
- }
-
- public HamaCluster(boolean startDfs) {
- super(startDfs);
- }
-
- protected void setUp() throws Exception {
- super.setUp();
- }
-
- protected static HamaConfiguration getConf() {
- return conf;
- }
-}
Index: src/test/org/apache/hama/ipc/TestRPC.java
===================================================================
--- src/test/org/apache/hama/ipc/TestRPC.java (revision 1152138)
+++ src/test/org/apache/hama/ipc/TestRPC.java (working copy)
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.VersionedProtocol;
-
-import junit.framework.TestCase;
-
-public class TestRPC extends TestCase {
- private static final int PORT = 1234;
- private static final String ADDRESS = "0.0.0.0";
-
- public static final Log LOG = LogFactory
- .getLog("org.apache.hadoop.ipc.TestRPC");
-
- private static Configuration conf = new Configuration();
-
- public TestRPC(String name) {
- super(name);
- }
-
- public interface TestProtocol extends VersionedProtocol {
- public static final long versionID = 1L;
-
- void ping() throws IOException;
-
- String echo(String value) throws IOException;
-
- String[] echo(String[] value) throws IOException;
-
- Writable echo(Writable value) throws IOException;
-
- int add(int v1, int v2) throws IOException;
-
- int add(int[] values) throws IOException;
-
- int error() throws IOException;
-
- void testServerGet() throws IOException;
- }
-
- public class TestImpl implements TestProtocol {
-
- public long getProtocolVersion(String protocol, long clientVersion) {
- return TestProtocol.versionID;
- }
-
- public void ping() {
- }
-
- public String echo(String value) throws IOException {
- return value;
- }
-
- public String[] echo(String[] values) throws IOException {
- return values;
- }
-
- public Writable echo(Writable writable) {
- return writable;
- }
-
- public int add(int v1, int v2) {
- return v1 + v2;
- }
-
- public int add(int[] values) {
- int sum = 0;
- for (int i = 0; i < values.length; i++) {
- sum += values[i];
- }
- return sum;
- }
-
- public int error() throws IOException {
- throw new IOException("bobo");
- }
-
- public void testServerGet() throws IOException {
- if (!(Server.get() instanceof RPC.Server)) {
- throw new IOException("Server.get() failed");
- }
- }
-
- }
-
- public void testCalls() throws Exception {
- Server server = RPC.getServer(new TestImpl(), ADDRESS, PORT, conf);
- server.start();
-
- InetSocketAddress addr = new InetSocketAddress(PORT);
- TestProtocol proxy = (TestProtocol) RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
-
- proxy.ping();
-
- String stringResult = proxy.echo("foo");
- assertEquals(stringResult, "foo");
-
- stringResult = proxy.echo((String) null);
- assertEquals(stringResult, null);
-
- String[] stringResults = proxy.echo(new String[] { "foo", "bar" });
- assertTrue(Arrays.equals(stringResults, new String[] { "foo", "bar" }));
-
- stringResults = proxy.echo((String[]) null);
- assertTrue(Arrays.equals(stringResults, null));
-
- int intResult = proxy.add(1, 2);
- assertEquals(intResult, 3);
-
- intResult = proxy.add(new int[] { 1, 2 });
- assertEquals(intResult, 3);
-
- boolean caught = false;
- try {
- proxy.error();
- } catch (IOException e) {
- LOG.debug("Caught " + e);
- caught = true;
- }
- assertTrue(caught);
-
- proxy.testServerGet();
-
- // try some multi-calls
- Method echo = TestProtocol.class.getMethod("echo",
- new Class[] { String.class });
- String[] strings = (String[]) RPC.call(echo, new String[][] { { "a" },
- { "b" } }, new InetSocketAddress[] { addr, addr }, null, conf);
- assertTrue(Arrays.equals(strings, new String[] { "a", "b" }));
-
- Method ping = TestProtocol.class.getMethod("ping", new Class[] {});
- Object[] voids = (Object[]) RPC.call(ping, new Object[][] { {}, {} },
- new InetSocketAddress[] { addr, addr }, null, conf);
- assertEquals(voids, null);
-
- server.stop();
- }
-
- public static void main(String[] args) throws Exception {
-
- new TestRPC("test").testCalls();
-
- }
-
-}
Index: src/test/org/apache/hama/ipc/TestIPC.java
===================================================================
--- src/test/org/apache/hama/ipc/TestIPC.java (revision 1152138)
+++ src/test/org/apache/hama/ipc/TestIPC.java (working copy)
@@ -1,232 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Random;
-
-import junit.framework.TestCase;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.StringUtils;
-
-public class TestIPC extends TestCase {
- public static final Log LOG = LogFactory.getLog(TestIPC.class);
-
- final private static Configuration conf = new Configuration();
- final static private int PING_INTERVAL = 1000;
-
- static {
- Client.setPingInterval(conf, PING_INTERVAL);
- }
-
- public TestIPC(String name) {
- super(name);
- }
-
- private static final Random RANDOM = new Random();
-
- private static final String ADDRESS = "0.0.0.0";
-
- private static class TestServer extends Server {
- private boolean sleep;
-
- public TestServer(int handlerCount, boolean sleep) throws IOException {
- super(ADDRESS, 0, LongWritable.class, handlerCount, conf);
- this.sleep = sleep;
- }
-
- @Override
- public Writable call(Class> protocol, Writable param, long receiveTime)
- throws IOException {
- if (sleep) {
- try {
- Thread.sleep(RANDOM.nextInt(2 * PING_INTERVAL)); // sleep a bit
- } catch (InterruptedException e) {
- }
- }
- return param; // echo param as result
- }
- }
-
- private static class SerialCaller extends Thread {
- private Client client;
- private InetSocketAddress server;
- private int count;
- private boolean failed;
-
- public SerialCaller(Client client, InetSocketAddress server, int count) {
- this.client = client;
- this.server = server;
- this.count = count;
- }
-
- public void run() {
- for (int i = 0; i < count; i++) {
- try {
- LongWritable param = new LongWritable(RANDOM.nextLong());
- LongWritable value = (LongWritable) client.call(param, server, null,
- null);
- if (!param.equals(value)) {
- LOG.fatal("Call failed!");
- failed = true;
- break;
- }
- } catch (Exception e) {
- LOG.fatal("Caught: " + StringUtils.stringifyException(e));
- failed = true;
- }
- }
- }
- }
-
- private static class ParallelCaller extends Thread {
- private Client client;
- private int count;
- private InetSocketAddress[] addresses;
- private boolean failed;
-
- public ParallelCaller(Client client, InetSocketAddress[] addresses,
- int count) {
- this.client = client;
- this.addresses = addresses;
- this.count = count;
- }
-
- public void run() {
- for (int i = 0; i < count; i++) {
- try {
- Writable[] params = new Writable[addresses.length];
- for (int j = 0; j < addresses.length; j++)
- params[j] = new LongWritable(RANDOM.nextLong());
- Writable[] values = client.call(params, addresses, null, null);
- for (int j = 0; j < addresses.length; j++) {
- if (!params[j].equals(values[j])) {
- LOG.fatal("Call failed!");
- failed = true;
- break;
- }
- }
- } catch (Exception e) {
- LOG.fatal("Caught: " + StringUtils.stringifyException(e));
- failed = true;
- }
- }
- }
- }
-
- public void testSerial() throws Exception {
- testSerial(3, false, 2, 5, 100);
- }
-
- public void testSerial(int handlerCount, boolean handlerSleep,
- int clientCount, int callerCount, int callCount) throws Exception {
- Server server = new TestServer(handlerCount, handlerSleep);
- InetSocketAddress addr = NetUtils.getConnectAddress(server);
- server.start();
-
- Client[] clients = new Client[clientCount];
- for (int i = 0; i < clientCount; i++) {
- clients[i] = new Client(LongWritable.class, conf);
- }
-
- SerialCaller[] callers = new SerialCaller[callerCount];
- for (int i = 0; i < callerCount; i++) {
- callers[i] = new SerialCaller(clients[i % clientCount], addr, callCount);
- callers[i].start();
- }
- for (int i = 0; i < callerCount; i++) {
- callers[i].join();
- assertFalse(callers[i].failed);
- }
- for (int i = 0; i < clientCount; i++) {
- clients[i].stop();
- }
- server.stop();
- }
-
- public void testParallel() throws Exception {
- testParallel(10, false, 2, 4, 2, 4, 100);
- }
-
- public void testParallel(int handlerCount, boolean handlerSleep,
- int serverCount, int addressCount, int clientCount, int callerCount,
- int callCount) throws Exception {
- Server[] servers = new Server[serverCount];
- for (int i = 0; i < serverCount; i++) {
- servers[i] = new TestServer(handlerCount, handlerSleep);
- servers[i].start();
- }
-
- InetSocketAddress[] addresses = new InetSocketAddress[addressCount];
- for (int i = 0; i < addressCount; i++) {
- addresses[i] = NetUtils.getConnectAddress(servers[i % serverCount]);
- }
-
- Client[] clients = new Client[clientCount];
- for (int i = 0; i < clientCount; i++) {
- clients[i] = new Client(LongWritable.class, conf);
- }
-
- ParallelCaller[] callers = new ParallelCaller[callerCount];
- for (int i = 0; i < callerCount; i++) {
- callers[i] = new ParallelCaller(clients[i % clientCount], addresses,
- callCount);
- callers[i].start();
- }
- for (int i = 0; i < callerCount; i++) {
- callers[i].join();
- assertFalse(callers[i].failed);
- }
- for (int i = 0; i < clientCount; i++) {
- clients[i].stop();
- }
- for (int i = 0; i < serverCount; i++) {
- servers[i].stop();
- }
- }
-
- public void testStandAloneClient() throws Exception {
- testParallel(10, false, 2, 4, 2, 4, 100);
- Client client = new Client(LongWritable.class, conf);
- InetSocketAddress address = new InetSocketAddress("127.0.0.1", 10);
- try {
- client.call(new LongWritable(RANDOM.nextLong()), address, null, null);
- fail("Expected an exception to have been thrown");
- } catch (IOException e) {
- String message = e.getMessage();
- String addressText = address.toString();
- assertTrue("Did not find " + addressText + " in " + message, message
- .contains(addressText));
- Throwable cause = e.getCause();
- assertNotNull("No nested exception in " + e, cause);
- String causeText = cause.getMessage();
- assertTrue("Did not find " + causeText + " in " + message, message
- .contains(causeText));
- }
- }
-
-}
Index: src/test/org/apache/hama/util/TestNumeric.java
===================================================================
--- src/test/org/apache/hama/util/TestNumeric.java (revision 1152138)
+++ src/test/org/apache/hama/util/TestNumeric.java (working copy)
@@ -1,34 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import junit.framework.TestCase;
-
-public class TestNumeric extends TestCase {
- final static int TEST_INT = 3;
- final static double TEST_DOUBLE = 0.4;
-
- /**
- * Double conversion test
- */
- public void testDouble() {
- assertEquals(Bytes.toDouble(Bytes.toBytes(TEST_DOUBLE)), TEST_DOUBLE);
- }
-}
Index: src/test/org/apache/hama/util/TestBytes.java
===================================================================
--- src/test/org/apache/hama/util/TestBytes.java (revision 1152138)
+++ src/test/org/apache/hama/util/TestBytes.java (working copy)
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import junit.framework.TestCase;
-
-public class TestBytes extends TestCase {
- public void testNullHashCode() {
- byte [] b = null;
- Exception ee = null;
- try {
- Bytes.hashCode(b);
- } catch (Exception e) {
- ee = e;
- }
- assertNotNull(ee);
- }
-
- public void testSplit() throws Exception {
- byte [] lowest = Bytes.toBytes("AAA");
- byte [] middle = Bytes.toBytes("CCC");
- byte [] highest = Bytes.toBytes("EEE");
- byte [][] parts = Bytes.split(lowest, highest, 1);
- for (int i = 0; i < parts.length; i++) {
- System.out.println(Bytes.toString(parts[i]));
- }
- assertEquals(3, parts.length);
- assertTrue(Bytes.equals(parts[1], middle));
- // Now divide into three parts. Change highest so split is even.
- highest = Bytes.toBytes("DDD");
- parts = Bytes.split(lowest, highest, 2);
- for (int i = 0; i < parts.length; i++) {
- System.out.println(Bytes.toString(parts[i]));
- }
- assertEquals(4, parts.length);
- // Assert that 3rd part is 'CCC'.
- assertTrue(Bytes.equals(parts[2], middle));
- }
-
- public void testSplit2() throws Exception {
- // More split tests.
- byte [] lowest = Bytes.toBytes("http://A");
- byte [] highest = Bytes.toBytes("http://z");
- byte [] middle = Bytes.toBytes("http://]");
- byte [][] parts = Bytes.split(lowest, highest, 1);
- for (int i = 0; i < parts.length; i++) {
- System.out.println(Bytes.toString(parts[i]));
- }
- assertEquals(3, parts.length);
- assertTrue(Bytes.equals(parts[1], middle));
- }
-
- public void testToLong() throws Exception {
- long [] longs = {-1l, 123l, 122232323232l};
- for (int i = 0; i < longs.length; i++) {
- byte [] b = Bytes.toBytes(longs[i]);
- assertEquals(longs[i], Bytes.toLong(b));
- }
- }
-
- public void testToFloat() throws Exception {
- float [] floats = {-1f, 123.123f, Float.MAX_VALUE};
- for (int i = 0; i < floats.length; i++) {
- byte [] b = Bytes.toBytes(floats[i]);
- assertEquals(floats[i], Bytes.toFloat(b));
- }
- }
-
- public void testToDouble() throws Exception {
- double [] doubles = {Double.MIN_VALUE, Double.MAX_VALUE};
- for (int i = 0; i < doubles.length; i++) {
- byte [] b = Bytes.toBytes(doubles[i]);
- assertEquals(doubles[i], Bytes.toDouble(b));
- }
- }
-
- public void testBinarySearch() throws Exception {
- byte [][] arr = {
- {1},
- {3},
- {5},
- {7},
- {9},
- {11},
- {13},
- {15},
- };
- byte [] key1 = {3,1};
- byte [] key2 = {4,9};
- byte [] key2_2 = {4};
- byte [] key3 = {5,11};
-
- assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(-(2+1), Arrays.binarySearch(arr, key2_2,
- Bytes.BYTES_COMPARATOR));
- assertEquals(-(2+1), Bytes.binarySearch(arr, key2, 0, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1,
- Bytes.BYTES_RAWCOMPARATOR));
- }
-
- public void testIncrementBytes() throws IOException {
-
- assertTrue(checkTestIncrementBytes(10, 1));
- assertTrue(checkTestIncrementBytes(12, 123435445));
- assertTrue(checkTestIncrementBytes(124634654, 1));
- assertTrue(checkTestIncrementBytes(10005460, 5005645));
- assertTrue(checkTestIncrementBytes(1, -1));
- assertTrue(checkTestIncrementBytes(10, -1));
- assertTrue(checkTestIncrementBytes(10, -5));
- assertTrue(checkTestIncrementBytes(1005435000, -5));
- assertTrue(checkTestIncrementBytes(10, -43657655));
- assertTrue(checkTestIncrementBytes(-1, 1));
- assertTrue(checkTestIncrementBytes(-26, 5034520));
- assertTrue(checkTestIncrementBytes(-10657200, 5));
- assertTrue(checkTestIncrementBytes(-12343250, 45376475));
- assertTrue(checkTestIncrementBytes(-10, -5));
- assertTrue(checkTestIncrementBytes(-12343250, -5));
- assertTrue(checkTestIncrementBytes(-12, -34565445));
- assertTrue(checkTestIncrementBytes(-1546543452, -34565445));
- }
-
- private static boolean checkTestIncrementBytes(long val, long amount)
- throws IOException {
- byte[] value = Bytes.toBytes(val);
- byte [] testValue = {-1, -1, -1, -1, -1, -1, -1, -1};
- if (value[0] > 0) {
- testValue = new byte[Bytes.SIZEOF_LONG];
- }
- System.arraycopy(value, 0, testValue, testValue.length - value.length,
- value.length);
-
- long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount));
-
- return (Bytes.toLong(testValue) + amount) == incrementResult;
- }
-}
Index: src/test/org/apache/hama/util/TestRandomVariable.java
===================================================================
--- src/test/org/apache/hama/util/TestRandomVariable.java (revision 1152138)
+++ src/test/org/apache/hama/util/TestRandomVariable.java (working copy)
@@ -1,74 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import org.apache.log4j.Logger;
-
-import junit.framework.TestCase;
-
-/**
- * Random variable generation test
- */
-public class TestRandomVariable extends TestCase {
- static final Logger LOG = Logger.getLogger(TestRandomVariable.class);
- final static int COUNT = 50;
-
- /**
- * Random object test
- *
- * @throws Exception
- */
- public void testRand() throws Exception {
- for (int i = 0; i < COUNT; i++) {
- double result = RandomVariable.rand();
- assertTrue(result >= 0.0d && result <= 1.0);
- }
- }
-
- /**
- * Random integer test
- *
- * @throws Exception
- */
- public void testRandInt() throws Exception {
- final int min = 122;
- final int max = 561;
-
- for (int i = 0; i < COUNT; i++) {
- int result = RandomVariable.randInt(min, max);
- assertTrue(result >= min && result <= max);
- }
- }
-
- /**
- * Uniform test
- *
- * @throws Exception
- */
- public void testUniform() throws Exception {
- final double min = 1.0d;
- final double max = 3.0d;
-
- for (int i = 0; i < COUNT; i++) {
- double result = RandomVariable.uniform(min, max);
- assertTrue(result >= min && result <= max);
- }
- }
-}
Index: src/test/testjar/testjob.jar
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: src/test/testjar/ClassSerializePrinting.java
===================================================================
--- src/test/testjar/ClassSerializePrinting.java (revision 1152138)
+++ src/test/testjar/ClassSerializePrinting.java (working copy)
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package testjar;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.Text;
-import org.apache.hama.bsp.BSP;
-import org.apache.hama.bsp.BSPPeer;
-import org.apache.zookeeper.KeeperException;
-
-public class ClassSerializePrinting {
- private static String TMP_OUTPUT = "/tmp/test-example/";
-
- public static class HelloBSP extends BSP {
- public static final Log LOG = LogFactory.getLog(HelloBSP.class);
- private Configuration conf;
- private final static int PRINT_INTERVAL = 1000;
- private FileSystem fileSys;
- private int num;
-
- public void bsp(BSPPeer bspPeer) throws IOException,
- KeeperException, InterruptedException {
-
- int i = 0;
- for (String otherPeer : bspPeer.getAllPeerNames()) {
- String peerName = bspPeer.getPeerName();
- if (peerName.equals(otherPeer)) {
- writeLogToFile(peerName, i);
- }
-
- Thread.sleep(PRINT_INTERVAL);
- bspPeer.sync();
- i++;
- }
- }
-
- private void writeLogToFile(String string, int i) throws IOException {
- SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf,
- new Path(TMP_OUTPUT + i), LongWritable.class, Text.class,
- CompressionType.NONE);
- writer.append(new LongWritable(System.currentTimeMillis()), new Text(
- "Hello BSP from " + (i + 1) + " of " + num + ": " + string));
- writer.close();
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- num = Integer.parseInt(conf.get("bsp.peers.num"));
- try {
- fileSys = FileSystem.get(conf);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- }
-
-}
Index: src/java/org/apache/hama/http/HttpServer.java
===================================================================
--- src/java/org/apache/hama/http/HttpServer.java (revision 1152138)
+++ src/java/org/apache/hama/http/HttpServer.java (working copy)
@@ -1,501 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.http;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.BindException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.log.LogLevel;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.mortbay.jetty.Connector;
-import org.mortbay.jetty.Handler;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
-import org.mortbay.jetty.nio.SelectChannelConnector;
-import org.mortbay.jetty.security.SslSocketConnector;
-import org.mortbay.jetty.servlet.Context;
-import org.mortbay.jetty.servlet.DefaultServlet;
-import org.mortbay.jetty.servlet.FilterHolder;
-import org.mortbay.jetty.servlet.FilterMapping;
-import org.mortbay.jetty.servlet.ServletHandler;
-import org.mortbay.jetty.servlet.ServletHolder;
-import org.mortbay.jetty.webapp.WebAppContext;
-import org.mortbay.thread.BoundedThreadPool;
-import org.mortbay.util.MultiException;
-
-/**
- * Create a Jetty embedded server to answer http requests. The primary goal is
- * to serve up status information for the server. There are three contexts:
- * "/logs/" -> points to the log directory "/static/" -> points to common static
- * files (src/webapps/static) "/" -> the jsp server code from
- * (src/webapps/)
- */
-public class HttpServer {
- public static final Log LOG = LogFactory.getLog(HttpServer.class);
-
- static final String FILTER_INITIALIZER_PROPERTY = "hama.http.filter.initializers";
-
- protected final Server webServer;
- protected final Connector listener;
- protected final WebAppContext webAppContext;
- protected final boolean findPort;
- protected final Map defaultContexts = new HashMap();
- protected final List filterNames = new ArrayList();
- private static final int MAX_RETRIES = 10;
-
- /** Same as this(name, bindAddress, port, findPort, null); */
- public HttpServer(String name, String bindAddress, int port, boolean findPort)
- throws IOException {
- this(name, bindAddress, port, findPort, new Configuration());
- }
-
- /**
- * Create a status server on the given port. The jsp scripts are taken from
- * src/webapps/.
- *
- * @param name The name of the server
- * @param port The port to use on the server
- * @param findPort whether the server should start at the given port and
- * increment by 1 until it finds a free port.
- * @param conf Configuration
- */
- public HttpServer(String name, String bindAddress, int port,
- boolean findPort, Configuration conf) throws IOException {
- webServer = new Server();
- this.findPort = findPort;
-
- listener = createBaseListener(conf);
- listener.setHost(bindAddress);
- listener.setPort(port);
- webServer.addConnector(listener);
-
- webServer.setThreadPool(new BoundedThreadPool());
-
- final String appDir = getWebAppsPath();
- ContextHandlerCollection contexts = new ContextHandlerCollection();
- webServer.setHandler(contexts);
- webAppContext = new WebAppContext();
-
- System.setProperty("java.naming.factory.initial",
- "org.mortbay.naming.InitialContextFactory");
- System.setProperty("java.naming.factory.url.pkgs", "org.mortbay.naming");
-
- webAppContext.setContextPath("/");
- webAppContext.setWar(appDir + "/" + name);
- webServer.addHandler(webAppContext);
-
- addDefaultApps(contexts, appDir);
-
- addDefaultServlets();
- }
-
- /**
- * Create a required listener for the Jetty instance listening on the port
- * provided. This wrapper and all subclasses must create at least one
- * listener.
- */
- protected Connector createBaseListener(Configuration conf) throws IOException {
- SelectChannelConnector ret = new SelectChannelConnector();
- ret.setLowResourceMaxIdleTime(10000);
- ret.setAcceptQueueSize(128);
- ret.setResolveNames(false);
- ret.setUseDirectBuffers(false);
- return ret;
- }
-
- /**
- * Add default apps.
- *
- * @param appDir The application directory
- * @throws IOException
- */
- protected void addDefaultApps(ContextHandlerCollection parent,
- final String appDir) throws IOException {
- // set up the context for "/logs/" if "hama.log.dir" property is defined.
- String logDir = System.getProperty("hama.log.dir");
- if (logDir != null) {
- Context logContext = new Context(parent, "/logs");
- logContext.setResourceBase(logDir);
- logContext.addServlet(DefaultServlet.class, "/");
- defaultContexts.put(logContext, true);
- }
- // set up the context for "/static/*"
- Context staticContext = new Context(parent, "/static");
- staticContext.setResourceBase(appDir + "/static");
- staticContext.addServlet(DefaultServlet.class, "/*");
- defaultContexts.put(staticContext, true);
- }
-
- /**
- * Add default servlets.
- */
- protected void addDefaultServlets() {
- // set up default servlets
- addServlet("stacks", "/stacks", StackServlet.class);
- addServlet("logLevel", "/logLevel", LogLevel.Servlet.class);
- }
-
- public void addContext(Context ctxt, boolean isFiltered) throws IOException {
- webServer.addHandler(ctxt);
- defaultContexts.put(ctxt, isFiltered);
- }
-
- /**
- * Add a context
- *
- * @param pathSpec The path spec for the context
- * @param dir The directory containing the context
- * @param isFiltered if true, the servlet is added to the filter path mapping
- * @throws IOException
- */
- protected void addContext(String pathSpec, String dir, boolean isFiltered)
- throws IOException {
- if (0 == webServer.getHandlers().length) {
- throw new RuntimeException("Couldn't find handler");
- }
- WebAppContext webAppCtx = new WebAppContext();
- webAppCtx.setContextPath(pathSpec);
- webAppCtx.setWar(dir);
- addContext(webAppCtx, true);
- }
-
- /**
- * Set a value in the webapp context. These values are available to the jsp
- * pages as "application.getAttribute(name)".
- *
- * @param name The name of the attribute
- * @param value The value of the attribute
- */
- public void setAttribute(String name, Object value) {
- webAppContext.setAttribute(name, value);
- }
-
- /**
- * Add a servlet in the server.
- *
- * @param name The name of the servlet (can be passed as null)
- * @param pathSpec The path spec for the servlet
- * @param clazz The servlet class
- */
- public void addServlet(String name, String pathSpec,
- Class extends HttpServlet> clazz) {
- addInternalServlet(name, pathSpec, clazz);
- addFilterPathMapping(pathSpec, webAppContext);
- }
-
- /**
- * Add an internal servlet in the server.
- *
- * @param name The name of the servlet (can be passed as null)
- * @param pathSpec The path spec for the servlet
- * @param clazz The servlet class
- * @deprecated this is a temporary method
- */
- @Deprecated
- public void addInternalServlet(String name, String pathSpec,
- Class extends HttpServlet> clazz) {
- ServletHolder holder = new ServletHolder(clazz);
- if (name != null) {
- holder.setName(name);
- }
- webAppContext.addServlet(holder, pathSpec);
- }
-
- /**
- * Define a filter for a context and set up default url mappings.
- */
- protected void defineFilter(Context ctx, String name, String classname,
- Map parameters, String[] urls) {
-
- FilterHolder holder = new FilterHolder();
- holder.setName(name);
- holder.setClassName(classname);
- holder.setInitParameters(parameters);
- FilterMapping fmap = new FilterMapping();
- fmap.setPathSpecs(urls);
- fmap.setDispatches(Handler.ALL);
- fmap.setFilterName(name);
- ServletHandler handler = ctx.getServletHandler();
- handler.addFilter(holder, fmap);
- }
-
- /**
- * Add the path spec to the filter path mapping.
- *
- * @param pathSpec The path spec
- * @param webAppCtx The WebApplicationContext to add to
- */
- protected void addFilterPathMapping(String pathSpec, Context webAppCtx) {
- ServletHandler handler = webAppCtx.getServletHandler();
- for (String name : filterNames) {
- FilterMapping fmap = new FilterMapping();
- fmap.setPathSpec(pathSpec);
- fmap.setFilterName(name);
- fmap.setDispatches(Handler.ALL);
- handler.addFilterMapping(fmap);
- }
- }
-
- /**
- * Get the value in the webapp context.
- *
- * @param name The name of the attribute
- * @return The value of the attribute
- */
- public Object getAttribute(String name) {
- return webAppContext.getAttribute(name);
- }
-
- /**
- * Get the pathname to the webapps files.
- *
- * @return the pathname as a URL
- * @throws IOException if 'webapps' directory cannot be found on CLASSPATH.
- */
- protected String getWebAppsPath() throws IOException {
- // URL url = BSPMaster.class.getClassLoader().getResource("webapps");
- // if (url == null)
- // throw new IOException("webapps not found in CLASSPATH");
- // return url.toString();
- return "src/webapps";
- }
-
- /**
- * Get the port that the server is on
- *
- * @return the port
- */
- public int getPort() {
- return webServer.getConnectors()[0].getLocalPort();
- }
-
- /**
- * Set the min, max number of worker threads (simultaneous connections).
- */
- public void setThreads(int min, int max) {
- BoundedThreadPool pool = (BoundedThreadPool) webServer.getThreadPool();
- pool.setMinThreads(min);
- pool.setMaxThreads(max);
- }
-
- /**
- * Configure an ssl listener on the server.
- *
- * @param addr address to listen on
- * @param keystore location of the keystore
- * @param storPass password for the keystore
- * @param keyPass password for the key
- * @deprecated Use
- * {@link #addSslListener(InetSocketAddress, Configuration, boolean)}
- */
- @Deprecated
- public void addSslListener(InetSocketAddress addr, String keystore,
- String storPass, String keyPass) throws IOException {
- if (webServer.isStarted()) {
- throw new IOException("Failed to add ssl listener");
- }
- SslSocketConnector sslListener = new SslSocketConnector();
- sslListener.setHost(addr.getHostName());
- sslListener.setPort(addr.getPort());
- sslListener.setKeystore(keystore);
- sslListener.setPassword(storPass);
- sslListener.setKeyPassword(keyPass);
- webServer.addConnector(sslListener);
- }
-
- /**
- * Configure an ssl listener on the server.
- *
- * @param addr address to listen on
- * @param sslConf conf to retrieve ssl options
- * @param needClientAuth whether client authentication is required
- */
- public void addSslListener(InetSocketAddress addr, Configuration sslConf,
- boolean needClientAuth) throws IOException {
- if (webServer.isStarted()) {
- throw new IOException("Failed to add ssl listener");
- }
- if (needClientAuth) {
- // setting up SSL truststore for authenticating clients
- System.setProperty("javax.net.ssl.trustStore", sslConf.get(
- "ssl.server.truststore.location", ""));
- System.setProperty("javax.net.ssl.trustStorePassword", sslConf.get(
- "ssl.server.truststore.password", ""));
- System.setProperty("javax.net.ssl.trustStoreType", sslConf.get(
- "ssl.server.truststore.type", "jks"));
- }
- SslSocketConnector sslListener = new SslSocketConnector();
- sslListener.setHost(addr.getHostName());
- sslListener.setPort(addr.getPort());
- sslListener.setKeystore(sslConf.get("ssl.server.keystore.location"));
- sslListener.setPassword(sslConf.get("ssl.server.keystore.password", ""));
- sslListener.setKeyPassword(sslConf.get("ssl.server.keystore.keypassword",
- ""));
- sslListener.setKeystoreType(sslConf.get("ssl.server.keystore.type", "jks"));
- sslListener.setNeedClientAuth(needClientAuth);
- webServer.addConnector(sslListener);
- }
-
- /**
- * Start the server. Does not wait for the server to start.
- */
- public void start() throws IOException {
- try {
- int port = 0;
- int oriPort = listener.getPort(); // The original requested port
- while (true) {
- try {
- port = webServer.getConnectors()[0].getLocalPort();
- LOG.info("Port returned by webServer.getConnectors()[0]."
- + "getLocalPort() before open() is " + port
- + ". Opening the listener on " + oriPort);
- listener.open();
- port = listener.getLocalPort();
- LOG.info("listener.getLocalPort() returned "
- + listener.getLocalPort()
- + " webServer.getConnectors()[0].getLocalPort() returned "
- + webServer.getConnectors()[0].getLocalPort());
- // Workaround to handle the problem reported in HADOOP-4744
- if (port < 0) {
- Thread.sleep(100);
- int numRetries = 1;
- while (port < 0) {
- LOG.warn("listener.getLocalPort returned " + port);
- if (numRetries++ > MAX_RETRIES) {
- throw new Exception(" listener.getLocalPort is returning "
- + "less than 0 even after " + numRetries + " resets");
- }
- for (int i = 0; i < 2; i++) {
- LOG.info("Retrying listener.getLocalPort()");
- port = listener.getLocalPort();
- if (port > 0) {
- break;
- }
- Thread.sleep(200);
- }
- if (port > 0) {
- break;
- }
- LOG.info("Bouncing the listener");
- listener.close();
- Thread.sleep(1000);
- listener.setPort(oriPort == 0 ? 0 : (oriPort += 1));
- listener.open();
- Thread.sleep(100);
- port = listener.getLocalPort();
- }
- } // Workaround end
- LOG.info("Jetty bound to port " + port);
- webServer.start();
- // Workaround for HADOOP-6386
- port = listener.getLocalPort();
- if (port < 0) {
- LOG.warn("Bounds port is " + port + " after webserver start");
- for (int i = 0; i < MAX_RETRIES / 2; i++) {
- try {
- webServer.stop();
- } catch (Exception e) {
- LOG.warn("Can't stop web-server", e);
- }
- Thread.sleep(1000);
-
- listener.setPort(oriPort == 0 ? 0 : (oriPort += 1));
- listener.open();
- Thread.sleep(100);
- webServer.start();
- LOG.info(i + "attempts to restart webserver");
- port = listener.getLocalPort();
- if (port > 0)
- break;
- }
- if (port < 0)
- throw new Exception("listener.getLocalPort() is returning "
- + "less than 0 even after " + MAX_RETRIES + " resets");
- }
- // End of HADOOP-6386 workaround
- break;
- } catch (IOException ex) {
- // if this is a bind exception,
- // then try the next port number.
- if (ex instanceof BindException) {
- if (!findPort) {
- throw (BindException) ex;
- }
- } else {
- LOG.info("HttpServer.start() threw a non Bind IOException");
- throw ex;
- }
- } catch (MultiException ex) {
- LOG.info("HttpServer.start() threw a MultiException");
- throw ex;
- }
- listener.setPort((oriPort += 1));
- }
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException("Problem starting http server", e);
- }
- }
-
- /**
- * stop the server
- */
- public void stop() throws Exception {
- listener.close();
- webServer.stop();
- }
-
- public void join() throws InterruptedException {
- webServer.join();
- }
-
- /**
- * A very simple servlet to serve up a text representation of the current
- * stack traces. It both returns the stacks to the caller and logs them.
- * Currently the stack traces are done sequentially rather than exactly the
- * same data.
- */
- public static class StackServlet extends HttpServlet {
- private static final long serialVersionUID = -6284183679759467039L;
-
- @Override
- public void doGet(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
-
- PrintWriter out = new PrintWriter(response.getOutputStream());
- ReflectionUtils.printThreadInfo(out, "");
- out.close();
- ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
- }
- }
-}
Index: src/java/org/apache/hama/http/package.html
===================================================================
--- src/java/org/apache/hama/http/package.html (revision 1152138)
+++ src/java/org/apache/hama/http/package.html (working copy)
@@ -1,23 +0,0 @@
-
-
-
-
-
-Contains the administrative web interfaces.
-
-
Index: src/java/org/apache/hama/MiniZooKeeperCluster.java
===================================================================
--- src/java/org/apache/hama/MiniZooKeeperCluster.java (revision 1152138)
+++ src/java/org/apache/hama/MiniZooKeeperCluster.java (working copy)
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Reader;
-import java.net.BindException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.zookeeper.server.NIOServerCnxn;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.apache.zookeeper.server.persistence.FileTxnLog;
-
-/**
- * This class starts and runs the MiniZookeeperCluster.
- */
-public class MiniZooKeeperCluster {
- private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class);
-
- private static final int TICK_TIME = 2000;
- private static final int CONNECTION_TIMEOUT = 30000;
-
- private boolean started;
- private int clientPort = 21810; // use non-standard port
-
- private NIOServerCnxn.Factory standaloneServerFactory;
- private int tickTime = 0;
-
- /** Create mini Zookeeper cluster. */
- public MiniZooKeeperCluster() {
- this.started = false;
- }
-
- public void setClientPort(int clientPort) {
- this.clientPort = clientPort;
- }
-
- public void setTickTime(int tickTime) {
- this.tickTime = tickTime;
- }
-
- private static void setupTestEnv() {
- // during the tests we run with 100K prealloc in the logs.
- // on windows systems prealloc of 64M was seen to take ~15seconds
- // resulting in test failure (client timeout on first session).
- // set env and directly in order to handle static init/gc issues
- System.setProperty("zookeeper.preAllocSize", "100");
- FileTxnLog.setPreallocSize(100);
- }
-
- /**
- * @param baseDir
- * @return ClientPort server bound to.
- * @throws IOException
- * @throws InterruptedException
- */
- public int startup(File baseDir) throws IOException,
- InterruptedException {
-
- setupTestEnv();
-
- shutdown();
-
- File dir = new File(baseDir, "zookeeper").getAbsoluteFile();
- recreateDir(dir);
-
- int tickTimeToUse;
- if (this.tickTime > 0) {
- tickTimeToUse = this.tickTime;
- } else {
- tickTimeToUse = TICK_TIME;
- }
- ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
- while (true) {
- try {
- standaloneServerFactory =
- new NIOServerCnxn.Factory(new InetSocketAddress(clientPort));
- } catch (BindException e) {
- LOG.info("Faild binding ZK Server to client port: " + clientPort);
- //this port is already in use. try to use another
- clientPort++;
- continue;
- }
- break;
- }
- standaloneServerFactory.startup(server);
-
- if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for startup of standalone server");
- }
-
- started = true;
-
- return clientPort;
- }
-
- private void recreateDir(File dir) throws IOException {
- if (dir.exists()) {
- FileUtil.fullyDelete(dir);
- }
- try {
- dir.mkdirs();
- } catch (SecurityException e) {
- throw new IOException("creating dir: " + dir, e);
- }
- }
-
- /**
- * @throws IOException
- */
- public void shutdown() throws IOException {
- if (!started) {
- return;
- }
-
- standaloneServerFactory.shutdown();
- if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for shutdown of standalone server");
- }
-
- started = false;
- }
-
- // XXX: From o.a.zk.t.ClientBase
- private static boolean waitForServerDown(int port, long timeout) {
- long start = System.currentTimeMillis();
- while (true) {
- try {
- Socket sock = new Socket("localhost", port);
- try {
- OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes());
- outstream.flush();
- } finally {
- sock.close();
- }
- } catch (IOException e) {
- return true;
- }
-
- if (System.currentTimeMillis() > start + timeout) {
- break;
- }
- try {
- Thread.sleep(250);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- return false;
- }
-
- // XXX: From o.a.zk.t.ClientBase
- private static boolean waitForServerUp(int port, long timeout) {
- long start = System.currentTimeMillis();
- while (true) {
- try {
- Socket sock = new Socket("localhost", port);
- BufferedReader reader = null;
- try {
- OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes());
- outstream.flush();
-
- Reader isr = new InputStreamReader(sock.getInputStream());
- reader = new BufferedReader(isr);
- String line = reader.readLine();
- if (line != null && line.startsWith("Zookeeper version:")) {
- return true;
- }
- } finally {
- sock.close();
- if (reader != null) {
- reader.close();
- }
- }
- } catch (IOException e) {
- // ignore as this is expected
- LOG.info("server localhost:" + port + " not up " + e);
- }
-
- if (System.currentTimeMillis() > start + timeout) {
- break;
- }
- try {
- Thread.sleep(250);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- return false;
- }
-}
Index: src/java/org/apache/hama/zookeeper/QuorumPeer.java
===================================================================
--- src/java/org/apache/hama/zookeeper/QuorumPeer.java (revision 1152138)
+++ src/java/org/apache/hama/zookeeper/QuorumPeer.java (working copy)
@@ -1,354 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.zookeeper;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.Properties;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.DNS;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hama.Constants;
-import org.apache.hama.HamaConfiguration;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.apache.zookeeper.server.quorum.QuorumPeerMain;
-
-/**
- * This class starts and runs the QuorumPeers
- */
-public class QuorumPeer implements Constants {
- private static final Log LOG = LogFactory.getLog(QuorumPeer.class);
-
- private static final String VARIABLE_START = "${";
- private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
- private static final String VARIABLE_END = "}";
- private static final int VARIABLE_END_LENGTH = VARIABLE_END.length();
-
- private static final String ZK_CFG_PROPERTY = "hama.zookeeper.property.";
- private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
-
- /**
- * Parse ZooKeeper configuration from Hama XML config and run a QuorumPeer.
- * @param baseConf Hadoop Configuration.
- */
- public static void run(Configuration baseConf) {
- Configuration conf = new HamaConfiguration(baseConf);
- try {
- Properties zkProperties = makeZKProps(conf);
- writeMyID(zkProperties);
- QuorumPeerConfig zkConfig = new QuorumPeerConfig();
- zkConfig.parseProperties(zkProperties);
- runZKServer(zkConfig);
- } catch (Exception e) {
- LOG.error("Exception during ZooKeeper startup - exiting...",e);
- System.exit(-1);
- }
- }
-
- private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException {
- if (zkConfig.isDistributed()) {
- QuorumPeerMain qp = new QuorumPeerMain();
- qp.runFromConfig(zkConfig);
- } else {
- ZooKeeperServerMain zk = new ZooKeeperServerMain();
- ServerConfig serverConfig = new ServerConfig();
- serverConfig.readFrom(zkConfig);
- zk.runFromConfig(serverConfig);
- }
- }
-
- private static boolean addressIsLocalHost(String address) {
- return address.equals("localhost") || address.equals("127.0.0.1");
- }
-
- private static void writeMyID(Properties properties) throws IOException {
- long myId = -1;
-
- Configuration conf = new HamaConfiguration();
- String myAddress = DNS.getDefaultHost(
- conf.get("hama.zookeeper.dns.interface","default"),
- conf.get("hama.zookeeper.dns.nameserver","default"));
-
- List ips = new ArrayList();
-
- // Add what could be the best (configured) match
- ips.add(myAddress.contains(".") ?
- myAddress :
- StringUtils.simpleHostname(myAddress));
-
- // For all nics get all hostnames and IPs
- Enumeration> nics = NetworkInterface.getNetworkInterfaces();
- while(nics.hasMoreElements()) {
- Enumeration> rawAdrs =
- ((NetworkInterface)nics.nextElement()).getInetAddresses();
- while(rawAdrs.hasMoreElements()) {
- InetAddress inet = (InetAddress) rawAdrs.nextElement();
- ips.add(StringUtils.simpleHostname(inet.getHostName()));
- ips.add(inet.getHostAddress());
- }
- }
-
- for (Entry entry : properties.entrySet()) {
- String key = entry.getKey().toString().trim();
- String value = entry.getValue().toString().trim();
-
- if (key.startsWith("server.")) {
- int dot = key.indexOf('.');
- long id = Long.parseLong(key.substring(dot + 1));
- String[] parts = value.split(":");
- String address = parts[0];
- if (addressIsLocalHost(address) || ips.contains(address)) {
- myId = id;
- break;
- }
- }
- }
-
- if (myId == -1) {
- throw new IOException("Could not find my address: " + myAddress +
- " in list of ZooKeeper quorum servers");
- }
-
- String dataDirStr = properties.get("dataDir").toString().trim();
- File dataDir = new File(dataDirStr);
- if (!dataDir.isDirectory()) {
- if (!dataDir.mkdirs()) {
- throw new IOException("Unable to create data dir " + dataDir);
- }
- }
-
- File myIdFile = new File(dataDir, "myid");
- PrintWriter w = new PrintWriter(myIdFile);
- w.println(myId);
- w.close();
- }
-
- /**
- * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
- * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
- * the corresponding config options from the Hama XML configs and generate
- * the appropriate ZooKeeper properties.
- * @param conf Configuration to read from.
- * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
- */
- public static Properties makeZKProps(Configuration conf) {
- // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
- // it and grab its configuration properties.
- ClassLoader cl = QuorumPeer.class.getClassLoader();
- InputStream inputStream = cl.getResourceAsStream(ZOOKEEPER_CONFIG_NAME);
- if (inputStream != null) {
- try {
- return parseZooCfg(conf, inputStream);
- } catch (IOException e) {
- LOG.warn("Cannot read " + ZOOKEEPER_CONFIG_NAME +
- ", loading from XML files", e);
- }
- }
-
- // Otherwise, use the configuration options from Hama's XML files.
- Properties zkProperties = new Properties();
-
- // Set the max session timeout from the provided client-side timeout
- zkProperties.setProperty("maxSessionTimeout",
- conf.get(Constants.ZOOKEEPER_SESSION_TIMEOUT, "1200000"));
-
- // Directly map all of the hama.zookeeper.property.KEY properties.
- for (Entry entry : conf) {
- String key = entry.getKey();
- if (key.startsWith(ZK_CFG_PROPERTY)) {
- String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE);
- String value = entry.getValue();
- // If the value has variables substitutions, need to do a get.
- if (value.contains(VARIABLE_START)) {
- value = conf.get(key);
- }
- zkProperties.put(zkKey, value);
- }
- }
-
- // If clientPort is not set, assign the default
- if (zkProperties.getProperty(ZOOKEEPER_CLIENT_PORT) == null) {
- zkProperties.put(ZOOKEEPER_CLIENT_PORT, DEFAULT_ZOOKEEPER_CLIENT_PORT);
- }
-
- // Create the server.X properties.
- int peerPort = conf.getInt("hama.zookeeper.peerport", 2888);
- int leaderPort = conf.getInt("hama.zookeeper.leaderport", 3888);
-
- String[] serverHosts = conf.getStrings(ZOOKEEPER_QUORUM, "localhost");
- for (int i = 0; i < serverHosts.length; ++i) {
- String serverHost = serverHosts[i];
- String address = serverHost + ":" + peerPort + ":" + leaderPort;
- String key = "server." + i;
- zkProperties.put(key, address);
- }
-
- return zkProperties;
- }
-
- /**
- * Parse ZooKeeper's zoo.cfg, injecting Hama Configuration variables in.
- * This method is used for testing so we can pass our own InputStream.
- * @param conf Configuration to use for injecting variables.
- * @param inputStream InputStream to read from.
- * @return Properties parsed from config stream with variables substituted.
- * @throws IOException if anything goes wrong parsing config
- */
- public static Properties parseZooCfg(Configuration conf,
- InputStream inputStream) throws IOException {
- Properties properties = new Properties();
- try {
- properties.load(inputStream);
- } catch (IOException e) {
- String msg = "fail to read properties from " + ZOOKEEPER_CONFIG_NAME;
- LOG.fatal(msg);
- throw new IOException(msg, e);
- }
- for (Entry entry : properties.entrySet()) {
- String value = entry.getValue().toString().trim();
- String key = entry.getKey().toString().trim();
- StringBuilder newValue = new StringBuilder();
- int varStart = value.indexOf(VARIABLE_START);
- int varEnd = 0;
- while (varStart != -1) {
- varEnd = value.indexOf(VARIABLE_END, varStart);
- if (varEnd == -1) {
- String msg = "variable at " + varStart + " has no end marker";
- LOG.fatal(msg);
- throw new IOException(msg);
- }
- String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd);
-
- String substituteValue = System.getProperty(variable);
- if (substituteValue == null) {
- substituteValue = conf.get(variable);
- }
- if (substituteValue == null) {
- String msg = "variable " + variable + " not set in system property "
- + "or hama configs";
- LOG.fatal(msg);
- throw new IOException(msg);
- }
-
- newValue.append(substituteValue);
-
- varEnd += VARIABLE_END_LENGTH;
- varStart = value.indexOf(VARIABLE_START, varEnd);
- }
- // Special case for 'hama.cluster.distributed' property being 'true'
- if (key.startsWith("server.")) {
- if (conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) &&
- value.startsWith("localhost")) {
- String msg = "The server in zoo.cfg cannot be set to localhost " +
- "in a fully-distributed setup because it won't be reachable. " +
- "See \"Getting Started\" for more information.";
- LOG.fatal(msg);
- throw new IOException(msg);
- }
- }
- newValue.append(value.substring(varEnd));
- properties.setProperty(key, newValue.toString());
- }
- return properties;
- }
-
- /**
- * Return the ZK Quorum servers string given zk properties returned by
- * makeZKProps
- * @param properties
- * @return Quorum servers String
- */
- public static String getZKQuorumServersString(Properties properties) {
- String clientPort = null;
- List servers = new ArrayList();
-
- // The clientPort option may come after the server.X hosts, so we need to
- // grab everything and then create the final host:port comma separated list.
- boolean anyValid = false;
- for (Entry property : properties.entrySet()) {
- String key = property.getKey().toString().trim();
- String value = property.getValue().toString().trim();
- if (key.equals("clientPort")) {
- clientPort = value;
- }
- else if (key.startsWith("server.")) {
- String host = value.substring(0, value.indexOf(':'));
- servers.add(host);
- try {
- //noinspection ResultOfMethodCallIgnored
- InetAddress.getByName(host);
- anyValid = true;
- } catch (UnknownHostException e) {
- LOG.warn(StringUtils.stringifyException(e));
- }
- }
- }
-
- if (!anyValid) {
- LOG.error("no valid quorum servers found in " + Constants.ZOOKEEPER_CONFIG_NAME);
- return null;
- }
-
- if (clientPort == null) {
- LOG.error("no clientPort found in " + Constants.ZOOKEEPER_CONFIG_NAME);
- return null;
- }
-
- if (servers.isEmpty()) {
- LOG.fatal("No server.X lines found in conf/zoo.cfg. Hama must have a " +
- "ZooKeeper cluster configured for its operation.");
- return null;
- }
-
- StringBuilder hostPortBuilder = new StringBuilder();
- for (int i = 0; i < servers.size(); ++i) {
- String host = servers.get(i);
- if (i > 0) {
- hostPortBuilder.append(',');
- }
- hostPortBuilder.append(host);
- hostPortBuilder.append(':');
- hostPortBuilder.append(clientPort);
- }
-
- return hostPortBuilder.toString();
- }
-
- /**
- * Return the ZK Quorum servers string given the specified configuration.
- * @param conf
- * @return Quorum servers
- */
- public static String getZKQuorumServersString(Configuration conf) {
- return getZKQuorumServersString(makeZKProps(conf));
- }
-}
Index: src/java/org/apache/hama/zookeeper/ZKServerTool.java
===================================================================
--- src/java/org/apache/hama/zookeeper/ZKServerTool.java (revision 1152138)
+++ src/java/org/apache/hama/zookeeper/ZKServerTool.java (working copy)
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.zookeeper;
-
-import java.util.Properties;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hama.HamaConfiguration;
-
-/**
- * A tool class for Zookeeper use.
- */
-public class ZKServerTool {
-
- /**
- * Run the tool.
- *
- * @param args Command line arguments. First arg is path to zookeepers file.
- */
- public static void main(String args[]) {
- Configuration conf = new HamaConfiguration();
- // Note that we do not simply grab the property ZOOKEEPER_QUORUM from
- // the HamaConfiguration because the user may be using a zoo.cfg file.
- Properties zkProps = QuorumPeer.makeZKProps(conf);
- for (Entry entry : zkProps.entrySet()) {
- String key = entry.getKey().toString().trim();
- String value = entry.getValue().toString().trim();
- if (key.startsWith("server.")) {
- String[] parts = value.split(":");
- String host = parts[0];
- System.out.println(host);
- }
- }
- }
-}
Index: src/java/org/apache/hama/zookeeper/package.html
===================================================================
--- src/java/org/apache/hama/zookeeper/package.html (revision 1152138)
+++ src/java/org/apache/hama/zookeeper/package.html (working copy)
@@ -1,23 +0,0 @@
-
-
-
-
-
-Tools to start the ZooKeeper server and quorum peers.
-
-
Index: src/java/org/apache/hama/GroomServerRunner.java
===================================================================
--- src/java/org/apache/hama/GroomServerRunner.java (revision 1152138)
+++ src/java/org/apache/hama/GroomServerRunner.java (working copy)
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hama.bsp.GroomServer;
-
-/**
- * This class starts and runs the GroomServer.
- */
-public class GroomServerRunner extends Configured implements Tool {
-
- public static final Log LOG = LogFactory.getLog(GroomServerRunner.class);
-
- @Override
- public int run(String[] args) throws Exception {
- StringUtils.startupShutdownMessage(GroomServer.class, args, LOG);
-
- if (args.length != 0) {
- System.out.println("usage: GroomServerRunner");
- System.exit(-1);
- }
-
- try {
- Configuration conf = new HamaConfiguration(getConf());
- GroomServer groom = GroomServer.constructGroomServer(GroomServer.class,
- conf);
- GroomServer.startGroomServer(groom).join();
- } catch (Throwable e) {
- LOG.fatal(StringUtils.stringifyException(e));
- return -1;
- }
-
- return 0;
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new GroomServerRunner(), args);
- System.exit(exitCode);
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSPMaster.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPMaster.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPMaster.java (working copy)
@@ -1,730 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.net.InetSocketAddress;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.http.HttpServer;
-import org.apache.hama.ipc.JobSubmissionProtocol;
-import org.apache.hama.ipc.MasterProtocol;
-import org.apache.hama.ipc.GroomProtocol;
-
-/**
- * BSPMaster is responsible to control all the groom servers and to manage bsp
- * jobs.
- */
-public class BSPMaster implements JobSubmissionProtocol, MasterProtocol,
- GroomServerManager {
- public static final Log LOG = LogFactory.getLog(BSPMaster.class);
-
- private HamaConfiguration conf;
-
- /**
- * Constants for BSPMaster's status.
- */
- public static enum State {
- INITIALIZING, RUNNING
- }
-
- private static final int FS_ACCESS_RETRY_PERIOD = 10000;
- public static final long GROOMSERVER_EXPIRY_INTERVAL = 10 * 60 * 1000;
- static long JOBINIT_SLEEP_INTERVAL = 2000;
-
- // States
- State state = State.INITIALIZING;
-
- // Attributes
- String masterIdentifier;
- // private Server interServer;
- private Server masterServer;
-
- // host and port
- private String host;
- private int port;
-
- // startTime
- private long startTime;
-
- // HTTP server
- private HttpServer infoServer;
- private int infoPort;
-
- // Filesystem
- static final String SUBDIR = "bspMaster";
- FileSystem fs = null;
- Path systemDir = null;
- // system directories are world-wide readable and owner readable
- final static FsPermission SYSTEM_DIR_PERMISSION = FsPermission
- .createImmutable((short) 0733); // rwx-wx-wx
- // system files should have 700 permission
- final static FsPermission SYSTEM_FILE_PERMISSION = FsPermission
- .createImmutable((short) 0700); // rwx------
-
- // Jobs' Meta Data
- private Integer nextJobId = Integer.valueOf(1);
- private int totalSubmissions = 0; // how many jobs has been submitted by
- // clients
- private int totalTasks = 0; // currnetly running tasks
- private int totalTaskCapacity; // max tasks that groom server can run
-
- private Map jobs = new TreeMap();
- private TaskScheduler taskScheduler;
-
- // GroomServers cache
- protected ConcurrentMap groomServers = new ConcurrentHashMap();
-
- private Instructor instructor;
-
- private final List jobInProgressListeners = new CopyOnWriteArrayList();
-
- private class ReportGroomStatusHandler implements DirectiveHandler {
-
- public void handle(Directive directive) throws DirectiveException {
- // update GroomServerStatus held in the groomServers cache.
- GroomServerStatus groomStatus = ((ReportGroomStatusDirective) directive)
- .getStatus();
- // groomServers cache contains groom server status reported back
- if (groomServers.containsKey(groomStatus)) {
- GroomServerStatus tmpStatus = null;
- for (GroomServerStatus old : groomServers.keySet()) {
- if (old.equals(groomStatus)) {
- tmpStatus = groomStatus;
- updateGroomServersKey(old, tmpStatus);
- break;
- }
- }// for
- if (null != tmpStatus) {
- List tlist = tmpStatus.getTaskReports();
- for (TaskStatus ts : tlist) {
- JobInProgress jip = whichJob(ts.getJobId());
- TaskInProgress tip = jip.findTaskInProgress(((TaskAttemptID) ts
- .getTaskId()).getTaskID());
-
- if (ts.getRunState() == TaskStatus.State.SUCCEEDED) {
- jip.completedTask(tip, ts);
- } else if (ts.getRunState() == TaskStatus.State.RUNNING) {
- // do nothing
- } else if (ts.getRunState() == TaskStatus.State.FAILED) {
- jip.status.setRunState(JobStatus.FAILED);
- jip.failedTask(tip, ts);
- }
- if (jip.getStatus().getRunState() == JobStatus.SUCCEEDED) {
- for (JobInProgressListener listener : jobInProgressListeners) {
- try {
- listener.jobRemoved(jip);
- } catch (IOException ioe) {
- LOG.error("Fail to alter scheduler a job is moved.", ioe);
- }
- }
- } else if (jip.getStatus().getRunState() == JobStatus.RUNNING) {
- jip.getStatus().setprogress(ts.getSuperstepCount());
- } else if (jip.getStatus().getRunState() == JobStatus.KILLED) {
- GroomProtocol worker = findGroomServer(tmpStatus);
- Directive d1 = new DispatchTasksDirective(
- currentGroomServerPeers(),
- new GroomServerAction[] { new KillTaskAction(ts.getTaskId()) });
- try {
- worker.dispatch(d1);
- } catch (IOException ioe) {
- throw new DirectiveException("Error when dispatching kill task"
- + " action.", ioe);
- }
- }
- }
- } else {
- throw new RuntimeException("BSPMaster contains GroomServerSatus, "
- + "but fail to retrieve it.");
- }
- } else {
- throw new RuntimeException("GroomServer not found."
- + groomStatus.getGroomName());
- }
- }
- }
-
- private class Instructor extends Thread {
- private final BlockingQueue buffer = new LinkedBlockingQueue();
- private final ConcurrentMap, DirectiveHandler> handlers = new ConcurrentHashMap, DirectiveHandler>();
-
- public void bind(Class extends Directive> instruction,
- DirectiveHandler handler) {
- handlers.putIfAbsent(instruction, handler);
- }
-
- public void put(Directive directive) {
- try {
- buffer.put(directive);
- } catch (InterruptedException ie) {
- LOG.error("Fail to put directive into queue.", ie);
- }
- }
-
- public void run() {
- while (true) {
- try {
- Directive directive = this.buffer.take();
- if (directive instanceof ReportGroomStatusDirective) {
- ((DirectiveHandler) handlers.get(ReportGroomStatusDirective.class))
- .handle(directive);
- } else {
- throw new RuntimeException("Directive is not supported."
- + directive);
- }
- } catch (InterruptedException ie) {
- LOG.error("Unable to retrieve directive from the queue.", ie);
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- LOG.error("Fail to execute directive command.", e);
- }
- }
- }
- }
-
- /**
- * Start the BSPMaster process, listen on the indicated hostname/port
- */
- public BSPMaster(HamaConfiguration conf) throws IOException,
- InterruptedException {
- this(conf, generateNewIdentifier());
- }
-
- BSPMaster(HamaConfiguration conf, String identifier) throws IOException,
- InterruptedException {
- this.conf = conf;
- this.masterIdentifier = identifier;
-
- // Create the scheduler and init scheduler services
- Class extends TaskScheduler> schedulerClass = conf.getClass(
- "bsp.master.taskscheduler", SimpleTaskScheduler.class,
- TaskScheduler.class);
- this.taskScheduler = (TaskScheduler) ReflectionUtils.newInstance(
- schedulerClass, conf);
-
- host = getAddress(conf).getHostName();
- port = getAddress(conf).getPort();
- LOG.info("RPC BSPMaster: host " + host + " port " + port);
-
- startTime = System.currentTimeMillis();
- this.masterServer = RPC.getServer(this, host, port, conf);
-
- infoPort = conf.getInt("bsp.http.infoserver.port", 40013);
-
- infoServer = new HttpServer("bspmaster", host, infoPort, true, conf);
- infoServer.setAttribute("bsp.master", this);
-
- // starting webserver
- infoServer.start();
-
- while (!Thread.currentThread().isInterrupted()) {
- try {
- if (fs == null) {
- fs = FileSystem.get(conf);
- }
- // clean up the system dir, which will only work if hdfs is out of
- // safe mode
- if (systemDir == null) {
- systemDir = new Path(getSystemDir());
- }
-
- LOG.info("Cleaning up the system directory");
- LOG.info(systemDir);
- fs.delete(systemDir, true);
- if (FileSystem.mkdirs(fs, systemDir, new FsPermission(
- SYSTEM_DIR_PERMISSION))) {
- break;
- }
- LOG.error("Mkdirs failed to create " + systemDir);
- LOG.info(SUBDIR);
-
- } catch (AccessControlException ace) {
- LOG.warn("Failed to operate on bsp.system.dir (" + systemDir
- + ") because of permissions.");
- LOG.warn("Manually delete the bsp.system.dir (" + systemDir
- + ") and then start the BSPMaster.");
- LOG.warn("Bailing out ... ");
- throw ace;
- } catch (IOException ie) {
- LOG.info("problem cleaning system directory: " + systemDir, ie);
- }
- Thread.sleep(FS_ACCESS_RETRY_PERIOD);
- }
-
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedException();
- }
-
- deleteLocalFiles(SUBDIR);
- }
-
- /**
- * A GroomServer registers with its status to BSPMaster when startup, which
- * will update GroomServers cache.
- *
- * @param status to be updated in cache.
- * @return true if registering successfully; false if fail.
- */
- @Override
- public boolean register(GroomServerStatus status) throws IOException {
- if (null == status) {
- LOG.error("No groom server status.");
- throw new NullPointerException("No groom server status.");
- }
- Throwable e = null;
- try {
- GroomProtocol wc = (GroomProtocol) RPC.waitForProxy(
- GroomProtocol.class, GroomProtocol.versionID,
- resolveWorkerAddress(status.getRpcServer()), this.conf);
- if (null == wc) {
- LOG.warn("Fail to create Worker client at host " + status.getPeerName());
- return false;
- }
- // TODO: need to check if peer name has changed
- groomServers.putIfAbsent(status, wc);
- } catch (UnsupportedOperationException u) {
- e = u;
- } catch (ClassCastException c) {
- e = c;
- } catch (NullPointerException n) {
- e = n;
- } catch (IllegalArgumentException i) {
- e = i;
- } catch (Exception ex) {
- e = ex;
- }
-
- if (null != e) {
- LOG.error("Fail to register GroomServer " + status.getGroomName(), e);
- return false;
- }
-
- return true;
- }
-
- private static InetSocketAddress resolveWorkerAddress(String data) {
- return new InetSocketAddress(data.split(":")[0], Integer.parseInt(data
- .split(":")[1]));
- }
-
- private void updateGroomServersKey(GroomServerStatus old,
- GroomServerStatus newKey) {
- synchronized (groomServers) {
- GroomProtocol worker = groomServers.remove(old);
- groomServers.put(newKey, worker);
- }
- }
-
- @Override
- public boolean report(Directive directive) throws IOException {
- instructor.put(directive);
- return true;
- }
-
- private JobInProgress whichJob(BSPJobID id) {
- for (JobInProgress job : taskScheduler
- .getJobs(SimpleTaskScheduler.PROCESSING_QUEUE)) {
- if (job.getJobID().equals(id)) {
- return job;
- }
- }
- return null;
- }
-
- // /////////////////////////////////////////////////////////////
- // BSPMaster methods
- // /////////////////////////////////////////////////////////////
-
- // Get the job directory in system directory
- Path getSystemDirectoryForJob(BSPJobID id) {
- return new Path(getSystemDir(), id.toString());
- }
-
- String[] getLocalDirs() throws IOException {
- return conf.getStrings("bsp.local.dir");
- }
-
- void deleteLocalFiles() throws IOException {
- String[] localDirs = getLocalDirs();
- for (int i = 0; i < localDirs.length; i++) {
- FileSystem.getLocal(conf).delete(new Path(localDirs[i]), true);
- }
- }
-
- void deleteLocalFiles(String subdir) throws IOException {
- try {
- String[] localDirs = getLocalDirs();
- for (int i = 0; i < localDirs.length; i++) {
- FileSystem.getLocal(conf).delete(new Path(localDirs[i], subdir), true);
- }
- } catch (NullPointerException e) {
- LOG.info(e);
- }
- }
-
- /**
- * Constructs a local file name. Files are distributed among configured local
- * directories.
- */
- Path getLocalPath(String pathString) throws IOException {
- return conf.getLocalPath("bsp.local.dir", pathString);
- }
-
- public static BSPMaster startMaster(HamaConfiguration conf)
- throws IOException, InterruptedException {
- return startMaster(conf, generateNewIdentifier());
- }
-
- public static BSPMaster startMaster(HamaConfiguration conf, String identifier)
- throws IOException, InterruptedException {
-
- BSPMaster result = new BSPMaster(conf, identifier);
- result.taskScheduler.setGroomServerManager(result);
- result.taskScheduler.start();
-
- return result;
- }
-
- public static InetSocketAddress getAddress(Configuration conf) {
- String hamaMasterStr = conf.get("bsp.master.address", "localhost");
- int defaultPort = conf.getInt("bsp.master.port", 40000);
-
- return NetUtils.createSocketAddr(hamaMasterStr, defaultPort);
- }
-
- /**
- * BSPMaster identifier
- *
- * @return String BSPMaster identification number
- */
- private static String generateNewIdentifier() {
- return new SimpleDateFormat("yyyyMMddHHmm").format(new Date());
- }
-
- public void offerService() throws InterruptedException, IOException {
-
- this.masterServer.start();
-
- synchronized (this) {
- state = State.RUNNING;
- }
-
- instructor = new Instructor();
- instructor.bind(ReportGroomStatusDirective.class,
- new ReportGroomStatusHandler());
- instructor.start();
-
- LOG.info("Starting RUNNING");
-
- this.masterServer.join();
-
- LOG.info("Stopped RPC Master server.");
- }
-
- // //////////////////////////////////////////////////
- // InterServerProtocol
- // //////////////////////////////////////////////////
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- if (protocol.equals(MasterProtocol.class.getName())) {
- return MasterProtocol.versionID;
- } else if (protocol.equals(JobSubmissionProtocol.class.getName())) {
- return JobSubmissionProtocol.versionID;
- } else {
- throw new IOException("Unknown protocol to BSPMaster: " + protocol);
- }
- }
-
- // //////////////////////////////////////////////////
- // JobSubmissionProtocol
- // //////////////////////////////////////////////////
- /**
- * This method returns new job id. The returned job id increases sequentially.
- */
- @Override
- public BSPJobID getNewJobId() throws IOException {
- int id;
- synchronized (nextJobId) {
- id = nextJobId;
- nextJobId = Integer.valueOf(id + 1);
- }
- return new BSPJobID(this.masterIdentifier, id);
- }
-
- @Override
- public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException {
- if (jobs.containsKey(jobID)) {
- // job already running, don't start twice
- LOG.info("The job (" + jobID + ") was already submitted");
- return jobs.get(jobID).getStatus();
- }
-
- JobInProgress job = new JobInProgress(jobID, new Path(jobFile), this,
- this.conf);
- return addJob(jobID, job);
- }
-
- // //////////////////////////////////////////////////
- // GroomServerManager functions
- // //////////////////////////////////////////////////
-
- @Override
- public ClusterStatus getClusterStatus(boolean detailed) {
- Map groomPeersMap = null;
-
- // give the caller a snapshot of the cluster status
- int numGroomServers = groomServers.size();
- if (detailed) {
- groomPeersMap = new HashMap();
- for (Map.Entry entry : groomServers
- .entrySet()) {
- GroomServerStatus s = entry.getKey();
- groomPeersMap.put(s.getGroomName(), s.getPeerName());
- }
- }
-
- // TODO currently we only have one task slot per groom server
- this.totalTaskCapacity = numGroomServers;
-
- if (detailed) {
- return new ClusterStatus(groomPeersMap, totalTasks, totalTaskCapacity,
- state);
- } else {
- return new ClusterStatus(numGroomServers, totalTasks, totalTaskCapacity,
- state);
- }
- }
-
- @Override
- public GroomProtocol findGroomServer(GroomServerStatus status) {
- return groomServers.get(status);
- }
-
- @Override
- public Collection findGroomServers() {
- return groomServers.values();
- }
-
- @Override
- public Collection groomServerStatusKeySet() {
- return groomServers.keySet();
- }
-
- @Override
- public void addJobInProgressListener(JobInProgressListener listener) {
- jobInProgressListeners.add(listener);
- }
-
- @Override
- public void removeJobInProgressListener(JobInProgressListener listener) {
- jobInProgressListeners.remove(listener);
- }
-
- @Override
- public Map currentGroomServerPeers() {
- Map tmp = new HashMap();
- for (GroomServerStatus status : groomServers.keySet()) {
- tmp.put(status.getGroomName(), status.getPeerName());
- }
- return tmp;
- }
-
- public String getBSPMasterName() {
- return host + ":" + port;
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public String getBSPMasterIdentifier() {
- return masterIdentifier;
- }
-
- public int getHttpPort() {
- return infoPort;
- }
-
- /**
- * Adds a job to the bsp master. Make sure that the checks are inplace before
- * adding a job. This is the core job submission logic
- *
- * @param jobId The id for the job submitted which needs to be added
- */
- private synchronized JobStatus addJob(BSPJobID jobId, JobInProgress job) {
- totalSubmissions++;
- synchronized (jobs) {
- jobs.put(job.getProfile().getJobID(), job);
- for (JobInProgressListener listener : jobInProgressListeners) {
- try {
- listener.jobAdded(job);
- } catch (IOException ioe) {
- LOG.error("Fail to alter Scheduler a job is added.", ioe);
- }
- }
- }
- return job.getStatus();
- }
-
- @Override
- public JobStatus[] jobsToComplete() throws IOException {
- return getJobStatus(jobs.values(), true);
- }
-
- @Override
- public JobStatus[] getAllJobs() throws IOException {
- LOG.debug("returns all jobs: " + jobs.size());
- return getJobStatus(jobs.values(), false);
- }
-
- private synchronized JobStatus[] getJobStatus(Collection jips,
- boolean toComplete) {
- if (jips == null) {
- return new JobStatus[] {};
- }
- List jobStatusList = new ArrayList();
- for (JobInProgress jip : jips) {
- JobStatus status = jip.getStatus();
-
- status.setStartTime(jip.getStartTime());
- // Sets the user name
- status.setUsername(jip.getProfile().getUser());
- status.setName(jip.getJobName());
-
- if (toComplete) {
- if (status.getRunState() == JobStatus.RUNNING
- || status.getRunState() == JobStatus.PREP) {
- jobStatusList.add(status);
- }
- } else {
- jobStatusList.add(status);
- }
- }
-
- return jobStatusList.toArray(new JobStatus[jobStatusList.size()]);
- }
-
- @Override
- public synchronized String getFilesystemName() throws IOException {
- if (fs == null) {
- throw new IllegalStateException("FileSystem object not available yet");
- }
- return fs.getUri().toString();
- }
-
- /**
- * Return system directory to which BSP store control files.
- */
- @Override
- public String getSystemDir() {
- Path sysDir = new Path(conf.get("bsp.system.dir", "/tmp/hadoop/bsp/system"));
- return fs.makeQualified(sysDir).toString();
- }
-
- @Override
- public JobProfile getJobProfile(BSPJobID jobid) throws IOException {
- synchronized (this) {
- JobInProgress job = jobs.get(jobid);
- if (job != null) {
- return job.getProfile();
- }
- }
- return null;
- }
-
- @Override
- public JobStatus getJobStatus(BSPJobID jobid) throws IOException {
- synchronized (this) {
- JobInProgress job = jobs.get(jobid);
- if (job != null) {
- return job.getStatus();
- }
- }
- return null;
- }
-
- @Override
- public void killJob(BSPJobID jobid) throws IOException {
- JobInProgress job = jobs.get(jobid);
-
- if (null == job) {
- LOG.info("killJob(): JobId " + jobid.toString() + " is not a valid job");
- return;
- }
-
- killJob(job);
- }
-
- private synchronized void killJob(JobInProgress job) {
- LOG.info("Killing job " + job.getJobID());
- job.kill();
- }
-
- @Override
- public boolean killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException {
- return false;
- }
-
- public static BSPMaster constructMaster(
- Class extends BSPMaster> masterClass, final Configuration conf) {
- try {
- Constructor extends BSPMaster> c = masterClass
- .getConstructor(Configuration.class);
- return c.newInstance(conf);
- } catch (Exception e) {
- throw new RuntimeException("Failed construction of " + "Master: "
- + masterClass.toString()
- + ((e.getCause() != null) ? e.getCause().getMessage() : ""), e);
- }
- }
-
- public void shutdown() {
- this.masterServer.stop();
- }
-
- public BSPMaster.State currentState() {
- return this.state;
- }
-}
Index: src/java/org/apache/hama/bsp/TaskScheduler.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskScheduler.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskScheduler.java (working copy)
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Used by a {@link BSPMaster} to schedule {@link Task}s on {@link GroomServer}
- * s.
- */
-abstract class TaskScheduler implements Configurable {
-
- protected Configuration conf;
- protected GroomServerManager groomServerManager;
-
- public Configuration getConf() {
- return conf;
- }
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- public synchronized void setGroomServerManager(
- GroomServerManager groomServerManager) {
- this.groomServerManager = groomServerManager;
- }
-
- /**
- * Lifecycle method to allow the scheduler to start any work in separate
- * threads.
- *
- * @throws IOException
- */
- public void start() throws IOException {
- // do nothing
- }
-
- /**
- * Lifecycle method to allow the scheduler to stop any work it is doing.
- *
- * @throws IOException
- */
- public void terminate() throws IOException {
- // do nothing
- }
-
- // public abstract void addJob(JobInProgress job);
-
- /**
- * Returns a collection of jobs in an order which is specific to the
- * particular scheduler.
- *
- * @param Queue name.
- * @return JobInProgress corresponded to the specified queue.
- */
- public abstract Collection getJobs(String queue);
-}
Index: src/java/org/apache/hama/bsp/Messagable.java
===================================================================
--- src/java/org/apache/hama/bsp/Messagable.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/Messagable.java (working copy)
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * A interface for BSP message class.
- */
-public interface Messagable {
-
-}
Index: src/java/org/apache/hama/bsp/TaskLog.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskLog.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskLog.java (working copy)
@@ -1,343 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hama.HamaConfiguration;
-
-/**
- * A simple logger to handle the task-specific user logs.
- */
-public class TaskLog {
- private static final Log LOG = LogFactory.getLog(TaskLog.class.getName());
-
- private static final File LOG_DIR = new File(
- System.getProperty("hama.log.dir"), "userlogs").getAbsoluteFile();
-
- static {
- if (!LOG_DIR.exists()) {
- LOG_DIR.mkdirs();
- }
- }
-
- public static File getTaskLogFile(TaskAttemptID taskid, LogName filter) {
- return new File(new File(LOG_DIR, taskid.toString()), filter.toString());
- }
-
- /**
- * The filter for userlogs.
- */
- public static enum LogName {
- /** Log on the stdout of the task. */
- STDOUT("stdout"),
-
- /** Log on the stderr of the task. */
- STDERR("stderr"),
-
- /** Log on the map-reduce system logs of the task. */
- SYSLOG("syslog"),
-
- /** The java profiler information. */
- PROFILE("profile.out"),
-
- /** Log the debug script's stdout */
- DEBUGOUT("debugout");
-
- private String prefix;
-
- private LogName(String prefix) {
- this.prefix = prefix;
- }
-
- @Override
- public String toString() {
- return prefix;
- }
- }
-
- private static class TaskLogsPurgeFilter implements FileFilter {
- long purgeTimeStamp;
-
- TaskLogsPurgeFilter(long purgeTimeStamp) {
- this.purgeTimeStamp = purgeTimeStamp;
- }
-
- public boolean accept(File file) {
- LOG.debug("PurgeFilter - file: " + file + ", mtime: "
- + file.lastModified() + ", purge: " + purgeTimeStamp);
- return file.lastModified() < purgeTimeStamp;
- }
- }
-
- /**
- * Purge old user logs.
- *
- * @throws IOException
- */
- public static synchronized void cleanup(int logsRetainHours)
- throws IOException {
- // Purge logs of tasks on this tasktracker if their
- // mtime has exceeded "mapred.task.log.retain" hours
- long purgeTimeStamp = System.currentTimeMillis()
- - (logsRetainHours * 60L * 60 * 1000);
- File[] oldTaskLogs = LOG_DIR.listFiles(new TaskLogsPurgeFilter(
- purgeTimeStamp));
- if (oldTaskLogs != null) {
- for (int i = 0; i < oldTaskLogs.length; ++i) {
- FileUtil.fullyDelete(oldTaskLogs[i]);
- }
- }
- }
-
- static class Reader extends InputStream {
- private long bytesRemaining;
- private FileInputStream file;
-
- /**
- * Read a log file from start to end positions. The offsets may be negative,
- * in which case they are relative to the end of the file. For example,
- * Reader(taskid, kind, 0, -1) is the entire file and Reader(taskid, kind,
- * -4197, -1) is the last 4196 bytes.
- *
- * @param taskid the id of the task to read the log file for
- * @param kind the kind of log to read
- * @param start the offset to read from (negative is relative to tail)
- * @param end the offset to read upto (negative is relative to tail)
- * @throws IOException
- */
- public Reader(TaskAttemptID taskid, LogName kind, long start, long end)
- throws IOException {
- // find the right log file
- File filename = getTaskLogFile(taskid, kind);
- // calculate the start and stop
- long size = filename.length();
- if (start < 0) {
- start += size + 1;
- }
- if (end < 0) {
- end += size + 1;
- }
- start = Math.max(0, Math.min(start, size));
- end = Math.max(0, Math.min(end, size));
- bytesRemaining = end - start;
- file = new FileInputStream(filename);
- // skip upto start
- long pos = 0;
- while (pos < start) {
- long result = file.skip(start - pos);
- if (result < 0) {
- bytesRemaining = 0;
- break;
- }
- pos += result;
- }
- }
-
- @Override
- public int read() throws IOException {
- int result = -1;
- if (bytesRemaining > 0) {
- bytesRemaining -= 1;
- result = file.read();
- }
- return result;
- }
-
- @Override
- public int read(byte[] buffer, int offset, int length) throws IOException {
- length = (int) Math.min(length, bytesRemaining);
- int bytes = file.read(buffer, offset, length);
- if (bytes > 0) {
- bytesRemaining -= bytes;
- }
- return bytes;
- }
-
- @Override
- public int available() throws IOException {
- return (int) Math.min(bytesRemaining, file.available());
- }
-
- @Override
- public void close() throws IOException {
- file.close();
- }
- }
-
- private static final String bashCommand = "bash";
- private static final String tailCommand = "tail";
-
- /**
- * Get the desired maximum length of task's logs.
- *
- * @param conf the job to look in
- * @return the number of bytes to cap the log files at
- */
- public static long getTaskLogLength(HamaConfiguration conf) {
- return conf.getLong("mapred.userlog.limit.kb", 100) * 1024;
- }
-
- /**
- * Wrap a command in a shell to capture stdout and stderr to files. If the
- * tailLength is 0, the entire output will be saved.
- *
- * @param cmd The command and the arguments that should be run
- * @param stdoutFilename The filename that stdout should be saved to
- * @param stderrFilename The filename that stderr should be saved to
- * @param tailLength The length of the tail to be saved.
- * @return the modified command that should be run
- */
- public static List captureOutAndError(List cmd,
- File stdoutFilename, File stderrFilename, long tailLength)
- throws IOException {
- return captureOutAndError(null, cmd, stdoutFilename, stderrFilename,
- tailLength);
- }
-
- /**
- * Wrap a command in a shell to capture stdout and stderr to files. Setup
- * commands such as setting memory limit can be passed which will be executed
- * before exec. If the tailLength is 0, the entire output will be saved.
- *
- * @param setup The setup commands for the execed process.
- * @param cmd The command and the arguments that should be run
- * @param stdoutFilename The filename that stdout should be saved to
- * @param stderrFilename The filename that stderr should be saved to
- * @param tailLength The length of the tail to be saved.
- * @return the modified command that should be run
- */
- public static List captureOutAndError(List setup,
- List cmd, File stdoutFilename, File stderrFilename,
- long tailLength) throws IOException {
- String stdout = FileUtil.makeShellPath(stdoutFilename);
- String stderr = FileUtil.makeShellPath(stderrFilename);
- List result = new ArrayList(3);
- result.add(bashCommand);
- result.add("-c");
- StringBuffer mergedCmd = new StringBuffer();
- if (setup != null && setup.size() > 0) {
- mergedCmd.append(addCommand(setup, false));
- mergedCmd.append(";");
- }
- if (tailLength > 0) {
- mergedCmd.append("(");
- } else {
- mergedCmd.append("exec ");
- }
- mergedCmd.append(addCommand(cmd, true));
- mergedCmd.append(" < /dev/null ");
- if (tailLength > 0) {
- mergedCmd.append(" | ");
- mergedCmd.append(tailCommand);
- mergedCmd.append(" -c ");
- mergedCmd.append(tailLength);
- mergedCmd.append(" >> ");
- mergedCmd.append(stdout);
- mergedCmd.append(" ; exit $PIPESTATUS ) 2>&1 | ");
- mergedCmd.append(tailCommand);
- mergedCmd.append(" -c ");
- mergedCmd.append(tailLength);
- mergedCmd.append(" >> ");
- mergedCmd.append(stderr);
- mergedCmd.append(" ; exit $PIPESTATUS");
- } else {
- mergedCmd.append(" 1>> ");
- mergedCmd.append(stdout);
- mergedCmd.append(" 2>> ");
- mergedCmd.append(stderr);
- }
- result.add(mergedCmd.toString());
- return result;
- }
-
- /**
- * Add quotes to each of the command strings and return as a single string
- *
- * @param cmd The command to be quoted
- * @param isExecutable makes shell path if the first argument is executable
- * @return returns The quoted string.
- * @throws IOException
- */
- public static String addCommand(List cmd, boolean isExecutable)
- throws IOException {
- StringBuffer command = new StringBuffer();
- for (String s : cmd) {
- command.append('\'');
- if (isExecutable) {
- // the executable name needs to be expressed as a shell path for the
- // shell to find it.
- command.append(FileUtil.makeShellPath(new File(s)));
- isExecutable = false;
- } else {
- command.append(s);
- }
- command.append('\'');
- command.append(" ");
- }
- return command.toString();
- }
-
- /**
- * Wrap a command in a shell to capture debug script's stdout and stderr to
- * debugout.
- *
- * @param cmd The command and the arguments that should be run
- * @param debugoutFilename The filename that stdout and stderr should be saved
- * to.
- * @return the modified command that should be run
- * @throws IOException
- */
- public static List captureDebugOut(List cmd,
- File debugoutFilename) throws IOException {
- String debugout = FileUtil.makeShellPath(debugoutFilename);
- List result = new ArrayList(3);
- result.add(bashCommand);
- result.add("-c");
- StringBuffer mergedCmd = new StringBuffer();
- mergedCmd.append("exec ");
- boolean isExecutable = true;
- for (String s : cmd) {
- if (isExecutable) {
- // the executable name needs to be expressed as a shell path for the
- // shell to find it.
- mergedCmd.append(FileUtil.makeShellPath(new File(s)));
- isExecutable = false;
- } else {
- mergedCmd.append(s);
- }
- mergedCmd.append(" ");
- }
- mergedCmd.append(" < /dev/null ");
- mergedCmd.append(" >");
- mergedCmd.append(debugout);
- mergedCmd.append(" 2>&1 ");
- result.add(mergedCmd.toString());
- return result;
- }
-
-} // TaskLog
Index: src/java/org/apache/hama/bsp/BSPInterface.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPInterface.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPInterface.java (working copy)
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * Interface BSP defines the basic operations needed to implement the BSP
- * algorithm.
- */
-public interface BSPInterface extends Configurable {
-
- /**
- * A user defined function for programming in the BSP style.
- *
- * Applications can use the {@link org.apache.hama.bsp.BSPPeer} to handle the
- * communication and synchronization between processors.
- *
- * @param bspPeer
- * @throws IOException
- * @throws KeeperException
- * @throws InterruptedException
- */
- public void bsp(BSPPeer bspPeer) throws IOException, KeeperException,
- InterruptedException;
-}
Index: src/java/org/apache/hama/bsp/TaskAttemptContext.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskAttemptContext.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskAttemptContext.java (working copy)
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.Progressable;
-
-/**
- * The context for task attempts.
- */
-public class TaskAttemptContext extends BSPJobContext implements Progressable {
- private final TaskAttemptID taskId;
- private String status = "";
-
- public TaskAttemptContext(Configuration conf, TaskAttemptID taskId) {
- super(conf, taskId.getJobID());
- this.taskId = taskId;
- }
-
- /**
- * Get the unique name for this task attempt.
- */
- public TaskAttemptID getTaskAttemptID() {
- return taskId;
- }
-
- /**
- * Set the current status of the task to the given string.
- */
- public void setStatus(String msg) throws IOException {
- status = msg;
- }
-
- /**
- * Get the last set status message.
- *
- * @return the current status message
- */
- public String getStatus() {
- return status;
- }
-
- /**
- * Report progress. The subtypes actually do work in this method.
- */
- public void progress() {
- }
-}
Index: src/java/org/apache/hama/bsp/TaskStatus.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskStatus.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskStatus.java (working copy)
@@ -1,279 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * Describes the current status of a task. This is not intended to be a
- * comprehensive piece of data.
- */
-class TaskStatus implements Writable, Cloneable {
- static final Log LOG = LogFactory.getLog(TaskStatus.class);
-
- // enumeration for reporting current phase of a task.
- public static enum Phase {
- STARTING, COMPUTE, BARRIER_SYNC, CLEANUP
- }
-
- // what state is the task in?
- public static enum State {
- RUNNING, SUCCEEDED, FAILED, UNASSIGNED, KILLED, COMMIT_PENDING, FAILED_UNCLEAN, KILLED_UNCLEAN
- }
-
- private BSPJobID jobId;
- private TaskAttemptID taskId;
- private float progress;
- private volatile State runState;
- private String stateString;
- private String groomServer;
- private long superstepCount;
-
- private long startTime;
- private long finishTime;
-
- private volatile Phase phase = Phase.STARTING;
-
- /**
- *
- */
- public TaskStatus() {
- jobId = new BSPJobID();
- taskId = new TaskAttemptID();
- this.superstepCount = 0;
- }
-
- public TaskStatus(BSPJobID jobId, TaskAttemptID taskId, float progress,
- State runState, String stateString, String groomServer, Phase phase) {
- this.jobId = jobId;
- this.taskId = taskId;
- this.progress = progress;
- this.runState = runState;
- this.stateString = stateString;
- this.groomServer = groomServer;
- this.phase = phase;
- this.superstepCount = 0;
- }
-
- // //////////////////////////////////////////////////
- // Accessors and Modifiers
- // //////////////////////////////////////////////////
-
- public BSPJobID getJobId() {
- return jobId;
- }
-
- public TaskAttemptID getTaskId() {
- return taskId;
- }
-
- public float getProgress() {
- return progress;
- }
-
- public void setProgress(float progress) {
- this.progress = progress;
- }
-
- public State getRunState() {
- return runState;
- }
-
- public void setRunState(State state) {
- this.runState = state;
- }
-
- public String getStateString() {
- return stateString;
- }
-
- public void setStateString(String stateString) {
- this.stateString = stateString;
- }
-
- public String getGroomServer() {
- return groomServer;
- }
-
- public void setGroomServer(String groomServer) {
- this.groomServer = groomServer;
- }
-
- public long getFinishTime() {
- return finishTime;
- }
-
- void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
- }
-
- /**
- * Get start time of the task.
- *
- * @return 0 is start time is not set, else returns start time.
- */
- public long getStartTime() {
- return startTime;
- }
-
- /**
- * Set startTime of the task.
- *
- * @param startTime start time
- */
- void setStartTime(long startTime) {
- this.startTime = startTime;
- }
-
- /**
- * Get current phase of this task.
- *
- * @return .
- */
- public Phase getPhase() {
- return this.phase;
- }
-
- /**
- * Set current phase of this task.
- *
- * @param phase phase of this task
- */
- void setPhase(Phase phase) {
- this.phase = phase;
- }
-
- /**
- * Update the status of the task.
- *
- * This update is done by ping thread before sending the status.
- *
- * @param progress
- * @param state
- * @param counters
- */
- synchronized void statusUpdate(float progress, String state) {
- setProgress(progress);
- setStateString(state);
- }
-
- /**
- * Update the status of the task.
- *
- * @param status updated status
- */
- synchronized void statusUpdate(TaskStatus status) {
- this.progress = status.getProgress();
- this.runState = status.getRunState();
- this.stateString = status.getStateString();
-
- if (status.getStartTime() != 0) {
- this.startTime = status.getStartTime();
- }
- if (status.getFinishTime() != 0) {
- this.finishTime = status.getFinishTime();
- }
-
- this.phase = status.getPhase();
- }
-
- /**
- * Update specific fields of task status
- *
- * This update is done in BSPMaster when a cleanup attempt of task reports its
- * status. Then update only specific fields, not all.
- *
- * @param runState
- * @param progress
- * @param state
- * @param phase
- * @param finishTime
- */
- synchronized void statusUpdate(State runState, float progress, String state,
- Phase phase, long finishTime) {
- setRunState(runState);
- setProgress(progress);
- setStateString(state);
- setPhase(phase);
- if (finishTime != 0) {
- this.finishTime = finishTime;
- }
- }
-
- /**
- * @return The number of BSP super steps executed by the task.
- */
- public long getSuperstepCount() {
- return superstepCount;
- }
-
- /**
- * Increments the number of BSP super steps executed by the task.
- */
- public void incrementSuperstepCount() {
- superstepCount += 1;
- }
-
- @Override
- public Object clone() {
- try {
- return super.clone();
- } catch (CloneNotSupportedException cnse) {
- // Shouldn't happen since we do implement Clonable
- throw new InternalError(cnse.toString());
- }
- }
-
- // ////////////////////////////////////////////
- // Writable
- // ////////////////////////////////////////////
-
- @Override
- public void readFields(DataInput in) throws IOException {
- this.jobId.readFields(in);
- this.taskId.readFields(in);
- this.progress = in.readFloat();
- this.runState = WritableUtils.readEnum(in, State.class);
- this.stateString = Text.readString(in);
- this.phase = WritableUtils.readEnum(in, Phase.class);
- this.startTime = in.readLong();
- this.finishTime = in.readLong();
- this.superstepCount = in.readLong();
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- jobId.write(out);
- taskId.write(out);
- out.writeFloat(progress);
- WritableUtils.writeEnum(out, runState);
- Text.writeString(out, stateString);
- WritableUtils.writeEnum(out, phase);
- out.writeLong(startTime);
- out.writeLong(finishTime);
- out.writeLong(superstepCount);
- }
-}
Index: src/java/org/apache/hama/bsp/BSPTask.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPTask.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPTask.java (working copy)
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hama.ipc.BSPPeerProtocol;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * Base class for tasks.
- */
-public class BSPTask extends Task {
-
- public static final Log LOG = LogFactory.getLog(BSPTask.class);
-
- private BSPJob conf;
-
- public BSPTask() {
- }
-
- public BSPTask(BSPJobID jobId, String jobFile, TaskAttemptID taskid, int partition) {
- this.jobId = jobId;
- this.jobFile = jobFile;
- this.taskId = taskid;
- this.partition = partition;
- }
-
- @Override
- public BSPTaskRunner createRunner(GroomServer groom) {
- return new BSPTaskRunner(this, groom, this.conf);
- }
-
- @Override
- public void run(BSPJob job, BSPPeer bspPeer, BSPPeerProtocol umbilical)
- throws IOException {
-
- BSP bsp = (BSP) ReflectionUtils.newInstance(job.getConf().getClass(
- "bsp.work.class", BSP.class), job.getConf());
-
- try {
- bsp.bsp(bspPeer);
- } catch (IOException e) {
- LOG.error("Exception during BSP execution!", e);
- } catch (KeeperException e) {
- LOG.error("Exception during BSP execution!", e);
- } catch (InterruptedException e) {
- LOG.error("Exception during BSP execution!", e);
- }
-
- done(umbilical);
- }
-
- public BSPJob getConf() {
- return conf;
- }
-
- public void setConf(BSPJob conf) {
- this.conf = conf;
- }
-
-}
Index: src/java/org/apache/hama/bsp/GroomServer.java
===================================================================
--- src/java/org/apache/hama/bsp/GroomServer.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/GroomServer.java (working copy)
@@ -1,920 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.lang.reflect.Constructor;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FSError;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.DNS;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.DiskChecker;
-import org.apache.hadoop.util.DiskChecker.DiskErrorException;
-import org.apache.hadoop.util.RunJar;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hama.Constants;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.ipc.BSPPeerProtocol;
-import org.apache.hama.ipc.MasterProtocol;
-import org.apache.hama.ipc.GroomProtocol;
-import org.apache.log4j.LogManager;
-
-/**
- * A Groom Server (shortly referred to as groom) is a process that performs bsp
- * tasks assigned by BSPMaster. Each groom contacts the BSPMaster, and it takes
- * assigned tasks and reports its status by means of periodical piggybacks with
- * BSPMaster. Each groom is designed to run with HDFS or other distributed
- * storages. Basically, a groom server and a data node should be run on one
- * physical node.
- */
-public class GroomServer implements Runnable, GroomProtocol, BSPPeerProtocol {
- public static final Log LOG = LogFactory.getLog(GroomServer.class);
- static final String SUBDIR = "groomServer";
-
- private volatile static int REPORT_INTERVAL = 1 * 1000;
-
- Configuration conf;
-
- // Constants
- static enum State {
- NORMAL, COMPUTE, SYNC, BARRIER, STALE, INTERRUPTED, DENIED
- };
-
- // Running States and its related things
- volatile boolean initialized = false;
- volatile boolean running = true;
- volatile boolean shuttingDown = false;
- boolean justInited = true;
- GroomServerStatus status = null;
-
- // Attributes
- String groomServerName;
- String localHostname;
- String groomHostName;
- InetSocketAddress bspMasterAddr;
- private Instructor instructor;
-
- // Filesystem
- // private LocalDirAllocator localDirAllocator;
- Path systemDirectory = null;
- FileSystem systemFS = null;
-
- // Job
- private int failures;
- private int maxCurrentTasks = 1;
- Map tasks = new HashMap();
- /** Map from taskId -> TaskInProgress. */
- Map runningTasks = null;
- Map finishedTasks = null;
- Map runningJobs = null;
-
- // new nexus between GroomServer and BSPMaster
- // holds/ manage all tasks
- // List tasksList = new
- // CopyOnWriteArrayList();
-
- private String rpcServer;
- private Server workerServer;
- MasterProtocol masterClient;
-
- InetSocketAddress taskReportAddress;
- Server taskReportServer = null;
-
- private PeerNames allPeerNames = null;
-
- // private BlockingQueue tasksToCleanup = new
- // LinkedBlockingQueue();
-
- private class DispatchTasksHandler implements DirectiveHandler {
-
- public void handle(Directive directive) throws DirectiveException {
- GroomServerAction[] actions = ((DispatchTasksDirective) directive)
- .getActions();
-
- allPeerNames = new PeerNames(((DispatchTasksDirective) directive)
- .getGroomServerPeers().values());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Got Response from BSPMaster with "
- + ((actions != null) ? actions.length : 0) + " actions");
- }
-
- if (actions != null) {
- for (GroomServerAction action : actions) {
- if (action instanceof LaunchTaskAction) {
- startNewTask((LaunchTaskAction) action);
- } else {
-
- // TODO Use the cleanup thread
- // tasksToCleanup.put(action);
-
- KillTaskAction killAction = (KillTaskAction) action;
- if (tasks.containsKey(killAction.getTaskID())) {
- TaskInProgress tip = tasks.get(killAction.getTaskID());
- tip.taskStatus.setRunState(TaskStatus.State.FAILED);
- try {
- tip.killAndCleanup(true);
- } catch (IOException ioe) {
- throw new DirectiveException("Error when killing a "
- + "TaskInProgress.", ioe);
- }
- }
- }
- }
- }
- }
- }
-
- private class Instructor extends Thread {
- final BlockingQueue buffer = new LinkedBlockingQueue();
- final ConcurrentMap, DirectiveHandler> handlers = new ConcurrentHashMap, DirectiveHandler>();
-
- public void bind(Class extends Directive> instruction,
- DirectiveHandler handler) {
- handlers.putIfAbsent(instruction, handler);
- }
-
- public void put(Directive directive) {
- try {
- buffer.put(directive);
- } catch (InterruptedException ie) {
- LOG.error("Unable to put directive into queue.", ie);
- Thread.currentThread().interrupt();
- }
- }
-
- public void run() {
- while (true) {
- try {
- Directive directive = buffer.take();
- if (directive instanceof DispatchTasksDirective) {
- ((DirectiveHandler) handlers.get(DispatchTasksDirective.class))
- .handle(directive);
- } else {
- throw new RuntimeException("Directive is not supported."
- + directive);
- }
- } catch (InterruptedException ie) {
- LOG.error("Unable to retrieve directive from the queue.", ie);
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- LOG.error("Fail to execute directive.", e);
- }
- }
- }
- }
-
- public GroomServer(Configuration conf) throws IOException {
- LOG.info("groom start");
- this.conf = conf;
-
- bspMasterAddr = BSPMaster.getAddress(conf);
- // FileSystem local = FileSystem.getLocal(conf);
- // this.localDirAllocator = new LocalDirAllocator("bsp.local.dir");
- }
-
- public synchronized void initialize() throws IOException {
- if (this.conf.get(Constants.PEER_HOST) != null) {
- this.localHostname = conf.get(Constants.PEER_HOST);
- }
-
- if (localHostname == null) {
- this.localHostname = DNS.getDefaultHost(
- conf.get("bsp.dns.interface", "default"),
- conf.get("bsp.dns.nameserver", "default"));
- }
- // check local disk
- checkLocalDirs(conf.getStrings("bsp.local.dir"));
- deleteLocalFiles("groomserver");
-
- // Clear out state tables
- this.tasks.clear();
- this.runningJobs = new TreeMap();
- this.runningTasks = new LinkedHashMap();
- this.finishedTasks = new LinkedHashMap();
- this.conf.set(Constants.PEER_HOST, localHostname);
- this.conf.set(Constants.GROOM_RPC_HOST, localHostname);
-
- int rpcPort = -1;
- String rpcAddr = null;
- if (false == this.initialized) {
- rpcAddr = conf.get(Constants.GROOM_RPC_HOST,
- Constants.DEFAULT_GROOM_RPC_HOST);
- rpcPort = conf.getInt(Constants.GROOM_RPC_PORT,
- Constants.DEFAULT_GROOM_RPC_PORT);
- if (-1 == rpcPort || null == rpcAddr)
- throw new IllegalArgumentException("Error rpc address " + rpcAddr
- + " port" + rpcPort);
- this.workerServer = RPC.getServer(this, rpcAddr, rpcPort, conf);
- this.workerServer.start();
- this.rpcServer = rpcAddr + ":" + rpcPort;
-
- LOG.info("Worker rpc server --> " + rpcServer);
- }
-
- @SuppressWarnings("deprecation")
- String address = NetUtils.getServerAddress(conf,
- "bsp.groom.report.bindAddress", "bsp.groom.report.port",
- "bsp.groom.report.address");
- InetSocketAddress socAddr = NetUtils.createSocketAddr(address);
- String bindAddress = socAddr.getHostName();
- int tmpPort = socAddr.getPort();
-
- // RPC initialization
- // TODO numHandlers should be a ..
- this.taskReportServer = RPC.getServer(this, bindAddress, tmpPort, 10,
- false, this.conf);
-
- this.taskReportServer.start();
-
- // get the assigned address
- this.taskReportAddress = taskReportServer.getListenerAddress();
- this.conf.set("bsp.groom.report.address", taskReportAddress.getHostName()
- + ":" + taskReportAddress.getPort());
- LOG.info("GroomServer up at: " + this.taskReportAddress);
-
- this.groomHostName = rpcAddr;
- this.groomServerName = "groomd_" + this.rpcServer.replace(':', '_');
- LOG.info("Starting groom: " + this.rpcServer);
-
- DistributedCache.purgeCache(this.conf);
-
- // establish the communication link to bsp master
- this.masterClient = (MasterProtocol) RPC.waitForProxy(MasterProtocol.class,
- MasterProtocol.versionID, bspMasterAddr, conf);
-
- // enroll in bsp master
- if (-1 == rpcPort || null == rpcAddr)
- throw new IllegalArgumentException("Error rpc address " + rpcAddr
- + " port" + rpcPort);
- if (!this.masterClient.register(new GroomServerStatus(groomServerName,
- getBspPeerName(), cloneAndResetRunningTaskStatuses(), failures,
- maxCurrentTasks, this.rpcServer))) {
- LOG.error("There is a problem in establishing communication"
- + " link with BSPMaster");
- throw new IOException("There is a problem in establishing"
- + " communication link with BSPMaster.");
- }
-
- this.instructor = new Instructor();
- this.instructor.bind(DispatchTasksDirective.class,
- new DispatchTasksHandler());
- instructor.start();
- this.running = true;
- this.initialized = true;
- }
-
- /** Return the port at which the tasktracker bound to */
- public synchronized InetSocketAddress getTaskTrackerReportAddress() {
- return taskReportAddress;
- }
-
- @Override
- public void dispatch(Directive directive) throws IOException {
- if (!instructor.isAlive())
- throw new IOException();
-
- instructor.put(directive);
- }
-
- private static void checkLocalDirs(String[] localDirs)
- throws DiskErrorException {
- boolean writable = false;
-
- LOG.info(localDirs);
-
- if (localDirs != null) {
- for (int i = 0; i < localDirs.length; i++) {
- try {
- LOG.info(localDirs[i]);
- DiskChecker.checkDir(new File(localDirs[i]));
- writable = true;
- } catch (DiskErrorException e) {
- LOG.warn("BSP Processor local " + e.getMessage());
- }
- }
- }
-
- if (!writable)
- throw new DiskErrorException("all local directories are not writable");
- }
-
- public String[] getLocalDirs() {
- return conf.getStrings("bsp.local.dir");
- }
-
- public void deleteLocalFiles() throws IOException {
- String[] localDirs = getLocalDirs();
- for (int i = 0; i < localDirs.length; i++) {
- FileSystem.getLocal(this.conf).delete(new Path(localDirs[i]), true);
- }
- }
-
- public void deleteLocalFiles(String subdir) throws IOException {
- try {
- String[] localDirs = getLocalDirs();
- for (int i = 0; i < localDirs.length; i++) {
- FileSystem.getLocal(this.conf).delete(new Path(localDirs[i], subdir),
- true);
- }
- } catch (NullPointerException e) {
- LOG.info(e);
- }
- }
-
- public void cleanupStorage() throws IOException {
- deleteLocalFiles();
- }
-
- private void startCleanupThreads() throws IOException {
-
- }
-
- public State offerService() throws Exception {
- while (running && !shuttingDown) {
- try {
-
- // Reports to a BSPMaster
- for (Map.Entry e : runningTasks
- .entrySet()) {
- Thread.sleep(REPORT_INTERVAL);
- TaskInProgress tip = e.getValue();
- TaskStatus taskStatus = tip.getStatus();
-
- if (taskStatus.getRunState() == TaskStatus.State.RUNNING) {
- taskStatus.setProgress(taskStatus.getSuperstepCount());
-
- if (!tip.runner.isAlive()) {
- if (taskStatus.getRunState() != TaskStatus.State.FAILED) {
- taskStatus.setRunState(TaskStatus.State.SUCCEEDED);
- }
- taskStatus.setPhase(TaskStatus.Phase.CLEANUP);
- }
- }
-
- doReport(taskStatus);
- }
-
- Thread.sleep(REPORT_INTERVAL);
- } catch (InterruptedException ie) {
- }
-
- try {
- if (justInited) {
- String dir = masterClient.getSystemDir();
- if (dir == null) {
- LOG.error("Fail to get system directory.");
- throw new IOException("Fail to get system directory.");
- }
- systemDirectory = new Path(dir);
- systemFS = systemDirectory.getFileSystem(conf);
- }
- justInited = false;
- } catch (DiskErrorException de) {
- String msg = "Exiting groom server for disk error:\n"
- + StringUtils.stringifyException(de);
- LOG.error(msg);
-
- return State.STALE;
- } catch (RemoteException re) {
- return State.DENIED;
- } catch (Exception except) {
- String msg = "Caught exception: "
- + StringUtils.stringifyException(except);
- LOG.error(msg);
- }
- }
- return State.NORMAL;
- }
-
- private void startNewTask(LaunchTaskAction action) {
- Task t = action.getTask();
- BSPJob jobConf = null;
- try {
- jobConf = new BSPJob(t.getJobID(), t.getJobFile());
- } catch (IOException e1) {
- LOG.error(e1);
- }
-
- TaskInProgress tip = new TaskInProgress(t, jobConf, this.groomServerName);
-
- synchronized (this) {
- tasks.put(t.getTaskID(), tip);
- runningTasks.put(t.getTaskID(), tip);
- }
-
- try {
- localizeJob(tip);
- } catch (Throwable e) {
- String msg = ("Error initializing " + tip.getTask().getTaskID() + ":\n" + StringUtils
- .stringifyException(e));
- LOG.warn(msg);
- }
- }
-
- /**
- * Update and report refresh status back to BSPMaster.
- */
- public void doReport(TaskStatus taskStatus) {
- GroomServerStatus groomStatus = new GroomServerStatus(groomServerName,
- getBspPeerName(), updateTaskStatus(taskStatus), failures,
- maxCurrentTasks, rpcServer);
- try {
- boolean ret = masterClient.report(new ReportGroomStatusDirective(
- groomStatus));
- if (!ret) {
- LOG.warn("Fail to renew BSPMaster's GroomServerStatus. "
- + " groom name: " + groomStatus.getGroomName() + " peer name:"
- + groomStatus.getPeerName() + " rpc server:" + rpcServer);
- }
- } catch (IOException ioe) {
- LOG.error("Fail to communicate with BSPMaster for reporting.", ioe);
- }
- }
-
- public List updateTaskStatus(TaskStatus taskStatus) {
- List tlist = new ArrayList();
- synchronized (runningTasks) {
-
- if (taskStatus.getRunState() == TaskStatus.State.SUCCEEDED
- || taskStatus.getRunState() == TaskStatus.State.FAILED) {
- synchronized (finishedTasks) {
- TaskInProgress tip = runningTasks.remove(taskStatus.getTaskId());
- tlist.add((TaskStatus) taskStatus.clone());
- finishedTasks.put(taskStatus.getTaskId(), tip);
- }
- } else if (taskStatus.getRunState() == TaskStatus.State.RUNNING) {
- tlist.add((TaskStatus) taskStatus.clone());
- }
-
- }
- return tlist;
- }
-
- private void localizeJob(TaskInProgress tip) throws IOException {
- Task task = tip.getTask();
- conf.addResource(task.getJobFile());
- BSPJob defaultJobConf = new BSPJob((HamaConfiguration) conf);
- Path localJobFile = defaultJobConf.getLocalPath(SUBDIR + "/"
- + task.getTaskID() + "/" + "job.xml");
-
- RunningJob rjob = addTaskToJob(task.getJobID(), localJobFile, tip);
- BSPJob jobConf = null;
-
- synchronized (rjob) {
- if (!rjob.localized) {
- Path localJarFile = defaultJobConf.getLocalPath(SUBDIR + "/"
- + task.getTaskID() + "/" + "job.jar");
- systemFS.copyToLocalFile(new Path(task.getJobFile()), localJobFile);
-
- HamaConfiguration conf = new HamaConfiguration();
- conf.addResource(localJobFile);
- jobConf = new BSPJob(conf, task.getJobID().toString());
-
- Path jarFile = new Path(jobConf.getJar());
- jobConf.setJar(localJarFile.toString());
-
- if (jarFile != null) {
- systemFS.copyToLocalFile(jarFile, localJarFile);
-
- // also unjar the job.jar files in workdir
- File workDir = new File(
- new File(localJobFile.toString()).getParent(), "work");
- if (!workDir.mkdirs()) {
- if (!workDir.isDirectory()) {
- throw new IOException("Mkdirs failed to create "
- + workDir.toString());
- }
- }
- RunJar.unJar(new File(localJarFile.toString()), workDir);
- }
- rjob.localized = true;
- }
- }
-
- launchTaskForJob(tip, jobConf);
- }
-
- private void launchTaskForJob(TaskInProgress tip, BSPJob jobConf) {
- try {
- tip.setJobConf(jobConf);
- tip.launchTask();
- } catch (Throwable ie) {
- tip.taskStatus.setRunState(TaskStatus.State.FAILED);
- String error = StringUtils.stringifyException(ie);
- LOG.info(error);
- }
- }
-
- private RunningJob addTaskToJob(BSPJobID jobId, Path localJobFile,
- TaskInProgress tip) {
- synchronized (runningJobs) {
- RunningJob rJob = null;
- if (!runningJobs.containsKey(jobId)) {
- rJob = new RunningJob(jobId, localJobFile);
- rJob.localized = false;
- rJob.tasks = new HashSet();
- rJob.jobFile = localJobFile;
- runningJobs.put(jobId, rJob);
- } else {
- rJob = runningJobs.get(jobId);
- }
- rJob.tasks.add(tip);
- return rJob;
- }
- }
-
- /**
- * The datastructure for initializing a job
- */
- static class RunningJob {
- private BSPJobID jobid;
- private Path jobFile;
- // keep this for later use
- Set tasks;
- boolean localized;
- boolean keepJobFiles;
-
- RunningJob(BSPJobID jobid, Path jobFile) {
- this.jobid = jobid;
- localized = false;
- tasks = new HashSet();
- this.jobFile = jobFile;
- keepJobFiles = false;
- }
-
- Path getJobFile() {
- return jobFile;
- }
-
- BSPJobID getJobId() {
- return jobid;
- }
- }
-
- private synchronized List cloneAndResetRunningTaskStatuses() {
- List result = new ArrayList(runningTasks.size());
- for (TaskInProgress tip : runningTasks.values()) {
- TaskStatus status = tip.getStatus();
- result.add((TaskStatus) status.clone());
- }
- return result;
- }
-
- public void run() {
- try {
- initialize();
- startCleanupThreads();
- boolean denied = false;
- while (running && !shuttingDown && !denied) {
-
- boolean staleState = false;
- try {
- while (running && !staleState && !shuttingDown && !denied) {
- try {
- State osState = offerService();
- if (osState == State.STALE) {
- staleState = true;
- } else if (osState == State.DENIED) {
- denied = true;
- }
- } catch (Exception e) {
- if (!shuttingDown) {
- LOG.info("Lost connection to BSP Master [" + bspMasterAddr
- + "]. Retrying...", e);
- try {
- Thread.sleep(5000);
- } catch (InterruptedException ie) {
- }
- }
- }
- }
- } finally {
- // close();
- }
- if (shuttingDown) {
- return;
- }
- LOG.warn("Reinitializing local state");
- initialize();
- }
- } catch (IOException ioe) {
- LOG.error("Got fatal exception while reinitializing GroomServer: "
- + StringUtils.stringifyException(ioe));
- return;
- }
- }
-
- public synchronized void shutdown() throws IOException {
- shuttingDown = true;
- close();
- }
-
- public synchronized void close() throws IOException {
- this.running = false;
- this.initialized = false;
- cleanupStorage();
- this.workerServer.stop();
- RPC.stopProxy(masterClient);
-
- if (taskReportServer != null) {
- taskReportServer.stop();
- taskReportServer = null;
- }
- }
-
- public static Thread startGroomServer(final GroomServer hrs) {
- return startGroomServer(hrs, "regionserver" + hrs.groomServerName);
- }
-
- public static Thread startGroomServer(final GroomServer hrs, final String name) {
- Thread t = new Thread(hrs);
- t.setName(name);
- t.start();
- return t;
- }
-
- // /////////////////////////////////////////////////////
- // TaskInProgress maintains all the info for a Task that
- // lives at this GroomServer. It maintains the Task object,
- // its TaskStatus, and the BSPTaskRunner.
- // /////////////////////////////////////////////////////
- class TaskInProgress {
- Task task;
- BSPJob jobConf;
- BSPJob localJobConf;
- BSPTaskRunner runner;
- volatile boolean done = false;
- volatile boolean wasKilled = false;
- private TaskStatus taskStatus;
-
- public TaskInProgress(Task task, BSPJob jobConf, String groomServer) {
- this.task = task;
- this.jobConf = jobConf;
- this.localJobConf = null;
- this.taskStatus = new TaskStatus(task.getJobID(), task.getTaskID(), 0,
- TaskStatus.State.UNASSIGNED, "running", groomServer,
- TaskStatus.Phase.STARTING);
- }
-
- private void localizeTask(Task task) throws IOException {
- Path localJobFile = this.jobConf.getLocalPath(SUBDIR + "/"
- + task.getTaskID() + "/job.xml");
- Path localJarFile = this.jobConf.getLocalPath(SUBDIR + "/"
- + task.getTaskID() + "/job.jar");
-
- String jobFile = task.getJobFile();
- systemFS.copyToLocalFile(new Path(jobFile), localJobFile);
- task.setJobFile(localJobFile.toString());
-
- localJobConf = new BSPJob(task.getJobID(), localJobFile.toString());
- localJobConf.set("bsp.task.id", task.getTaskID().toString());
- String jarFile = localJobConf.getJar();
-
- if (jarFile != null) {
- systemFS.copyToLocalFile(new Path(jarFile), localJarFile);
- localJobConf.setJar(localJarFile.toString());
- }
-
- LOG.debug("localizeTask : " + localJobConf.getJar());
- LOG.debug("localizeTask : " + localJobFile.toString());
-
- task.setConf(localJobConf);
- }
-
- public synchronized void setJobConf(BSPJob jobConf) {
- this.jobConf = jobConf;
- }
-
- public synchronized BSPJob getJobConf() {
- return localJobConf;
- }
-
- public void launchTask() throws IOException {
- localizeTask(task);
- taskStatus.setRunState(TaskStatus.State.RUNNING);
- this.runner = task.createRunner(GroomServer.this);
- this.runner.start();
- }
-
- /**
- * This task has run on too long, and should be killed.
- */
- public synchronized void killAndCleanup(boolean wasFailure)
- throws IOException {
- runner.kill();
- }
-
- /**
- */
- public Task getTask() {
- return task;
- }
-
- /**
- */
- public synchronized TaskStatus getStatus() {
- return taskStatus;
- }
-
- /**
- */
- public TaskStatus.State getRunState() {
- return taskStatus.getRunState();
- }
-
- public boolean wasKilled() {
- return wasKilled;
- }
-
- @Override
- public boolean equals(Object obj) {
- return (obj instanceof TaskInProgress)
- && task.getTaskID().equals(
- ((TaskInProgress) obj).getTask().getTaskID());
- }
-
- @Override
- public int hashCode() {
- return task.getTaskID().hashCode();
- }
- }
-
- public boolean isRunning() {
- return running;
- }
-
- public static GroomServer constructGroomServer(
- Class extends GroomServer> groomServerClass, final Configuration conf2) {
- try {
- Constructor extends GroomServer> c = groomServerClass
- .getConstructor(Configuration.class);
- return c.newInstance(conf2);
- } catch (Exception e) {
- throw new RuntimeException("Failed construction of " + "Master: "
- + groomServerClass.toString(), e);
- }
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- if (protocol.equals(GroomProtocol.class.getName())) {
- return GroomProtocol.versionID;
- } else if (protocol.equals(BSPPeerProtocol.class.getName())) {
- return BSPPeerProtocol.versionID;
- } else {
- throw new IOException("Unknown protocol to GroomServer: " + protocol);
- }
- }
-
- /**
- * @return bsp peer information in the form of "address:port".
- */
- public String getBspPeerName() {
- // TODO Later, peers list should be returned.
- return this.groomHostName + ":" + Constants.DEFAULT_PEER_PORT;
- }
-
- /**
- * The main() for child processes.
- */
- public static class Child {
-
- public static void main(String[] args) throws Throwable {
- LOG.debug("Child starting");
-
- HamaConfiguration defaultConf = new HamaConfiguration();
- // report address
- String host = args[0];
- int port = Integer.parseInt(args[1]);
- InetSocketAddress address = new InetSocketAddress(host, port);
- TaskAttemptID taskid = TaskAttemptID.forName(args[2]);
-
- // //////////////////
- BSPPeerProtocol umbilical = (BSPPeerProtocol) RPC.getProxy(
- BSPPeerProtocol.class, BSPPeerProtocol.versionID, address,
- defaultConf);
-
- Task task = umbilical.getTask(taskid);
-
- defaultConf.addResource(new Path(task.getJobFile()));
- BSPJob job = new BSPJob(task.getJobID(), task.getJobFile());
-
- defaultConf.set(Constants.PEER_HOST, args[3]);
- defaultConf.setInt(Constants.PEER_PORT, Constants.DEFAULT_PEER_PORT);
-
- BSPPeer bspPeer = new BSPPeer(defaultConf, taskid, umbilical);
- bspPeer.setJobConf(job);
- bspPeer.setAllPeerNames(umbilical.getAllPeerNames().getAllPeerNames());
-
- TaskStatus taskStatus = new TaskStatus(task.getJobID(), task.getTaskID(),
- 0, TaskStatus.State.RUNNING, "running", host,
- TaskStatus.Phase.STARTING);
-
- bspPeer.setCurrentTaskStatus(taskStatus);
-
- try {
- // use job-specified working directory
- FileSystem.get(job.getConf()).setWorkingDirectory(
- job.getWorkingDirectory());
-
- task.run(job, bspPeer, umbilical); // run the task
-
- } catch (FSError e) {
- LOG.fatal("FSError from child", e);
- umbilical.fsError(taskid, e.getMessage());
- } catch (Throwable throwable) {
- LOG.warn("Error running child", throwable);
- // Report back any failures, for diagnostic purposes
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- throwable.printStackTrace(new PrintStream(baos));
- } finally {
- bspPeer.close(); // close peer.
-
- RPC.stopProxy(umbilical);
- // Shutting down log4j of the child-vm...
- // This assumes that on return from Task.run()
- // there is no more logging done.
- LogManager.shutdown();
- }
- }
- }
-
- @Override
- public Task getTask(TaskAttemptID taskid) throws IOException {
- TaskInProgress tip = tasks.get(taskid);
- if (tip != null) {
- return tip.getTask();
- } else {
- return null;
- }
- }
-
- public void incrementSuperstepCount(TaskAttemptID taskid) throws IOException {
- TaskInProgress tip = tasks.get(taskid);
- tip.getStatus().incrementSuperstepCount();
- }
-
- @Override
- public boolean ping(TaskAttemptID taskid) throws IOException {
- // TODO Auto-generated method stub
- return false;
- }
-
- @Override
- public void done(TaskAttemptID taskid, boolean shouldBePromoted)
- throws IOException {
- // TODO Auto-generated method stub
- }
-
- @Override
- public void fsError(TaskAttemptID taskId, String message) throws IOException {
- // TODO Auto-generated method stub
- }
-
- @Override
- public PeerNames getAllPeerNames() {
- return allPeerNames;
- }
-}
\ No newline at end of file
Index: src/java/org/apache/hama/bsp/Directive.java
===================================================================
--- src/java/org/apache/hama/bsp/Directive.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/Directive.java (working copy)
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-
-/**
- * A generic directive from the {@link org.apache.hama.bsp.BSPMaster} to the
- * {@link org.apache.hama.bsp.GroomServer} to take some 'action'.
- */
-public class Directive implements Writable{
-
- protected long timestamp;
- protected Directive.Type type;
-
- public static enum Type {
- Request(1), Response(2);
- int t;
-
- Type(int t) {
- this.t = t;
- }
-
- public int value() {
- return this.t;
- }
- };
-
- public Directive(){}
-
- public Directive(Directive.Type type) {
- this.timestamp = System.currentTimeMillis();
- this.type = type;
- }
-
- public long getTimestamp() {
- return this.timestamp;
- }
-
- public Directive.Type getType() {
- return this.type;
- }
-
- /**
- * Command for BSPMaster or GroomServer to execute.
- public abstract void execute() throws Exception;
- */
-
- public void write(DataOutput out) throws IOException {
- out.writeLong(this.timestamp);
- out.writeInt(this.type.value());
- }
-
- public void readFields(DataInput in) throws IOException {
- this.timestamp = in.readLong();
- int t = in.readInt();
- if (Directive.Type.Request.value() == t) {
- this.type = Directive.Type.Request;
- }else{
- this.type = Directive.Type.Response;
- }
- }
-}
Index: src/java/org/apache/hama/bsp/JobStatus.java
===================================================================
--- src/java/org/apache/hama/bsp/JobStatus.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobStatus.java (working copy)
@@ -1,279 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-
-/**
- * Describes the current status of a job.
- */
-public class JobStatus implements Writable, Cloneable {
- public static final Log LOG = LogFactory.getLog(JobStatus.class);
-
- static {
- WritableFactories.setFactory(JobStatus.class, new WritableFactory() {
- public Writable newInstance() {
- return new JobStatus();
- }
- });
- }
-
- public static enum State {
- RUNNING(1), SUCCEEDED(2), FAILED(3), PREP(4), KILLED(5);
- int s;
-
- State(int s) {
- this.s = s;
- }
-
- public int value() {
- return this.s;
- }
-
- @Override
- public String toString() {
- String name = null;
- switch (this) {
- case RUNNING:
- name = "RUNNING";
- break;
- case SUCCEEDED:
- name = "SUCCEEDED";
- break;
- case FAILED:
- name = "FAILED";
- break;
- case PREP:
- name = "SETUP";
- break;
- case KILLED:
- name = "KILLED";
- break;
- }
-
- return name;
- }
-
- }
-
- public static final int RUNNING = 1;
- public static final int SUCCEEDED = 2;
- public static final int FAILED = 3;
- public static final int PREP = 4;
- public static final int KILLED = 5;
-
- private BSPJobID jobid;
- private long progress;
- private long cleanupProgress;
- private long setupProgress;
- private volatile State state;// runState in enum
- private int runState;
- private long startTime;
- private String schedulingInfo = "NA";
- private String user;
- private long superstepCount;
- private String name;
-
- private long finishTime;
-
- public JobStatus() {
- }
-
- public JobStatus(BSPJobID jobid, String user, long progress, int runState) {
- this(jobid, user, progress, 0, runState);
- }
-
- public JobStatus(BSPJobID jobid, String user, long progress,
- long cleanupProgress, int runState) {
- this(jobid, user, 0, progress, cleanupProgress, runState);
- }
-
- public JobStatus(BSPJobID jobid, String user, long setupProgress,
- long progress, long cleanupProgress, int runState) {
- this(jobid, user, 0, progress, cleanupProgress, runState, 0);
- }
-
- public JobStatus(BSPJobID jobid, String user, long setupProgress,
- long progress, long cleanupProgress, int runState, long superstepCount) {
- this.jobid = jobid;
- this.setupProgress = setupProgress;
- this.progress = progress;
- this.cleanupProgress = cleanupProgress;
- this.runState = runState;
- this.state = State.values()[runState - 1];
- this.superstepCount = superstepCount;
- this.user = user;
- }
-
- public BSPJobID getJobID() {
- return jobid;
- }
-
- public synchronized long progress() {
- return progress;
- }
-
- synchronized void setprogress(long p) {
- this.progress = p;
- }
-
- public synchronized long cleanupProgress() {
- return cleanupProgress;
- }
-
- synchronized void setCleanupProgress(int p) {
- this.cleanupProgress = p;
- }
-
- public synchronized long setupProgress() {
- return setupProgress;
- }
-
- synchronized void setSetupProgress(long p) {
- this.setupProgress = p;
- }
-
- public JobStatus.State getState() {
- return this.state;
- }
-
- public void setState(JobStatus.State state) {
- this.state = state;
- }
-
- public synchronized int getRunState() {
- return runState;
- }
-
- public synchronized void setRunState(int state) {
- this.runState = state;
- }
-
- public synchronized long getSuperstepCount() {
- return superstepCount;
- }
-
- public synchronized void setSuperstepCount(long superstepCount) {
- this.superstepCount = superstepCount;
- }
-
- public synchronized void setStartTime(long startTime) {
- this.startTime = startTime;
- }
-
- public synchronized long getStartTime() {
- return startTime;
- }
-
- public synchronized void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
- }
-
- /**
- * Get the finish time of the job.
- */
- public synchronized long getFinishTime() {
- return finishTime;
- }
-
- /**
- * @param user The username of the job
- */
- public synchronized void setUsername(String user) {
- this.user = user;
- }
-
- /**
- * @return the username of the job
- */
- public synchronized String getUsername() {
- return user;
- }
-
- @Override
- public Object clone() {
- try {
- return super.clone();
- } catch (CloneNotSupportedException cnse) {
- throw new InternalError(cnse.toString());
- }
- }
-
- public synchronized String getSchedulingInfo() {
- return schedulingInfo;
- }
-
- public synchronized void setSchedulingInfo(String schedulingInfo) {
- this.schedulingInfo = schedulingInfo;
- }
-
- public synchronized boolean isJobComplete() {
- return (runState == JobStatus.SUCCEEDED || runState == JobStatus.FAILED || runState == JobStatus.KILLED);
- }
-
- public synchronized void write(DataOutput out) throws IOException {
- jobid.write(out);
- out.writeLong(setupProgress);
- out.writeLong(progress);
- out.writeLong(cleanupProgress);
- out.writeInt(runState);
- out.writeLong(startTime);
- out.writeLong(finishTime);
- Text.writeString(out, user);
- Text.writeString(out, schedulingInfo);
- out.writeLong(superstepCount);
- }
-
- public synchronized void readFields(DataInput in) throws IOException {
- this.jobid = new BSPJobID();
- jobid.readFields(in);
- this.setupProgress = in.readLong();
- this.progress = in.readLong();
- this.cleanupProgress = in.readLong();
- this.runState = in.readInt();
- this.startTime = in.readLong();
- this.finishTime = in.readLong();
- this.user = Text.readString(in);
- this.schedulingInfo = Text.readString(in);
- this.superstepCount = in.readLong();
- }
-
- /**
- * @param name the name to set
- */
- public synchronized void setName(String name) {
- this.name = name;
- }
-
- /**
- * @return the name
- */
- public synchronized String getName() {
- return name;
- }
-
-}
Index: src/java/org/apache/hama/bsp/TaskInProgress.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskInProgress.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskInProgress.java (working copy)
@@ -1,288 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobStatus;
-
-/**
- *TaskInProgress maintains all the info needed for a Task in the lifetime of
- * its owning Job.
- */
-class TaskInProgress {
- public static final Log LOG = LogFactory.getLog(TaskInProgress.class);
-
- private Configuration conf;
-
- // Constants
- static final int MAX_TASK_EXECS = 1;
- int maxTaskAttempts = 4;
- private boolean failed = false;
- private static final int NUM_ATTEMPTS_PER_RESTART = 1000;
-
- // Job Meta
- private String jobFile = null;
- private int partition;
- private BSPMaster bspMaster;
- private TaskID id;
- private JobInProgress job;
- private int completes = 0;
-
- // Status
- // private double progress = 0;
- // private String state = "";
- private long startTime = 0;
-
- // The 'next' usable taskid of this tip
- int nextTaskId = 0;
-
- // The taskid that took this TIP to SUCCESS
- private TaskAttemptID successfulTaskId;
-
- // The first taskid of this tip
- private TaskAttemptID firstTaskId;
-
- // Map from task Id -> GroomServer Id, contains tasks that are
- // currently runnings
- private TreeMap activeTasks = new TreeMap();
- // All attempt Ids of this TIP
- // private TreeSet tasks = new TreeSet();
- /**
- * Map from taskId -> TaskStatus
- */
- private TreeMap taskStatuses = new TreeMap();
-
- private BSPJobID jobId;
-
- /**
- * Constructor for new nexus between BSPMaster and GroomServer.
- *
- * @param jobId is identification of JobInProgress.
- * @param jobFile the path of job file
- * @param partition which partition this TaskInProgress owns.
- */
- public TaskInProgress(BSPJobID jobId, String jobFile, int partition) {
- this.jobId = jobId;
- this.jobFile = jobFile;
- this.partition = partition;
-
- this.id = new TaskID(jobId, partition);
- }
-
- public TaskInProgress(BSPJobID jobId, String jobFile, BSPMaster master,
- Configuration conf, JobInProgress job, int partition) {
- this.jobId = jobId;
- this.jobFile = jobFile;
- this.setBspMaster(master);
- this.job = job;
- this.setConf(conf);
- this.partition = partition;
-
- this.id = new TaskID(jobId, partition);
- }
-
- /**
- * Return a Task that can be sent to a GroomServer for execution.
- */
- public Task getTaskToRun(GroomServerStatus status) throws IOException {
- Task t = null;
-
- TaskAttemptID taskid = null;
- if (nextTaskId < (MAX_TASK_EXECS + maxTaskAttempts)) {
- int attemptId = job.getNumRestarts() * NUM_ATTEMPTS_PER_RESTART
- + nextTaskId;
- taskid = new TaskAttemptID(id, attemptId);
- ++nextTaskId;
- } else {
- LOG.warn("Exceeded limit of " + (MAX_TASK_EXECS + maxTaskAttempts)
- + " attempts for the tip '" + getTIPId() + "'");
- return null;
- }
-
- t = new BSPTask(jobId, jobFile, taskid, partition);
- activeTasks.put(taskid, status.getGroomName());
-
- return t;
- }
-
- // //////////////////////////////////
- // Accessors
- // //////////////////////////////////
- /**
- * Return the start time
- */
- public long getStartTime() {
- return startTime;
- }
-
- /**
- * Return the parent job
- */
- public JobInProgress getJob() {
- return job;
- }
-
- public TaskID getTIPId() {
- return id;
- }
-
- public TaskID getTaskId() {
- return this.id;
- }
-
- public TreeMap getTasks() {
- return activeTasks;
- }
-
- /**
- * Is the Task associated with taskid is the first attempt of the tip?
- *
- * @param taskId
- * @return Returns true if the Task is the first attempt of the tip
- */
- public boolean isFirstAttempt(TaskAttemptID taskId) {
- return firstTaskId == null ? false : firstTaskId.equals(taskId);
- }
-
- /**
- * Is this tip currently running any tasks?
- *
- * @return true if any tasks are running
- */
- public boolean isRunning() {
- return !activeTasks.isEmpty();
- }
-
- /**
- * Is this tip complete?
- *
- * @return true if the tip is complete, else false
- */
- public synchronized boolean isComplete() {
- return (completes > 0);
- }
-
- /**
- * Is the given taskid the one that took this tip to completion?
- *
- * @param taskid taskid of attempt to check for completion
- * @return true if taskid is complete, else false
- */
- public boolean isComplete(TaskAttemptID taskid) {
- return (completes > 0 && taskid.equals(getSuccessfulTaskid()));
- }
-
- private TreeSet tasksReportedClosed = new TreeSet();
-
- public boolean shouldCloseForClosedJob(TaskAttemptID taskid) {
- TaskStatus ts = (TaskStatus) taskStatuses.get(taskid);
- if ((ts != null) && (!tasksReportedClosed.contains(taskid))
- && (job.getStatus().getRunState() != JobStatus.RUNNING)) {
- tasksReportedClosed.add(taskid);
- return true;
- } else {
- return false;
- }
- }
-
- public void completed(TaskAttemptID taskid) {
- LOG.info("Task '" + taskid.getTaskID().toString() + "' has completed.");
-
- TaskStatus status = (TaskStatus) taskStatuses.get(taskid);
- status.setRunState(TaskStatus.State.SUCCEEDED);
- activeTasks.remove(taskid);
-
- // Note the successful taskid
- setSuccessfulTaskid(taskid);
-
- //
- // Now that the TIP is complete, the other speculative
- // subtasks will be closed when the owning groom server
- // reports in and calls shouldClose() on this object.
- //
-
- this.completes++;
- }
-
- public void terminated(TaskAttemptID taskid) {
- LOG.info("Task '" + taskid.getTaskID().toString() + "' has failed.");
-
- TaskStatus status = (TaskStatus) taskStatuses.get(taskid);
- status.setRunState(TaskStatus.State.FAILED);
- activeTasks.remove(taskid);
- }
-
- private void setSuccessfulTaskid(TaskAttemptID taskid) {
- this.successfulTaskId = taskid;
- }
-
- private TaskAttemptID getSuccessfulTaskid() {
- return successfulTaskId;
- }
-
- public void updateStatus(TaskStatus status) {
- taskStatuses.put(status.getTaskId(), status);
- }
-
- public TaskStatus getTaskStatus(TaskAttemptID taskId) {
- return this.taskStatuses.get(taskId);
- }
-
- public void kill() {
- this.failed = true;
- }
-
- public boolean isFailed() {
- return failed;
- }
-
- /**
- * @param conf the conf to set
- */
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- /**
- * @return the conf
- */
- public Configuration getConf() {
- return conf;
- }
-
- /**
- * @param bspMaster the bspMaster to set
- */
- public void setBspMaster(BSPMaster bspMaster) {
- this.bspMaster = bspMaster;
- }
-
- /**
- * @return the bspMaster
- */
- public BSPMaster getBspMaster() {
- return bspMaster;
- }
-
-}
Index: src/java/org/apache/hama/bsp/DoubleMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/DoubleMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/DoubleMessage.java (working copy)
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * A message that consists of a string tag and a double data.
- */
-public class DoubleMessage extends BSPMessage {
-
- private String tag;
- private double data;
-
- public DoubleMessage() {
- super();
- }
-
- public DoubleMessage(String tag, double data) {
- super();
- this.data = data;
- this.tag = tag;
- }
-
- @Override
- public String getTag() {
- return tag;
- }
-
- @Override
- public Double getData() {
- return data;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(tag);
- out.writeDouble(data);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- tag = in.readUTF();
- data = in.readDouble();
- }
-
-}
Index: src/java/org/apache/hama/bsp/QueueManager.java
===================================================================
--- src/java/org/apache/hama/bsp/QueueManager.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/QueueManager.java (working copy)
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A BSPJob Queue Manager.
- */
-public class QueueManager{
-
- private ConcurrentMap> queues =
- new ConcurrentHashMap>();
-
- public QueueManager(Configuration conf){ }
-
- /**
- * Initialize a job.
- * @param job required initialzied.
- */
- public void initJob(JobInProgress job){
- try{
- //job.updateStatus();
- job.initTasks();
- }catch(IOException ioe){
- ioe.printStackTrace();
- }
- }
-
- /**
- * Add a job to the specified queue.
- * @param name of the queue.
- * @param job to be added.
- */
- public void addJob(String name, JobInProgress job){
- Queue queue = queues.get(name);
- if(null != queue) queue.addJob(job);
- }
-
- /**
- * Remove a job from the head of a designated queue.
- * @param name from which a job is removed.
- * @param job to be removed from the queue.
- */
- public void removeJob(String name, JobInProgress job){
- Queue queue = queues.get(name);
- if(null != queue) queue.removeJob(job);
- }
-
- /**
- * Move a job from a queue to another.
- * @param from a queue a job is to be removed.
- * @param to a queue a job is to be added.
- */
- public void moveJob(String from, String to, JobInProgress job){
- synchronized(queues){
- removeJob(from, job);
- addJob(to, job);
- }
- }
-
- /**
- * Create a FCFS queue with the name provided.
- * @param name of the queue.
- */
- public void createFCFSQueue(String name){
- queues.putIfAbsent(name, new FCFSQueue(name));
- }
-
- /**
- * Find Queue according to the name specified.
- * @param name of the queue.
- * @return queue of JobInProgress
- */
- public Queue findQueue(String name){
- return queues.get(name);
- }
-
-}
Index: src/java/org/apache/hama/bsp/ReportGroomStatusDirective.java
===================================================================
--- src/java/org/apache/hama/bsp/ReportGroomStatusDirective.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/ReportGroomStatusDirective.java (working copy)
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Writable;
-
-/**
- * Reports status of GroomServer.
- */
-public class ReportGroomStatusDirective extends Directive implements Writable {
- public static final Log LOG = LogFactory.getLog(ReportGroomStatusDirective.class);
-
- private GroomServerStatus status;
-
- public ReportGroomStatusDirective(){ super(); }
-
- public ReportGroomStatusDirective(GroomServerStatus status) {
- super(Directive.Type.Response);
- this.status = status;
- }
-
- public GroomServerStatus getStatus() {
- return this.status;
- }
-
- public void write(DataOutput out) throws IOException {
- super.write(out);
- this.status.write(out);
- }
-
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- this.status = new GroomServerStatus();
- this.status.readFields(in);
- }
-}
Index: src/java/org/apache/hama/bsp/TaskLogAppender.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskLogAppender.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskLogAppender.java (working copy)
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.util.LinkedList;
-import java.util.Queue;
-
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.spi.LoggingEvent;
-
-/**
- * A simple log4j-appender for the task child's BSP system logs.
- */
-public class TaskLogAppender extends FileAppender {
- private String taskId; // taskId should be managed as String rather than
- // TaskID object
- // so that log4j can configure it from the configuration(log4j.properties).
- private int maxEvents;
- private Queue tail = null;
-
- @Override
- public void activateOptions() {
- synchronized (this) {
- if (maxEvents > 0) {
- tail = new LinkedList();
- }
- setFile(TaskLog.getTaskLogFile(TaskAttemptID.forName(taskId),
- TaskLog.LogName.SYSLOG).toString());
- setAppend(true);
- super.activateOptions();
- }
- }
-
- @Override
- public void append(LoggingEvent event) {
- synchronized (this) {
- if (tail == null) {
- super.append(event);
- } else {
- if (tail.size() >= maxEvents) {
- tail.remove();
- }
- tail.add(event);
- }
- }
- }
-
- @Override
- public synchronized void close() {
- if (tail != null) {
- for (LoggingEvent event : tail) {
- super.append(event);
- }
- }
- super.close();
- }
-
- /**
- * Getter/Setter methods for log4j.
- */
-
- public String getTaskId() {
- return taskId;
- }
-
- public void setTaskId(String taskId) {
- this.taskId = taskId;
- }
-
- private static final int EVENT_SIZE = 100;
-
- public long getTotalLogFileSize() {
- return maxEvents * EVENT_SIZE;
- }
-
- public void setTotalLogFileSize(long logSize) {
- maxEvents = (int) logSize / EVENT_SIZE;
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSPJobContext.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPJobContext.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPJobContext.java (working copy)
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hama.HamaConfiguration;
-
-/**
- * A read-only view of the bsp job that is provided to the tasks while they are
- * running.
- */
-public class BSPJobContext {
- // Put all of the attribute names in here so that BSPJob and JobContext are
- // consistent.
- protected static final String WORK_CLASS_ATTR = "bsp.work.class";
- protected static final String INPUT_FORMAT_CLASS_ATTR = "bsp.inputformat.class";
- protected static final String OUTPUT_FORMAT_CLASS_ATTR = "bsp.outputformat.class";
- protected static final String WORKING_DIR = "bsp.working.dir";
-
- protected final Configuration conf;
- private final BSPJobID jobId;
-
- public BSPJobContext(Configuration conf, BSPJobID jobId) {
- this.conf = conf;
- this.jobId = jobId;
- }
-
- public BSPJobContext(Path config, BSPJobID jobId) throws IOException {
- this.conf = new HamaConfiguration();
- this.jobId = jobId;
- this.conf.addResource(config);
- }
-
- public BSPJobID getJobID() {
- return jobId;
- }
-
- public Path getWorkingDirectory() throws IOException {
- String name = conf.get(WORKING_DIR);
-
- if (name != null) {
- return new Path(name);
- } else {
- try {
- Path dir = FileSystem.get(conf).getWorkingDirectory();
- conf.set(WORKING_DIR, dir.toString());
- return dir;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- public String getJobName() {
- return conf.get("bsp.job.name", "");
- }
-
- public String getJar() {
- return conf.get("bsp.jar");
- }
-
- /**
- * Constructs a local file name. Files are distributed among configured
- * local directories.
- */
- public Path getLocalPath(String pathString) throws IOException {
- return conf.getLocalPath("bsp.local.dir", pathString);
- }
-
- public String getUser() {
- return conf.get("user.name");
- }
-
- public void writeXml(OutputStream out) throws IOException {
- conf.writeXml(out);
- }
-
- public Configuration getConf() {
- return this.conf;
- }
-
- public String get(String name) {
- return conf.get(name);
- }
-
- public int getInt(String name, int defaultValue) {
- return conf.getInt(name, defaultValue);
- }
-}
Index: src/java/org/apache/hama/bsp/BooleanMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/BooleanMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BooleanMessage.java (working copy)
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * A message that consists of a string tag and a boolean value.
- */
-public class BooleanMessage extends BSPMessage {
-
- String tag;
- boolean data;
-
- public BooleanMessage() {
- super();
- }
-
- public BooleanMessage(String tag, boolean data) {
- super();
- this.tag = tag;
- this.data = data;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(tag);
- out.writeBoolean(data);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- tag = in.readUTF();
- data = in.readBoolean();
- }
-
- @Override
- public String getTag() {
- return tag;
- }
-
- @Override
- public Boolean getData() {
- return data;
- }
-}
\ No newline at end of file
Index: src/java/org/apache/hama/bsp/ClusterStatus.java
===================================================================
--- src/java/org/apache/hama/bsp/ClusterStatus.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/ClusterStatus.java (working copy)
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * Status information on the current state of the BSP cluster.
- *
- * ClusterStatus provides clients with information such as:
- *
- *
- * Size of the cluster.
- *
- *
- * Name of the grooms.
- *
- *
- * Task capacity of the cluster.
- *
- *
- * The number of currently running bsp tasks.
- *
- *
- * State of the BSPMaster.
- *
- *
- *
- * Clients can query for the latest ClusterStatus, via
- * {@link BSPJobClient#getClusterStatus(boolean)}.
- *
- * @see BSPMaster
- */
-public class ClusterStatus implements Writable {
-
- private int numActiveGrooms;
- private Map activeGrooms = new HashMap();
- private int tasks;
- private int maxTasks;
- private BSPMaster.State state;
-
- /**
- *
- */
- public ClusterStatus() {}
-
- public ClusterStatus(int grooms, int tasks, int maxTasks, BSPMaster.State state) {
- this.numActiveGrooms = grooms;
- this.tasks = tasks;
- this.maxTasks = maxTasks;
- this.state = state;
- }
-
- public ClusterStatus(Map activeGrooms, int tasks, int maxTasks,
- BSPMaster.State state) {
- this(activeGrooms.size(), tasks, maxTasks, state);
- this.activeGrooms = activeGrooms;
- }
-
- /**
- * Get the number of groom servers in the cluster.
- *
- * @return the number of groom servers in the cluster.
- */
- public int getGroomServers() {
- return numActiveGrooms;
- }
-
- /**
- * Get the names of groom servers, and their peers, in the cluster.
- *
- * @return the active groom servers in the cluster.
- */
- public Map getActiveGroomNames() {
- return activeGrooms;
- }
-
- /**
- * Get the number of currently running tasks in the cluster.
- *
- * @return the number of currently running tasks in the cluster.
- */
- public int getTasks() {
- return tasks;
- }
-
- /**
- * Get the maximum capacity for running tasks in the cluster.
- *
- * @return the maximum capacity for running tasks in the cluster.
- */
- public int getMaxTasks() {
- return maxTasks;
- }
-
- /**
- * Get the current state of the BSPMaster,
- * as {@link BSPMaster.State}
- *
- * @return the current state of the BSPMaster.
- */
- public BSPMaster.State getBSPMasterState() {
- return state;
- }
-
- //////////////////////////////////////////////
- // Writable
- //////////////////////////////////////////////
- @Override
- public void write(DataOutput out) throws IOException {
- if (activeGrooms.isEmpty()) {
- out.writeInt(numActiveGrooms);
- out.writeBoolean(false);
- } else {
- out.writeInt(activeGrooms.size());
- out.writeBoolean(true);
-
- String[] groomNames = activeGrooms.keySet().toArray(new String[0]);
- List peerNames = new ArrayList();
-
- for (String groomName : groomNames) {
- peerNames.add(activeGrooms.get(groomName));
- }
-
- WritableUtils.writeCompressedStringArray(out, groomNames);
- WritableUtils.writeCompressedStringArray(out, peerNames.toArray(new String[0]));
- }
- out.writeInt(tasks);
- out.writeInt(maxTasks);
- WritableUtils.writeEnum(out, state);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- numActiveGrooms = in.readInt();
- boolean groomListFollows = in.readBoolean();
-
- if (groomListFollows) {
- String[] groomNames = WritableUtils.readCompressedStringArray(in);
- String[] peerNames = WritableUtils.readCompressedStringArray(in);
- activeGrooms = new HashMap(groomNames.length);
-
- for (int i = 0; i < groomNames.length; i++) {
- activeGrooms.put(groomNames[i], peerNames[i]);
- }
- }
-
- tasks = in.readInt();
- maxTasks = in.readInt();
- state = WritableUtils.readEnum(in, BSPMaster.State.class);
- }
-}
Index: src/java/org/apache/hama/bsp/FCFSQueue.java
===================================================================
--- src/java/org/apache/hama/bsp/FCFSQueue.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/FCFSQueue.java (working copy)
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.util.Collection;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-class FCFSQueue implements Queue {
-
- public static final Log LOG = LogFactory.getLog(FCFSQueue.class);
- private final String name;
- private BlockingQueue queue = new LinkedBlockingQueue();
-
- public FCFSQueue(String name) {
- this.name = name;
- }
-
- @Override
- public String getName() {
- return this.name;
- }
-
- @Override
- public void addJob(JobInProgress job) {
- try {
- queue.put(job);
- } catch (InterruptedException ie) {
- LOG.error("Fail to add a job to the " + this.name + " queue.", ie);
- }
- }
-
- @Override
- public void removeJob(JobInProgress job) {
- queue.remove(job);
- }
-
- @Override
- public JobInProgress removeJob() {
- try {
- return queue.take();
- } catch (InterruptedException ie) {
- LOG.error("Fail to remove a job from the " + this.name + " queue.", ie);
- }
- return null;
- }
-
- @Override
- public Collection jobs() {
- return queue;
- }
-
-}
Index: src/java/org/apache/hama/bsp/JobInProgress.java
===================================================================
--- src/java/org/apache/hama/bsp/JobInProgress.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobInProgress.java (working copy)
@@ -1,360 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-
-/**
- * JobInProgress maintains all the info for keeping a Job on the straight and
- * narrow. It keeps its JobProfile and its latest JobStatus, plus a set of
- * tables for doing bookkeeping of its Tasks.ss
- */
-class JobInProgress {
-
- /**
- * Used when the a kill is issued to a job which is initializing.
- */
- static class KillInterruptedException extends InterruptedException {
- private static final long serialVersionUID = 1L;
-
- public KillInterruptedException(String msg) {
- super(msg);
- }
- }
-
- static final Log LOG = LogFactory.getLog(JobInProgress.class);
- boolean tasksInited = false;
- boolean jobInited = false;
-
- Configuration conf;
- JobProfile profile;
- JobStatus status;
- Path jobFile = null;
- Path localJobFile = null;
- Path localJarFile = null;
- private LocalFileSystem localFs;
- // Indicates how many times the job got restarted
- private int restartCount;
-
- long startTime;
- long launchTime;
- long finishTime;
-
- private String jobName;
-
- // private LocalFileSystem localFs;
- private BSPJobID jobId;
- final BSPMaster master;
- TaskInProgress tasks[] = new TaskInProgress[0];
- private long superstepCounter;
-
- int numBSPTasks = 0;
- int clusterSize;
-
- public JobInProgress(BSPJobID jobId, Path jobFile, BSPMaster master,
- Configuration conf) throws IOException {
- this.conf = conf;
- this.jobId = jobId;
- this.localFs = FileSystem.getLocal(conf);
- this.jobFile = jobFile;
- this.master = master;
- this.status = new JobStatus(jobId, null, 0L, 0L,
- JobStatus.State.PREP.value());
- this.startTime = System.currentTimeMillis();
- this.superstepCounter = 0;
- this.restartCount = 0;
-
- this.localJobFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId
- + ".xml");
- this.localJarFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId
- + ".jar");
-
- Path jobDir = master.getSystemDirectoryForJob(jobId);
- FileSystem fs = jobDir.getFileSystem(conf);
- fs.copyToLocalFile(jobFile, localJobFile);
- BSPJob job = new BSPJob(jobId, localJobFile.toString());
- this.numBSPTasks = job.getNumBspTask();
-
- this.profile = new JobProfile(job.getUser(), jobId, jobFile.toString(),
- job.getJobName());
-
- this.setJobName(job.getJobName());
-
- status.setUsername(job.getUser());
- status.setStartTime(startTime);
-
- String jarFile = job.getJar();
- if (jarFile != null) {
- fs.copyToLocalFile(new Path(jarFile), localJarFile);
- }
-
- }
-
- public JobProfile getProfile() {
- return profile;
- }
-
- public JobStatus getStatus() {
- return status;
- }
-
- public synchronized long getLaunchTime() {
- return launchTime;
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public long getFinishTime() {
- return finishTime;
- }
-
- /**
- * @return the number of desired tasks.
- */
- public int desiredBSPTasks() {
- return numBSPTasks;
- }
-
- /**
- * @return The JobID of this JobInProgress.
- */
- public BSPJobID getJobID() {
- return jobId;
- }
-
- public synchronized TaskInProgress findTaskInProgress(TaskID id) {
- if (areTasksInited()) {
- for (TaskInProgress tip : tasks) {
- if (tip.getTaskId().equals(id)) {
- return tip;
- }
- }
- }
- return null;
- }
-
- public synchronized boolean areTasksInited() {
- return this.tasksInited;
- }
-
- public String toString() {
- return "jobName:" + profile.getJobName() + "\n" + "submit user:"
- + profile.getUser() + "\n" + "JobId:" + jobId + "\n" + "JobFile:"
- + jobFile + "\n";
- }
-
- // ///////////////////////////////////////////////////
- // Create/manage tasks
- // ///////////////////////////////////////////////////
-
- public synchronized void initTasks() throws IOException {
- if (tasksInited) {
- return;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("numBSPTasks: " + numBSPTasks);
- }
-
- // adjust number of map tasks to actual number of splits
- this.tasks = new TaskInProgress[numBSPTasks];
- for (int i = 0; i < numBSPTasks; i++) {
- tasks[i] = new TaskInProgress(getJobID(), this.jobFile.toString(),
- this.master, this.conf, this, i);
- }
-
- // Update job status
- this.status = new JobStatus(this.status.getJobID(), this.profile.getUser(),
- 0L, 0L, JobStatus.RUNNING);
-
- tasksInited = true;
- LOG.debug("Job is initialized.");
- }
-
- public synchronized Task obtainNewTask(GroomServerStatus status,
- int clusterSize) {
- this.clusterSize = clusterSize;
-
- if (this.status.getRunState() != JobStatus.RUNNING) {
- LOG.info("Cannot create task split for " + profile.getJobID());
- return null;
- }
-
- Task result = null;
- try {
- for (int i = 0; i < tasks.length; i++) {
- if (!tasks[i].isRunning() && !tasks[i].isComplete()) {
- result = tasks[i].getTaskToRun(status);
- break;
- }
- }
-
- } catch (IOException e) {
- LOG.error("Exception while obtaining new task!", e);
- }
- return result;
- }
-
- public synchronized void completedTask(TaskInProgress tip, TaskStatus status) {
- TaskAttemptID taskid = status.getTaskId();
- updateTaskStatus(tip, status);
- LOG.info("Taskid '" + taskid + "' has finished successfully.");
- tip.completed(taskid);
-
- //
- // If all tasks are complete, then the job is done!
- //
-
- boolean allDone = true;
- for (TaskInProgress taskInProgress : tasks) {
- if (!taskInProgress.isComplete()) {
- allDone = false;
- break;
- }
- }
-
- if (allDone) {
- this.status = new JobStatus(this.status.getJobID(),
- this.profile.getUser(), superstepCounter, superstepCounter,
- superstepCounter, JobStatus.SUCCEEDED, superstepCounter);
- this.finishTime = System.currentTimeMillis();
- this.status.setFinishTime(this.finishTime);
-
- LOG.debug("Job successfully done.");
-
- garbageCollect();
- }
- }
-
- public void failedTask(TaskInProgress tip, TaskStatus status) {
- TaskAttemptID taskid = status.getTaskId();
- updateTaskStatus(tip, status);
- LOG.info("Taskid '" + taskid + "' has failed.");
- tip.terminated(taskid);
-
- //
- // If all tasks are complete, then the job is done!
- //
-
- boolean allDone = true;
- for (TaskInProgress taskInProgress : tasks) {
- if (!taskInProgress.isFailed()) {
- allDone = false;
- break;
- }
- }
-
- if (allDone) {
- this.status = new JobStatus(this.status.getJobID(),
- this.profile.getUser(), superstepCounter, superstepCounter,
- superstepCounter, JobStatus.FAILED, superstepCounter);
- this.finishTime = System.currentTimeMillis();
- this.status.setFinishTime(this.finishTime);
-
- LOG.debug("Job failed.");
-
- garbageCollect();
- }
- }
-
- public synchronized void updateTaskStatus(TaskInProgress tip,
- TaskStatus taskStatus) {
- tip.updateStatus(taskStatus); // update tip
-
- if (superstepCounter < taskStatus.getSuperstepCount()) {
- superstepCounter = taskStatus.getSuperstepCount();
- // TODO Later, we have to update JobInProgress status here
-
- }
- }
-
- public synchronized void kill() {
- if (status.getRunState() != JobStatus.KILLED) {
- this.status = new JobStatus(status.getJobID(), this.profile.getUser(),
- 0L, 0L, 0L, JobStatus.KILLED);
- this.finishTime = System.currentTimeMillis();
- this.status.setFinishTime(this.finishTime);
- //
- // kill all TIPs.
- //
- for (int i = 0; i < tasks.length; i++) {
- tasks[i].kill();
- }
-
- garbageCollect();
- }
-
- }
-
- /**
- * The job is dead. We're now GC'ing it, getting rid of the job from all
- * tables. Be sure to remove all of this job's tasks from the various tables.
- */
- synchronized void garbageCollect() {
- try {
- // Definitely remove the local-disk copy of the job file
- if (localJobFile != null) {
- localFs.delete(localJobFile, true);
- localJobFile = null;
- }
- if (localJarFile != null) {
- localFs.delete(localJarFile, true);
- localJarFile = null;
- }
-
- // JobClient always creates a new directory with job files
- // so we remove that directory to cleanup
- FileSystem fs = FileSystem.get(conf);
- fs.delete(new Path(profile.getJobFile()).getParent(), true);
-
- } catch (IOException e) {
- LOG.info("Error cleaning up " + profile.getJobID() + ": " + e);
- }
- }
-
- /**
- * Get the number of times the job has restarted
- */
- int getNumRestarts() {
- return restartCount;
- }
-
- /**
- * @param jobName the jobName to set
- */
- public void setJobName(String jobName) {
- this.jobName = jobName;
- }
-
- /**
- * @return the jobName
- */
- public String getJobName() {
- return jobName;
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSPMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPMessage.java (working copy)
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import org.apache.hadoop.io.Writable;
-
-/**
- * BSPMessage consists of the tag and the arbitrary amount of data to be
- * communicated.
- */
-public abstract class BSPMessage implements Messagable, Writable {
-
- /**
- * BSP messages are typically identified with tags. This allows to get the tag
- * of data.
- *
- * @return tag of data of BSP message
- */
- public abstract Object getTag();
-
- /**
- * @return data of BSP message
- */
- public abstract Object getData();
-
-}
Index: src/java/org/apache/hama/bsp/ID.java
===================================================================
--- src/java/org/apache/hama/bsp/ID.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/ID.java (working copy)
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.WritableComparable;
-
-/**
- * A general identifier, which internally stores the id as an integer. This is
- * the super class of {@link BSPJobID}, {@link TaskID} and {@link TaskAttemptID}
- * .
- */
-public abstract class ID implements WritableComparable {
- protected static final char SEPARATOR = '_';
- protected int id;
-
- public ID(int id) {
- this.id = id;
- }
-
- protected ID() {
- }
-
- public int getId() {
- return id;
- }
-
- @Override
- public String toString() {
- return String.valueOf(id);
- }
-
- @Override
- public int hashCode() {
- return Integer.valueOf(id).hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null)
- return false;
- if (o.getClass() == this.getClass()) {
- ID that = (ID) o;
- return this.id == that.id;
- } else
- return false;
- }
-
- public int compareTo(ID that) {
- return this.id - that.id;
- }
-
- public void readFields(DataInput in) throws IOException {
- this.id = in.readInt();
- }
-
- public void write(DataOutput out) throws IOException {
- out.writeInt(id);
- }
-}
Index: src/java/org/apache/hama/bsp/IntegerDoubleMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/IntegerDoubleMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/IntegerDoubleMessage.java (working copy)
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * A message that consists of a int tag and a double data.
- */
-public class IntegerDoubleMessage extends BSPMessage {
-
- int tag;
- double data;
-
- public IntegerDoubleMessage() {
- super();
- }
-
- public IntegerDoubleMessage(int tag, double data) {
- super();
- this.tag = tag;
- this.data = data;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(tag);
- out.writeDouble(data);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- tag = in.readInt();
- data = in.readDouble();
- }
-
- @Override
- public Integer getTag() {
- return tag;
- }
-
- @Override
- public Double getData() {
- return data;
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSPPeer.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPPeer.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPPeer.java (working copy)
@@ -1,423 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hama.Constants;
-import org.apache.hama.ipc.BSPPeerProtocol;
-import org.apache.hama.util.Bytes;
-import org.apache.hama.zookeeper.QuorumPeer;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.Stat;
-
-/**
- * This class represents a BSP peer.
- */
-public class BSPPeer implements Watcher, BSPPeerInterface {
-
- public static final Log LOG = LogFactory.getLog(BSPPeer.class);
-
- private Configuration conf;
- private BSPJob jobConf;
-
- private Server server = null;
- private ZooKeeper zk = null;
- private volatile Integer mutex = 0;
-
- private final String bspRoot;
- private final String quorumServers;
-
- private final Map peers = new ConcurrentHashMap();
- private final Map> outgoingQueues = new ConcurrentHashMap>();
- private ConcurrentLinkedQueue localQueue = new ConcurrentLinkedQueue();
- private ConcurrentLinkedQueue localQueueForNextIteration = new ConcurrentLinkedQueue();
- private final Map peerSocketCache = new ConcurrentHashMap();
-
- private SortedSet allPeerNames = new TreeSet();
- private InetSocketAddress peerAddress;
- private TaskStatus currentTaskStatus;
-
- private TaskAttemptID taskid;
- private BSPPeerProtocol umbilical;
-
- /**
- * Protected default constructor for LocalBSPRunner.
- */
- protected BSPPeer() {
- bspRoot = null;
- quorumServers = null;
- }
-
- /**
- * Constructor
- *
- * @param umbilical
- * @param taskid
- */
- public BSPPeer(Configuration conf, TaskAttemptID taskid,
- BSPPeerProtocol umbilical) throws IOException {
- this.conf = conf;
- this.taskid = taskid;
- this.umbilical = umbilical;
-
- String bindAddress = conf.get(Constants.PEER_HOST,
- Constants.DEFAULT_PEER_HOST);
- int bindPort = conf
- .getInt(Constants.PEER_PORT, Constants.DEFAULT_PEER_PORT);
- bspRoot = conf.get(Constants.ZOOKEEPER_ROOT,
- Constants.DEFAULT_ZOOKEEPER_ROOT);
- quorumServers = QuorumPeer.getZKQuorumServersString(conf);
- LOG.debug("Quorum " + quorumServers);
-
- // TODO: may require to dynamic reflect the underlying
- // network e.g. ip address, port.
- peerAddress = new InetSocketAddress(bindAddress, bindPort);
- reinitialize();
- }
-
- public void reinitialize() {
- try {
- LOG.debug("reinitialize(): " + getPeerName());
- server = RPC.getServer(this, peerAddress.getHostName(),
- peerAddress.getPort(), conf);
- server.start();
- LOG.info(" BSPPeer address:" + peerAddress.getHostName() + " port:"
- + peerAddress.getPort());
- } catch (IOException e) {
- LOG.error("Exception during reinitialization!", e);
- }
-
- try {
- zk = new ZooKeeper(quorumServers, conf.getInt(
- Constants.ZOOKEEPER_SESSION_TIMEOUT, 1200000), this);
- } catch (IOException e) {
- LOG.error("Exception during reinitialization!", e);
- }
-
- Stat s = null;
- if (zk != null) {
- try {
- s = zk.exists(Constants.DEFAULT_ZOOKEEPER_ROOT, false);
- } catch (Exception e) {
- LOG.error(s, e);
- }
-
- if (s == null) {
- try {
- zk.create(Constants.DEFAULT_ZOOKEEPER_ROOT, new byte[0],
- Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- } catch (KeeperException e) {
- LOG.error(e);
- } catch (InterruptedException e) {
- LOG.error(e);
- }
- }
- }
-
- }
-
- @Override
- public BSPMessage getCurrentMessage() throws IOException {
- return localQueue.poll();
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.BSPPeerInterface#send(java.net.InetSocketAddress,
- * org.apache.hadoop.io.Writable, org.apache.hadoop.io.Writable)
- */
- @Override
- public void send(String peerName, BSPMessage msg) throws IOException {
- if (peerName.equals(getPeerName())) {
- LOG.debug("Local send bytes (" + msg.getData().toString() + ")");
- localQueueForNextIteration.add(msg);
- } else {
- LOG.debug("Send bytes (" + msg.getData().toString() + ") to " + peerName);
- InetSocketAddress targetPeerAddress = null;
- // Get socket for target peer.
- if (peerSocketCache.containsKey(peerName)) {
- targetPeerAddress = peerSocketCache.get(peerName);
- } else {
- targetPeerAddress = getAddress(peerName);
- peerSocketCache.put(peerName, targetPeerAddress);
- }
- ConcurrentLinkedQueue queue = outgoingQueues
- .get(targetPeerAddress);
- if (queue == null) {
- queue = new ConcurrentLinkedQueue();
- }
- queue.add(msg);
- outgoingQueues.put(targetPeerAddress, queue);
- }
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.BSPPeerInterface#sync()
- */
- @Override
- public void sync() throws IOException, KeeperException, InterruptedException {
- enterBarrier();
- long startTime = System.currentTimeMillis();
- Iterator>> it = this.outgoingQueues
- .entrySet().iterator();
-
- while (it.hasNext()) {
- Entry> entry = it
- .next();
-
- BSPPeerInterface peer = peers.get(entry.getKey());
- if (peer == null) {
- peer = getBSPPeerConnection(entry.getKey());
- }
- Iterable messages = entry.getValue();
- BSPMessageBundle bundle = new BSPMessageBundle();
- for (BSPMessage message : messages) {
- bundle.addMessage(message);
- }
- peer.put(bundle);
- }
-
- if ((System.currentTimeMillis() - startTime) < 200) {
- Thread.sleep(200);
- }
-
- leaveBarrier();
- currentTaskStatus.incrementSuperstepCount();
- umbilical.incrementSuperstepCount(taskid);
-
- startTime = System.currentTimeMillis();
- // Clear outgoing queues.
- clearOutgoingQueues();
-
- // Add non-processed messages from this iteration for the next's queue.
- while (!localQueue.isEmpty()) {
- BSPMessage message = localQueue.poll();
- localQueueForNextIteration.add(message);
- }
- // Switch local queues.
- localQueue = localQueueForNextIteration;
- localQueueForNextIteration = new ConcurrentLinkedQueue();
-
- // TODO: This is a quite temporary solution of HAMA-387.
- // If zk.getChildren() response is slower than 200 milliseconds,
- // BSP system will be hanged.
-
- // We have to consider another way to avoid this problem.
- if ((System.currentTimeMillis() - startTime) < 200) {
- Thread.sleep(200); // at least wait
- }
- }
-
- protected boolean enterBarrier() throws KeeperException, InterruptedException {
- LOG.debug("[" + getPeerName() + "] enter the enterbarrier: "
- + this.getSuperstepCount());
- zk.create(bspRoot + "/" + getPeerName(),
- Bytes.toBytes(this.getSuperstepCount()), Ids.OPEN_ACL_UNSAFE,
- CreateMode.EPHEMERAL);
-
- while (true) {
- synchronized (mutex) {
- List list = zk.getChildren(bspRoot, true);
-
- if (list.size() < jobConf.getNumBspTask()) {
- mutex.wait();
- } else {
- return true;
- }
- }
- }
- }
-
- protected boolean leaveBarrier() throws KeeperException, InterruptedException {
- zk.delete(bspRoot + "/" + getPeerName(), 0);
- while (true) {
- synchronized (mutex) {
- List list = zk.getChildren(bspRoot, true);
-
- if (list.size() > 0) {
- mutex.wait();
- } else {
- return true;
- }
- }
- }
- }
-
- @Override
- public void process(WatchedEvent event) {
- synchronized (mutex) {
- mutex.notify();
- }
- }
-
- public void clear() {
- this.localQueue.clear();
- this.outgoingQueues.clear();
- }
-
- @Override
- public void close() throws IOException {
- server.stop();
- }
-
- @Override
- public void put(BSPMessage msg) throws IOException {
- this.localQueueForNextIteration.add(msg);
- }
-
- @Override
- public void put(BSPMessageBundle messages) throws IOException {
- for (BSPMessage message : messages.getMessages()) {
- this.localQueueForNextIteration.add(message);
- }
- }
-
- @Override
- public long getProtocolVersion(String arg0, long arg1) throws IOException {
- return BSPPeerInterface.versionID;
- }
-
- protected BSPPeerInterface getBSPPeerConnection(InetSocketAddress addr) {
- BSPPeerInterface peer;
- synchronized (this.peers) {
- peer = peers.get(addr);
-
- if (peer == null) {
- try {
- peer = (BSPPeerInterface) RPC.getProxy(BSPPeerInterface.class,
- BSPPeerInterface.versionID, addr, this.conf);
- } catch (IOException e) {
-
- }
- this.peers.put(addr, peer);
- }
- }
-
- return peer;
- }
-
- /**
- * @return the string as host:port of this Peer
- */
- public String getPeerName() {
- return peerAddress.getHostName() + ":" + peerAddress.getPort();
- }
-
- private InetSocketAddress getAddress(String peerName) {
- String[] peerAddrParts = peerName.split(":");
- return new InetSocketAddress(peerAddrParts[0],
- Integer.parseInt(peerAddrParts[1]));
- }
-
- @Override
- public String[] getAllPeerNames() {
- return allPeerNames.toArray(new String[0]);
- }
-
- /**
- * To be invoked by the Groom Server with a list of peers received from an
- * heartbeat response (BSPMaster).
- *
- * @param allPeers
- */
- void setAllPeerNames(Collection allPeerNames) {
- this.allPeerNames = new TreeSet(allPeerNames);
- }
-
- /**
- * @return the number of messages
- */
- public int getNumCurrentMessages() {
- return localQueue.size();
- }
-
- /**
- * Sets the current status
- *
- * @param currentTaskStatus
- */
- public void setCurrentTaskStatus(TaskStatus currentTaskStatus) {
- this.currentTaskStatus = currentTaskStatus;
- }
-
- /**
- * @return the count of current super-step
- */
- public long getSuperstepCount() {
- return currentTaskStatus.getSuperstepCount();
- }
-
- /**
- * Sets the job configuration
- *
- * @param jobConf
- */
- public void setJobConf(BSPJob jobConf) {
- this.jobConf = jobConf;
- }
-
- /**
- * @return the size of local queue
- */
- public int getLocalQueueSize() {
- return localQueue.size();
- }
-
- /**
- * @return the size of outgoing queue
- */
- public int getOutgoingQueueSize() {
- return outgoingQueues.size();
- }
-
- /**
- * Clears local queue
- */
- public void clearLocalQueue() {
- this.localQueue.clear();
- }
-
- /**
- * Clears outgoing queues
- */
- public void clearOutgoingQueues() {
- this.outgoingQueues.clear();
- }
-}
Index: src/java/org/apache/hama/bsp/BSPTaskRunner.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPTaskRunner.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPTaskRunner.java (working copy)
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * Base class that runs a task in a separate process.
- */
-public class BSPTaskRunner extends TaskRunner {
-
- public static final Log LOG = LogFactory.getLog(BSPTaskRunner.class);
-
- public BSPTaskRunner(BSPTask bspTask, GroomServer groom, BSPJob conf) {
- super(bspTask, groom, conf);
- }
-
-}
Index: src/java/org/apache/hama/bsp/IntegerMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/IntegerMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/IntegerMessage.java (working copy)
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * A message that consists of a string tag and a int data.
- */
-public class IntegerMessage extends BSPMessage {
-
- String tag;
- int data;
-
- public IntegerMessage() {
- super();
- }
-
- public IntegerMessage(String tag, int data) {
- super();
- this.tag = tag;
- this.data = data;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(tag);
- out.writeInt(data);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- tag = in.readUTF();
- data = in.readInt();
- }
-
- @Override
- public String getTag() {
- return tag;
- }
-
- @Override
- public Integer getData() {
- return data;
- }
-
-}
\ No newline at end of file
Index: src/java/org/apache/hama/bsp/Task.java
===================================================================
--- src/java/org/apache/hama/bsp/Task.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/Task.java (working copy)
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hama.ipc.BSPPeerProtocol;
-
-/**
- * Base class for tasks.
- */
-public abstract class Task implements Writable {
- public static final Log LOG = LogFactory.getLog(Task.class);
- // //////////////////////////////////////////
- // Fields
- // //////////////////////////////////////////
-
- protected BSPJobID jobId;
- protected String jobFile;
- protected TaskAttemptID taskId;
- protected int partition;
-
- protected LocalDirAllocator lDirAlloc;
-
- public Task() {
- jobId = new BSPJobID();
- taskId = new TaskAttemptID();
- }
-
- public Task(BSPJobID jobId, String jobFile, TaskAttemptID taskId,
- int partition) {
- this.jobId = jobId;
- this.jobFile = jobFile;
- this.taskId = taskId;
- this.partition = partition;
- }
-
- // //////////////////////////////////////////
- // Accessors
- // //////////////////////////////////////////
- public void setJobFile(String jobFile) {
- this.jobFile = jobFile;
- }
-
- public String getJobFile() {
- return jobFile;
- }
-
- public TaskAttemptID getTaskAttemptId() {
- return this.taskId;
- }
-
- public TaskAttemptID getTaskID() {
- return taskId;
- }
-
- /**
- * Get the job name for this task.
- *
- * @return the job name
- */
- public BSPJobID getJobID() {
- return jobId;
- }
-
- /**
- * Get the index of this task within the job.
- *
- * @return the integer part of the task id
- */
- public int getPartition() {
- return partition;
- }
-
- @Override
- public String toString() {
- return taskId.toString();
- }
-
- // //////////////////////////////////////////
- // Writable
- // //////////////////////////////////////////
- @Override
- public void write(DataOutput out) throws IOException {
- jobId.write(out);
- Text.writeString(out, jobFile);
- taskId.write(out);
- out.writeInt(partition);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- jobId.readFields(in);
- jobFile = Text.readString(in);
- taskId.readFields(in);
- partition = in.readInt();
- }
-
- /**
- * Run this task as a part of the named job. This method is executed in the
- * child process.
- *
- * @param bspPeer for communications
- * @param umbilical for communications with GroomServer
- */
- public abstract void run(BSPJob job, BSPPeer bspPeer, BSPPeerProtocol umbilical)
- throws IOException;
-
- public abstract BSPTaskRunner createRunner(GroomServer groom);
-
- public void done(BSPPeerProtocol umbilical) throws IOException {
- umbilical.done(getTaskID(), true);
- }
-
- public abstract BSPJob getConf();
- public abstract void setConf(BSPJob localJobConf);
-
-}
Index: src/java/org/apache/hama/bsp/DispatchTasksDirective.java
===================================================================
--- src/java/org/apache/hama/bsp/DispatchTasksDirective.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/DispatchTasksDirective.java (working copy)
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * Handles the tasks dispatching between the BSPMaster and the GroomServers.
- */
-public final class DispatchTasksDirective extends Directive implements Writable {
-
- public static final Log LOG = LogFactory.getLog(DispatchTasksDirective.class);
-
- private Map groomServerPeers;
- private GroomServerAction[] actions;
-
- public DispatchTasksDirective() {
- super();
- }
-
- public DispatchTasksDirective(Map groomServerPeers,
- GroomServerAction[] actions) {
- super(Directive.Type.Request);
- this.groomServerPeers = groomServerPeers;
- this.actions = actions;
- }
-
- public Map getGroomServerPeers() {
- return this.groomServerPeers;
- }
-
- public GroomServerAction[] getActions() {
- return this.actions;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- if (this.actions == null) {
- WritableUtils.writeVInt(out, 0);
- } else {
- WritableUtils.writeVInt(out, actions.length);
- for (GroomServerAction action : this.actions) {
- WritableUtils.writeEnum(out, action.getActionType());
- action.write(out);
- }
- }
- String[] groomServerNames = groomServerPeers.keySet()
- .toArray(new String[0]);
- WritableUtils.writeCompressedStringArray(out, groomServerNames);
-
- List groomServerAddresses = new ArrayList(
- groomServerNames.length);
- for (String groomName : groomServerNames) {
- groomServerAddresses.add(groomServerPeers.get(groomName));
- }
- WritableUtils.writeCompressedStringArray(out, groomServerAddresses
- .toArray(new String[0]));
-
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- int length = WritableUtils.readVInt(in);
- if (length > 0) {
- this.actions = new GroomServerAction[length];
- for (int i = 0; i < length; ++i) {
- GroomServerAction.ActionType actionType = WritableUtils.readEnum(in,
- GroomServerAction.ActionType.class);
- actions[i] = GroomServerAction.createAction(actionType);
- actions[i].readFields(in);
- }
- } else {
- this.actions = null;
- }
- String[] groomServerNames = WritableUtils.readCompressedStringArray(in);
- String[] groomServerAddresses = WritableUtils.readCompressedStringArray(in);
- groomServerPeers = new HashMap(groomServerNames.length);
-
- for (int i = 0; i < groomServerNames.length; i++) {
- groomServerPeers.put(groomServerNames[i], groomServerAddresses[i]);
- }
- }
-}
Index: src/java/org/apache/hama/bsp/GroomServerManager.java
===================================================================
--- src/java/org/apache/hama/bsp/GroomServerManager.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/GroomServerManager.java (working copy)
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.util.Collection;
-import java.util.Map;
-
-import org.apache.hama.ipc.GroomProtocol;
-
-/**
- * Manages information about the {@link GroomServer}s in the cluster
- * environment. This interface is not intended to be implemented by users.
- */
-interface GroomServerManager {
-
- /**
- * Get the current status of the cluster
- * @param detailed if true then report groom names as well
- * @return summary of the state of the cluster
- */
- ClusterStatus getClusterStatus(boolean detailed);
-
- /**
- * Find WorkerProtocol with corresponded groom server status
- *
- * @param groomId The identification value of GroomServer
- * @return GroomServerStatus
- */
- GroomProtocol findGroomServer(GroomServerStatus status);
-
- /**
- * Find the collection of groom servers.
- *
- * @return Collection of groom servers list.
- */
- Collection findGroomServers();
-
- /**
- * Collection of GroomServerStatus as the key set.
- *
- * @return Collection of GroomServerStatus.
- */
- Collection groomServerStatusKeySet();
-
- /**
- * Registers a JobInProgressListener to GroomServerManager. Therefore,
- * adding a JobInProgress will trigger the jobAdded function.
- * @param the JobInProgressListener listener to be added.
- */
- void addJobInProgressListener(JobInProgressListener listener);
-
- /**
- * Unregisters a JobInProgressListener to GroomServerManager. Therefore,
- * the remove of a JobInProgress will trigger the jobRemoved action.
- * @param the JobInProgressListener to be removed.
- */
- void removeJobInProgressListener(JobInProgressListener listener);
-
- /**
- * Current GroomServer Peers.
- * @return GroomName and PeerName(host:port) in pair.
- */
- Map currentGroomServerPeers();
-}
Index: src/java/org/apache/hama/bsp/GroomServerAction.java
===================================================================
--- src/java/org/apache/hama/bsp/GroomServerAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/GroomServerAction.java (working copy)
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * A generic directive from the {@link org.apache.hama.bsp.BSPMaster}
- * to the {@link org.apache.hama.bsp.GroomServer} to take some 'action'.
- */
-abstract class GroomServerAction implements Writable {
-
- /**
- * Ennumeration of various 'actions' that the {@link BSPMaster}
- * directs the {@link GroomServer} to perform periodically.
- *
- */
- public static enum ActionType {
- /** Launch a new task. */
- LAUNCH_TASK,
-
- /** Kill a task. */
- KILL_TASK,
-
- /** Kill any tasks of this job and cleanup. */
- KILL_JOB,
-
- /** Reinitialize the groomserver. */
- REINIT_GROOM,
-
- /** Ask a task to save its output. */
- COMMIT_TASK
- };
-
- /**
- * A factory-method to create objects of given {@link ActionType}.
- * @param actionType the {@link ActionType} of object to create.
- * @return an object of {@link ActionType}.
- */
- public static GroomServerAction createAction(ActionType actionType) {
- GroomServerAction action = null;
-
- switch (actionType) {
- case LAUNCH_TASK:
- {
- action = new LaunchTaskAction();
- }
- break;
- case KILL_TASK:
- {
- action = new KillTaskAction();
- }
- break;
- case KILL_JOB:
- {
- action = new KillJobAction();
- }
- break;
- case REINIT_GROOM:
- {
- action = new ReinitGroomAction();
- }
- break;
- case COMMIT_TASK:
- {
- action = new CommitTaskAction();
- }
- break;
- }
-
- return action;
- }
-
- private ActionType actionType;
-
- protected GroomServerAction(ActionType actionType) {
- this.actionType = actionType;
- }
-
- /**
- * Return the {@link ActionType}.
- * @return the {@link ActionType}.
- */
- ActionType getActionType() {
- return actionType;
- }
-
- public void write(DataOutput out) throws IOException {
- WritableUtils.writeEnum(out, actionType);
- }
-
- public void readFields(DataInput in) throws IOException {
- actionType = WritableUtils.readEnum(in, ActionType.class);
- }
-}
Index: src/java/org/apache/hama/bsp/ByteMessage.java
===================================================================
--- src/java/org/apache/hama/bsp/ByteMessage.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/ByteMessage.java (working copy)
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * A message that consists of a byte tag and a byte data.
- */
-public class ByteMessage extends BSPMessage {
-
- private byte[] tag;
- private byte[] data;
-
- public ByteMessage() {
- super();
- }
-
- public ByteMessage(byte[] tag, byte[] data) {
- super();
- this.tag = tag;
- this.data = data;
- }
-
- @Override
- public byte[] getTag() {
- return this.tag;
- }
-
- @Override
- public byte[] getData() {
- return this.data;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- this.tag = new byte[in.readInt()];
- in.readFully(tag, 0, this.tag.length);
- this.data = new byte[in.readInt()];
- in.readFully(data, 0, this.data.length);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(tag.length);
- out.write(tag);
- out.writeInt(data.length);
- out.write(data);
- }
-
-}
Index: src/java/org/apache/hama/bsp/CommitTaskAction.java
===================================================================
--- src/java/org/apache/hama/bsp/CommitTaskAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/CommitTaskAction.java (working copy)
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster}
- * to the {@link org.apache.hama.bsp.GroomServer} to commit the output
- * of the task.
- */
-class CommitTaskAction extends GroomServerAction {
- private TaskAttemptID taskId;
-
- public CommitTaskAction() {
- super(ActionType.COMMIT_TASK);
- taskId = new TaskAttemptID();
- }
-
- public CommitTaskAction(TaskAttemptID taskId) {
- super(ActionType.COMMIT_TASK);
- this.taskId = taskId;
- }
-
- public TaskAttemptID getTaskID() {
- return taskId;
- }
-
- public void write(DataOutput out) throws IOException {
- taskId.write(out);
- }
-
- public void readFields(DataInput in) throws IOException {
- taskId.readFields(in);
- }
-}
Index: src/java/org/apache/hama/bsp/RunningJob.java
===================================================================
--- src/java/org/apache/hama/bsp/RunningJob.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/RunningJob.java (working copy)
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-/**
- * RunningJob is the user-interface to query for details on a
- * running BSP job.
- *
- *
- * Clients can get hold of RunningJob via the {@link BSPJobClient}
- * and then query the running-job for details such as name, configuration,
- * progress etc.
- *
- *
- * @see BSPJobClient
- */
-public interface RunningJob {
- /**
- * Get the job identifier.
- *
- * @return the job identifier.
- */
- public BSPJobID getID();
-
- /**
- * Get the name of the job.
- *
- * @return the name of the job.
- */
- public String getJobName();
-
- /**
- * Get the path of the submitted job configuration.
- *
- * @return the path of the submitted job configuration.
- */
- public String getJobFile();
-
- /**
- * Get the progress of the job's tasks, as a float between 0.0 and 1.0.
- * When all bsp tasks have completed, the function returns 1.0.
- *
- * @return the progress of the job's tasks.
- * @throws IOException
- */
- public long progress() throws IOException;
-
- /**
- * Check if the job is finished or not. This is a non-blocking call.
- *
- * @return true if the job is complete, else false.
- * @throws IOException
- */
- public boolean isComplete() throws IOException;
-
- /**
- * Check if the job completed successfully.
- *
- * @return true if the job succeeded, else false.
- * @throws IOException
- */
- public boolean isSuccessful() throws IOException;
-
- /**
- * Blocks until the job is complete.
- *
- * @throws IOException
- */
- public void waitForCompletion() throws IOException;
-
- /**
- * Returns the current state of the Job. {@link JobStatus}
- *
- * @throws IOException
- */
- public int getJobState() throws IOException;
-
- /**
- * Kill the running job. Blocks until all job tasks have been killed as well.
- * If the job is no longer running, it simply returns.
- *
- * @throws IOException
- */
- public void killJob() throws IOException;
-
- /**
- * Kill indicated task attempt.
- *
- * @param taskId the id of the task to be terminated.
- * @param shouldFail if true the task is failed and added to failed tasks
- * list, otherwise it is just killed, w/o affecting job failure
- * status.
- * @throws IOException
- */
- public void killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException;
-
- public long getSuperstepCount() throws IOException;
-}
Index: src/java/org/apache/hama/bsp/Schedulable.java
===================================================================
--- src/java/org/apache/hama/bsp/Schedulable.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/Schedulable.java (working copy)
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-/**
- * This is the class that schedules commands to GroomServer(s)
- */
-public interface Schedulable{
-
- /**
- * Schedule job to designated GroomServer(s) immediately.
- * @param job to be scheduled.
- * @param statuses of GroomServer(s).
- * @throws IOException
- */
- void schedule(JobInProgress job, GroomServerStatus... statuses)
- throws IOException;
-}
Index: src/java/org/apache/hama/bsp/SimpleTaskScheduler.java
===================================================================
--- src/java/org/apache/hama/bsp/SimpleTaskScheduler.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/SimpleTaskScheduler.java (working copy)
@@ -1,183 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hama.ipc.GroomProtocol;
-
-/**
- * A simple task scheduler.
- */
-class SimpleTaskScheduler extends TaskScheduler {
-
- private static final Log LOG = LogFactory.getLog(SimpleTaskScheduler.class);
-
- public static final String WAIT_QUEUE = "waitQueue";
- public static final String PROCESSING_QUEUE = "processingQueue";
- public static final String FINISHED_QUEUE = "finishedQueue";
-
- private QueueManager queueManager;
- private volatile boolean initialized;
- private JobListener jobListener;
- private JobProcessor jobProcessor;
-
- private class JobListener extends JobInProgressListener {
- @Override
- public void jobAdded(JobInProgress job) throws IOException {
- queueManager.initJob(job); // init task
- queueManager.addJob(WAIT_QUEUE, job);
- }
-
- @Override
- public void jobRemoved(JobInProgress job) throws IOException {
- // queueManager.removeJob(WAIT_QUEUE, job);
- queueManager.moveJob(PROCESSING_QUEUE, FINISHED_QUEUE, job);
- }
- }
-
- private class JobProcessor extends Thread implements Schedulable {
- JobProcessor() {
- super("JobProcess");
- }
-
- /**
- * Main logic scheduling task to GroomServer(s). Also, it will move
- * JobInProgress from Wait Queue to Processing Queue.
- */
- public void run() {
- if (false == initialized) {
- throw new IllegalStateException("SimpleTaskScheduler initialization"
- + " is not yet finished!");
- }
- while (initialized) {
- Queue queue = queueManager.findQueue(WAIT_QUEUE);
- if (null == queue) {
- LOG.error(WAIT_QUEUE + " does not exist.");
- throw new NullPointerException(WAIT_QUEUE + " does not exist.");
- }
- // move a job from the wait queue to the processing queue
- JobInProgress j = queue.removeJob();
- queueManager.addJob(PROCESSING_QUEUE, j);
- // schedule
- Collection glist = groomServerManager
- .groomServerStatusKeySet();
- schedule(j, (GroomServerStatus[]) glist
- .toArray(new GroomServerStatus[glist.size()]));
- }
- }
-
- /**
- * Schedule job to designated GroomServer(s) immediately.
- *
- * @param Targeted GroomServer(s).
- * @param Job to be scheduled.
- */
- @Override
- public void schedule(JobInProgress job, GroomServerStatus... statuses) {
- ClusterStatus clusterStatus = groomServerManager.getClusterStatus(false);
- final int numGroomServers = clusterStatus.getGroomServers();
- final ScheduledExecutorService sched = Executors
- .newScheduledThreadPool(statuses.length + 5);
- for (GroomServerStatus status : statuses) {
- sched
- .schedule(new TaskWorker(status, numGroomServers, job), 0, SECONDS);
- }// for
- }
- }
-
- private class TaskWorker implements Runnable {
- private final GroomServerStatus stus;
- private final int groomNum;
- private final JobInProgress jip;
-
- TaskWorker(final GroomServerStatus stus, final int num,
- final JobInProgress jip) {
- this.stus = stus;
- this.groomNum = num;
- this.jip = jip;
- if (null == this.stus)
- throw new NullPointerException("Target groom server is not "
- + "specified.");
- if (-1 == this.groomNum)
- throw new IllegalArgumentException("Groom number is not specified.");
- if (null == this.jip)
- throw new NullPointerException("No job is specified.");
- }
-
- public void run() {
- // obtain tasks
- Task t = jip.obtainNewTask(this.stus, groomNum);
-
- // assembly into actions
- // List tasks = new ArrayList();
- if (jip.getStatus().getRunState() == JobStatus.RUNNING) {
- GroomProtocol worker = groomServerManager.findGroomServer(this.stus);
- try {
- // dispatch() to the groom server
- Directive d1 = new DispatchTasksDirective(groomServerManager
- .currentGroomServerPeers(), new GroomServerAction[] {
- new LaunchTaskAction(t)});
- worker.dispatch(d1);
- } catch (IOException ioe) {
- LOG.error("Fail to dispatch tasks to GroomServer "
- + this.stus.getGroomName(), ioe);
- }
- } else {
- LOG.warn("Currently master only shcedules job in running state. "
- + "This may be refined in the future. JobId:" + jip.getJobID());
- }
- }
- }
-
- public SimpleTaskScheduler() {
- this.jobListener = new JobListener();
- this.jobProcessor = new JobProcessor();
- }
-
- @Override
- public void start() {
- this.queueManager = new QueueManager(getConf()); // TODO: need factory?
- this.queueManager.createFCFSQueue(WAIT_QUEUE);
- this.queueManager.createFCFSQueue(PROCESSING_QUEUE);
- this.queueManager.createFCFSQueue(FINISHED_QUEUE);
- groomServerManager.addJobInProgressListener(this.jobListener);
- this.initialized = true;
- this.jobProcessor.start();
- }
-
- @Override
- public void terminate() {
- this.initialized = false;
- if (null != this.jobListener)
- groomServerManager.removeJobInProgressListener(this.jobListener);
- }
-
- @Override
- public Collection getJobs(String queue) {
- return (queueManager.findQueue(queue)).jobs();
- // return jobQueue;
- }
-}
Index: src/java/org/apache/hama/bsp/DirectiveHandler.java
===================================================================
--- src/java/org/apache/hama/bsp/DirectiveHandler.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/DirectiveHandler.java (working copy)
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * A DirectiveHandler interface.
- */
-public interface DirectiveHandler{
-
- /**
- * Handle directives on demand.
- * @param directive to be handled.
- */
- void handle(Directive directive) throws DirectiveException;
-}
Index: src/java/org/apache/hama/bsp/KillTaskAction.java
===================================================================
--- src/java/org/apache/hama/bsp/KillTaskAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/KillTaskAction.java (working copy)
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster}
- * to the {@link org.apache.hama.bsp.GroomServer} to kill a task.
- */
-class KillTaskAction extends GroomServerAction {
- TaskAttemptID taskId;
-
- public KillTaskAction() {
- super(ActionType.KILL_TASK);
- taskId = new TaskAttemptID();
- }
-
- public KillTaskAction(TaskAttemptID killTaskId) {
- super(ActionType.KILL_TASK);
- this.taskId = killTaskId;
- }
-
- public TaskAttemptID getTaskID() {
- return taskId;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- taskId.write(out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- taskId.readFields(in);
- }
-}
Index: src/java/org/apache/hama/bsp/GroomServerStatus.java
===================================================================
--- src/java/org/apache/hama/bsp/GroomServerStatus.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/GroomServerStatus.java (working copy)
@@ -1,227 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-
-/**
- * A GroomServerStatus is a BSP primitive. Keeps info on a BSPMaster. The
- * BSPMaster maintains a set of the most recent GroomServerStatus objects for
- * each unique GroomServer it knows about.
- */
-public class GroomServerStatus implements Writable {
- public static final Log LOG = LogFactory.getLog(GroomServerStatus.class);
-
- static {
- WritableFactories.setFactory(GroomServerStatus.class,
- new WritableFactory() {
- public Writable newInstance() {
- return new GroomServerStatus();
- }
- });
- }
-
- String groomName;
- String peerName;
- String rpcServer;
- int failures;
- List taskReports;
-
- volatile long lastSeen;
- private int maxTasks;
-
- public GroomServerStatus() {
- // taskReports = new ArrayList();
- taskReports = new CopyOnWriteArrayList();
- }
-
- public GroomServerStatus(String groomName, String peerName,
- List taskReports, int failures, int maxTasks) {
- this(groomName, peerName, taskReports, failures, maxTasks, "");
- }
-
- public GroomServerStatus(String groomName, String peerName,
- List taskReports, int failures, int maxTasks, String rpc) {
- this.groomName = groomName;
- this.peerName = peerName;
- this.taskReports = new ArrayList(taskReports);
- this.failures = failures;
- this.maxTasks = maxTasks;
- this.rpcServer = rpc;
- }
-
- public String getGroomName() {
- return groomName;
- }
-
- /**
- * The host (and port) from where the groom server can be reached.
- *
- * @return The groom server address in the form of "hostname:port"
- */
- public String getPeerName() {
- return peerName;
- }
-
- public String getRpcServer() {
- return rpcServer;
- }
-
- /**
- * Get the current tasks at the GroomServer. Tasks are tracked by a
- * {@link TaskStatus} object.
- *
- * @return a list of {@link TaskStatus} representing the current tasks at the
- * GroomServer.
- */
- public List getTaskReports() {
- return taskReports;
- }
-
- public int getFailures() {
- return failures;
- }
-
- public long getLastSeen() {
- return lastSeen;
- }
-
- public void setLastSeen(long lastSeen) {
- this.lastSeen = lastSeen;
- }
-
- public int getMaxTasks() {
- return maxTasks;
- }
-
- /**
- * Return the current MapTask count
- */
- public int countTasks() {
- int taskCount = 0;
- for (Iterator it = taskReports.iterator(); it.hasNext();) {
- TaskStatus ts = it.next();
- TaskStatus.State state = ts.getRunState();
- if (state == TaskStatus.State.RUNNING
- || state == TaskStatus.State.UNASSIGNED) {
- taskCount++;
- }
- }
-
- return taskCount;
- }
-
- /**
- * For BSPMaster to distinguish between different GroomServers, because
- * BSPMaster stores using GroomServerStatus as key.
- */
- @Override
- public int hashCode() {
- int result = 17;
- result = 37 * result + groomName.hashCode();
- result = 37 * result + peerName.hashCode();
- result = 37 * result + rpcServer.hashCode();
- /*
- * result = 37*result + (int)failures; result = 37*result +
- * taskReports.hashCode(); result = 37*result +
- * (int)(lastSeen^(lastSeen>>>32)); result = 37*result + (int)maxTasks;
- */
- return result;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this)
- return true;
- if (null == o)
- return false;
- if (getClass() != o.getClass())
- return false;
-
- GroomServerStatus s = (GroomServerStatus) o;
- if (!s.groomName.equals(groomName))
- return false;
- if (!s.peerName.equals(peerName))
- return false;
- if (!s.rpcServer.equals(rpcServer))
- return false;
- /*
- * if(s.failures != failures) return false; if(null == s.taskReports){
- * if(null != s.taskReports) return false; }else
- * if(!s.taskReports.equals(taskReports)){ return false; } if(s.lastSeen !=
- * lastSeen) return false; if(s.maxTasks != maxTasks) return false;
- */
- return true;
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- this.groomName = Text.readString(in);
- this.peerName = Text.readString(in);
- this.rpcServer = Text.readString(in);
- this.failures = in.readInt();
- this.maxTasks = in.readInt();
- taskReports.clear();
- int numTasks = in.readInt();
-
- TaskStatus status;
- for (int i = 0; i < numTasks; i++) {
- status = new TaskStatus();
- status.readFields(in);
- taskReports.add(status);
- }
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, groomName);
- Text.writeString(out, peerName);
- Text.writeString(out, rpcServer);
- out.writeInt(failures);
- out.writeInt(maxTasks);
- out.writeInt(taskReports.size());
- for (TaskStatus taskStatus : taskReports) {
- taskStatus.write(out);
- }
- }
-
- public Iterator taskReports() {
- return taskReports.iterator();
- }
-}
Index: src/java/org/apache/hama/bsp/BSPJobClient.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPJobClient.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPJobClient.java (working copy)
@@ -1,679 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Random;
-
-import javax.security.auth.login.LoginException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UnixUserGroupInformation;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.ipc.JobSubmissionProtocol;
-
-/**
- * BSPJobClient is the primary interface for the user-job to interact with the
- * BSPMaster.
- *
- * BSPJobClient provides facilities to submit jobs, track their progress, access
- * component-tasks' reports/logs, get the BSP cluster status information etc.
- */
-public class BSPJobClient extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(BSPJobClient.class);
-
- public static enum TaskStatusFilter {
- NONE, KILLED, FAILED, SUCCEEDED, ALL
- }
-
- private static final long MAX_JOBPROFILE_AGE = 1000 * 2;
-
- class NetworkedJob implements RunningJob {
- JobProfile profile;
- JobStatus status;
- long statustime;
-
- public NetworkedJob(JobStatus job) throws IOException {
- this.status = job;
- this.profile = jobSubmitClient.getJobProfile(job.getJobID());
- this.statustime = System.currentTimeMillis();
- }
-
- /**
- * Some methods rely on having a recent job profile object. Refresh it, if
- * necessary
- */
- synchronized void ensureFreshStatus() throws IOException {
- if (System.currentTimeMillis() - statustime > MAX_JOBPROFILE_AGE) {
- updateStatus();
- }
- }
-
- /**
- * Some methods need to update status immediately. So, refresh immediately
- *
- * @throws IOException
- */
- synchronized void updateStatus() throws IOException {
- this.status = jobSubmitClient.getJobStatus(profile.getJobID());
- this.statustime = System.currentTimeMillis();
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.RunningJob#getID()
- */
- @Override
- public BSPJobID getID() {
- return profile.getJobID();
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.RunningJob#getJobName()
- */
- @Override
- public String getJobName() {
- return profile.getJobName();
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.hama.bsp.RunningJob#getJobFile()
- */
- @Override
- public String getJobFile() {
- return profile.getJobFile();
- }
-
- @Override
- public long progress() throws IOException {
- ensureFreshStatus();
- return status.progress();
- }
-
- /**
- * Returns immediately whether the whole job is done yet or not.
- */
- public synchronized boolean isComplete() throws IOException {
- updateStatus();
- return (status.getRunState() == JobStatus.SUCCEEDED
- || status.getRunState() == JobStatus.FAILED || status.getRunState() == JobStatus.KILLED);
- }
-
- /**
- * True if job completed successfully.
- */
- public synchronized boolean isSuccessful() throws IOException {
- updateStatus();
- return status.getRunState() == JobStatus.SUCCEEDED;
- }
-
- public synchronized long getSuperstepCount() throws IOException {
- ensureFreshStatus();
- return status.getSuperstepCount();
- }
-
- /**
- * Blocks until the job is finished
- */
- public void waitForCompletion() throws IOException {
- while (!isComplete()) {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException ie) {
- }
- }
- }
-
- /**
- * Tells the service to get the state of the current job.
- */
- public synchronized int getJobState() throws IOException {
- updateStatus();
- return status.getRunState();
- }
-
- /**
- * Tells the service to terminate the current job.
- */
- public synchronized void killJob() throws IOException {
- jobSubmitClient.killJob(getID());
- }
-
- @Override
- public void killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException {
- jobSubmitClient.killTask(taskId, shouldFail);
- }
- }
-
- private JobSubmissionProtocol jobSubmitClient = null;
- private Path sysDir = null;
- private FileSystem fs = null;
-
- // job files are world-wide readable and owner writable
- final private static FsPermission JOB_FILE_PERMISSION = FsPermission
- .createImmutable((short) 0644); // rw-r--r--
-
- // job submission directory is world readable/writable/executable
- final static FsPermission JOB_DIR_PERMISSION = FsPermission
- .createImmutable((short) 0777); // rwx-rwx-rwx
-
- public BSPJobClient(Configuration conf) throws IOException {
- setConf(conf);
- init(conf);
- }
-
- public BSPJobClient() {
- }
-
- public void init(Configuration conf) throws IOException {
- String masterAdress = conf.get("bsp.master.address");
- if (masterAdress != null && !masterAdress.equals("local")) {
- this.jobSubmitClient = (JobSubmissionProtocol) RPC.getProxy(
- JobSubmissionProtocol.class, JobSubmissionProtocol.versionID,
- BSPMaster.getAddress(conf), conf,
- NetUtils.getSocketFactory(conf, JobSubmissionProtocol.class));
- } else {
- LOG.debug("Using local BSP runner.");
- this.jobSubmitClient = new LocalBSPRunner(conf);
- }
- }
-
- /**
- * Close the JobClient.
- */
- public synchronized void close() throws IOException {
- RPC.stopProxy(jobSubmitClient);
- }
-
- /**
- * Get a filesystem handle. We need this to prepare jobs for submission to the
- * BSP system.
- *
- * @return the filesystem handle.
- */
- public synchronized FileSystem getFs() throws IOException {
- if (this.fs == null) {
- Path sysDir = getSystemDir();
- this.fs = sysDir.getFileSystem(getConf());
- }
- return fs;
- }
-
- /**
- * Gets the jobs that are submitted.
- *
- * @return array of {@link JobStatus} for the submitted jobs.
- * @throws IOException
- */
- public JobStatus[] getAllJobs() throws IOException {
- return jobSubmitClient.getAllJobs();
- }
-
- /**
- * Gets the jobs that are not completed and not failed.
- *
- * @return array of {@link JobStatus} for the running/to-be-run jobs.
- * @throws IOException
- */
- public JobStatus[] jobsToComplete() throws IOException {
- return jobSubmitClient.jobsToComplete();
- }
-
- private UnixUserGroupInformation getUGI(Configuration conf)
- throws IOException {
- UnixUserGroupInformation ugi = null;
- try {
- ugi = UnixUserGroupInformation.login(conf, true);
- } catch (LoginException e) {
- throw (IOException) (new IOException(
- "Failed to get the current user's information.").initCause(e));
- }
- return ugi;
- }
-
- /**
- * Submit a job to the BSP system. This returns a handle to the
- * {@link RunningJob} which can be used to track the running-job.
- *
- * @param job the job configuration.
- * @return a handle to the {@link RunningJob} which can be used to track the
- * running-job.
- * @throws FileNotFoundException
- * @throws IOException
- */
- public RunningJob submitJob(BSPJob job) throws FileNotFoundException,
- IOException {
- return submitJobInternal(job);
- }
-
- static Random r = new Random();
-
- public RunningJob submitJobInternal(BSPJob job) throws IOException {
- BSPJobID jobId = jobSubmitClient.getNewJobId();
-
- Path submitJobDir = new Path(getSystemDir(), "submit_"
- + Integer.toString(Math.abs(r.nextInt()), 36));
- Path submitJarFile = new Path(submitJobDir, "job.jar");
- Path submitJobFile = new Path(submitJobDir, "job.xml");
- LOG.debug("BSPJobClient.submitJobDir: " + submitJobDir);
-
- /*
- * set this user's id in job configuration, so later job files can be
- * accessed using this user's id
- */
- UnixUserGroupInformation ugi = getUGI(job.getConf());
-
- ClusterStatus clusterStatus = getClusterStatus(true);
-
- // check the number of BSP tasks
- int tasks = job.getNumBspTask();
- int maxTasks = clusterStatus.getMaxTasks();
-
- if (tasks <= 0 || tasks > maxTasks) {
- LOG.warn("The number of tasks you've entered was invalid. Using default value of "
- + maxTasks + "!");
- job.setNumBspTask(maxTasks);
- }
-
- // Create a number of filenames in the BSPMaster's fs namespace
- FileSystem fs = getFs();
- fs.delete(submitJobDir, true);
- submitJobDir = fs.makeQualified(submitJobDir);
- submitJobDir = new Path(submitJobDir.toUri().getPath());
- FsPermission bspSysPerms = new FsPermission(JOB_DIR_PERMISSION);
- FileSystem.mkdirs(fs, submitJobDir, bspSysPerms);
- fs.mkdirs(submitJobDir);
- short replication = (short) job.getInt("bsp.submit.replication", 10);
-
- String originalJarPath = job.getJar();
-
- if (originalJarPath != null) { // copy jar to BSPMaster's fs
- // use jar name if job is not named.
- if ("".equals(job.getJobName())) {
- job.setJobName(new Path(originalJarPath).getName());
- }
- job.setJar(submitJarFile.toString());
- fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile);
-
- fs.setReplication(submitJarFile, replication);
- fs.setPermission(submitJarFile, new FsPermission(JOB_FILE_PERMISSION));
- } else {
- LOG.warn("No job jar file set. User classes may not be found. "
- + "See BSPJob#setJar(String) or check Your jar file.");
- }
-
- // Set the user's name and working directory
- job.setUser(ugi.getUserName());
- if (ugi.getGroupNames().length > 0) {
- job.set("group.name", ugi.getGroupNames()[0]);
- }
- if (job.getWorkingDirectory() == null) {
- job.setWorkingDirectory(fs.getWorkingDirectory());
- }
-
- // Write job file to BSPMaster's fs
- FSDataOutputStream out = FileSystem.create(fs, submitJobFile,
- new FsPermission(JOB_FILE_PERMISSION));
-
- try {
- job.writeXml(out);
- } finally {
- out.close();
- }
-
- //
- // Now, actually submit the job (using the submit name)
- //
- JobStatus status = jobSubmitClient.submitJob(jobId,
- submitJobFile.toString());
- if (status != null) {
- return new NetworkedJob(status);
- } else {
- throw new IOException("Could not launch job");
- }
- }
-
- /**
- * Monitor a job and print status in real-time as progress is made and tasks
- * fail.
- *
- * @param job
- * @param info
- * @return true, if job is successful
- * @throws IOException
- * @throws InterruptedException
- */
- public boolean monitorAndPrintJob(BSPJob job, RunningJob info)
- throws IOException, InterruptedException {
-
- String lastReport = null;
- LOG.info("Running job: " + info.getID());
-
- while (!job.isComplete()) {
- Thread.sleep(3000);
- long step = job.progress();
- String report = "Current supersteps number: " + step;
-
- if (!report.equals(lastReport)) {
- LOG.info(report);
- lastReport = report;
- }
- }
-
- LOG.info("The total number of supersteps: " + info.getSuperstepCount());
- return job.isSuccessful();
- }
-
- /**
- * Grab the bspmaster system directory path where job-specific files are to be
- * placed.
- *
- * @return the system directory where job-specific files are to be placed.
- */
- public Path getSystemDir() {
- if (sysDir == null) {
- sysDir = new Path(jobSubmitClient.getSystemDir());
- }
- return sysDir;
- }
-
- public static void runJob(BSPJob job) throws FileNotFoundException,
- IOException {
- BSPJobClient jc = new BSPJobClient(job.getConf());
-
- if (job.getNumBspTask() == 0
- || job.getNumBspTask() > jc.getClusterStatus(false).getMaxTasks()) {
- job.setNumBspTask(jc.getClusterStatus(false).getMaxTasks());
- }
-
- RunningJob running = jc.submitJobInternal(job);
- BSPJobID jobId = running.getID();
- LOG.info("Running job: " + jobId.toString());
-
- while (true) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
-
- if (running.isComplete()) {
- break;
- }
-
- running = jc.getJob(jobId);
- }
-
- LOG.info("Job complete: " + jobId);
- LOG.info("The total number of supersteps: " + running.getSuperstepCount());
-
- // TODO if error found, kill job
- // running.killJob();
- jc.close();
- }
-
- /**
- * Get an RunningJob object to track an ongoing job. Returns null if the id
- * does not correspond to any known job.
- *
- * @throws IOException
- */
- private RunningJob getJob(BSPJobID jobId) throws IOException {
- JobStatus status = jobSubmitClient.getJobStatus(jobId);
- if (status != null) {
- return new NetworkedJob(status);
- } else {
- return null;
- }
- }
-
- /**
- * Get status information about the BSP cluster
- *
- * @param detailed if true then get a detailed status including the
- * groomserver names
- *
- * @return the status information about the BSP cluster as an object of
- * {@link ClusterStatus}.
- *
- * @throws IOException
- */
- public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
- return jobSubmitClient.getClusterStatus(detailed);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- int exitCode = -1;
- if (args.length < 1) {
- displayUsage("");
- return exitCode;
- }
-
- // process arguments
- String cmd = args[0];
- boolean listJobs = false;
- boolean listAllJobs = false;
- boolean listActiveGrooms = false;
- boolean killJob = false;
- boolean submitJob = false;
- boolean getStatus = false;
- String submitJobFile = null;
- String jobid = null;
-
- HamaConfiguration conf = new HamaConfiguration(getConf());
- init(conf);
-
- if ("-list".equals(cmd)) {
- if (args.length != 1 && !(args.length == 2 && "all".equals(args[1]))) {
- displayUsage(cmd);
- return exitCode;
- }
- if (args.length == 2 && "all".equals(args[1])) {
- listAllJobs = true;
- } else {
- listJobs = true;
- }
- } else if ("-list-active-grooms".equals(cmd)) {
- if (args.length != 1) {
- displayUsage(cmd);
- return exitCode;
- }
- listActiveGrooms = true;
- } else if ("-submit".equals(cmd)) {
- if (args.length == 1) {
- displayUsage(cmd);
- return exitCode;
- }
-
- submitJob = true;
- submitJobFile = args[1];
- } else if ("-kill".equals(cmd)) {
- if (args.length == 1) {
- displayUsage(cmd);
- return exitCode;
- }
- killJob = true;
- jobid = args[1];
-
- } else if ("-status".equals(cmd)) {
- if (args.length != 2) {
- displayUsage(cmd);
- return exitCode;
- }
- jobid = args[1];
- getStatus = true;
-
- // TODO Later, below functions should be implemented
- // with the Fault Tolerant mechanism.
- } else if ("-list-attempt-ids".equals(cmd)) {
- System.out.println("This function is not implemented yet.");
- return exitCode;
- } else if ("-kill-task".equals(cmd)) {
- System.out.println("This function is not implemented yet.");
- return exitCode;
- } else if ("-fail-task".equals(cmd)) {
- System.out.println("This function is not implemented yet.");
- return exitCode;
- }
-
- BSPJobClient jc = new BSPJobClient(new HamaConfiguration());
- if (listJobs) {
- listJobs();
- exitCode = 0;
- } else if (listAllJobs) {
- listAllJobs();
- exitCode = 0;
- } else if (listActiveGrooms) {
- listActiveGrooms();
- exitCode = 0;
- } else if (submitJob) {
- HamaConfiguration tConf = new HamaConfiguration(new Path(submitJobFile));
- RunningJob job = jc.submitJob(new BSPJob(tConf));
- System.out.println("Created job " + job.getID().toString());
- } else if (killJob) {
- RunningJob job = jc.getJob(BSPJobID.forName(jobid));
- if (job == null) {
- System.out.println("Could not find job " + jobid);
- } else {
- job.killJob();
- System.out.println("Killed job " + jobid);
- }
- exitCode = 0;
- } else if (getStatus) {
- RunningJob job = jc.getJob(BSPJobID.forName(jobid));
- if (job == null) {
- System.out.println("Could not find job " + jobid);
- } else {
- JobStatus jobStatus = jobSubmitClient.getJobStatus(job.getID());
- System.out.println("Job name: " + job.getJobName());
- System.out.printf("States are:\n\tRunning : 1\tSucceded : 2"
- + "\tFailed : 3\tPrep : 4\n");
- System.out.printf("%s\t%d\t%d\t%s\n", jobStatus.getJobID(),
- jobStatus.getRunState(), jobStatus.getStartTime(),
- jobStatus.getUsername());
-
- exitCode = 0;
- }
- }
-
- return 0;
- }
-
- /**
- * Display usage of the command-line tool and terminate execution
- */
- private void displayUsage(String cmd) {
- String prefix = "Usage: hama job ";
- String taskStates = "running, completed";
- if ("-submit".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " ]");
- } else if ("-status".equals(cmd) || "-kill".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " ]");
- } else if ("-list".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " [all]]");
- } else if ("-kill-task".equals(cmd) || "-fail-task".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " ]");
- } else if ("-list-active-grooms".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + "]");
- } else if ("-list-attempt-ids".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " ]. "
- + "Valid values for are " + taskStates);
- } else {
- System.err.printf(prefix + " \n");
- System.err.printf("\t[-submit ]\n");
- System.err.printf("\t[-status ]\n");
- System.err.printf("\t[-kill ]\n");
- System.err.printf("\t[-list [all]]\n");
- System.err.printf("\t[-list-active-grooms]\n");
- System.err.println("\t[-list-attempt-ids " + "]\n");
- System.err.printf("\t[-kill-task ]\n");
- System.err.printf("\t[-fail-task ]\n\n");
- }
- }
-
- /**
- * Dump a list of currently running jobs
- *
- * @throws IOException
- */
- private void listJobs() throws IOException {
- JobStatus[] jobs = jobsToComplete();
- if (jobs == null)
- jobs = new JobStatus[0];
-
- System.out.printf("%d jobs currently running\n", jobs.length);
- displayJobList(jobs);
- }
-
- /**
- * Dump a list of all jobs submitted.
- *
- * @throws IOException
- */
- private void listAllJobs() throws IOException {
- JobStatus[] jobs = getAllJobs();
- if (jobs == null)
- jobs = new JobStatus[0];
- System.out.printf("%d jobs submitted\n", jobs.length);
- System.out.printf("States are:\n\tRunning : 1\tSucceded : 2"
- + "\tFailed : 3\tPrep : 4\n");
- displayJobList(jobs);
- }
-
- void displayJobList(JobStatus[] jobs) {
- System.out.printf("JobId\tState\tStartTime\tUserName\n");
- for (JobStatus job : jobs) {
- System.out.printf("%s\t%d\t%d\t%s\n", job.getJobID(), job.getRunState(),
- job.getStartTime(), job.getUsername());
- }
- }
-
- /**
- * Display the list of active groom servers
- */
- private void listActiveGrooms() throws IOException {
- ClusterStatus c = jobSubmitClient.getClusterStatus(true);
- Map grooms = c.getActiveGroomNames();
- for (String groomName : grooms.keySet()) {
- System.out.println(groomName);
- }
- }
-
- /**
- */
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new BSPJobClient(), args);
- System.exit(res);
- }
-}
Index: src/java/org/apache/hama/bsp/TaskAttemptID.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskAttemptID.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskAttemptID.java (working copy)
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * TaskAttemptID is a unique identifier for a task attempt.
- */
-public class TaskAttemptID extends ID {
- protected static final String ATTEMPT = "attempt";
- private TaskID taskId;
-
- public TaskAttemptID(TaskID taskId, int id) {
- super(id);
- if (taskId == null) {
- throw new IllegalArgumentException("taskId cannot be null");
- }
- this.taskId = taskId;
- }
-
- public TaskAttemptID(String jtIdentifier, int jobId, int taskId, int id) {
- this(new TaskID(jtIdentifier, jobId, taskId), id);
- }
-
- public TaskAttemptID() {
- taskId = new TaskID();
- }
-
- public BSPJobID getJobID() {
- return taskId.getJobID();
- }
-
- public TaskID getTaskID() {
- return taskId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- TaskAttemptID that = (TaskAttemptID) o;
- return this.taskId.equals(that.taskId);
- }
-
- protected StringBuilder appendTo(StringBuilder builder) {
- return taskId.appendTo(builder).append(SEPARATOR).append(id);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- taskId.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- taskId.write(out);
- }
-
- @Override
- public int hashCode() {
- return taskId.hashCode() * 5 + id;
- }
-
- @Override
- public int compareTo(ID o) {
- TaskAttemptID that = (TaskAttemptID) o;
- int tipComp = this.taskId.compareTo(that.taskId);
- if (tipComp == 0) {
- return this.id - that.id;
- } else
- return tipComp;
- }
-
- @Override
- public String toString() {
- return appendTo(new StringBuilder(ATTEMPT)).toString();
- }
-
- public static TaskAttemptID forName(String str)
- throws IllegalArgumentException {
- if (str == null)
- return null;
- try {
- String[] parts = str.split(Character.toString(SEPARATOR));
- if (parts.length == 5) {
- if (parts[0].equals(ATTEMPT)) {
- return new TaskAttemptID(parts[1], Integer.parseInt(parts[2]),
- Integer.parseInt(parts[3]), Integer.parseInt(parts[4]));
- }
- }
- } catch (Exception ex) {
- // fall below
- }
- throw new IllegalArgumentException("TaskAttemptId string : " + str
- + " is not properly formed");
- }
-}
Index: src/java/org/apache/hama/bsp/package.html
===================================================================
--- src/java/org/apache/hama/bsp/package.html (revision 1152138)
+++ src/java/org/apache/hama/bsp/package.html (working copy)
@@ -1,23 +0,0 @@
-
-
-
-
-
-BSP computing framework.
-
-
Index: src/java/org/apache/hama/bsp/BSPJob.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPJob.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPJob.java (working copy)
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.Enumeration;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hama.HamaConfiguration;
-
-/**
- * A BSP job configuration.
- *
- * BSPJob is the primary interface for a user to describe a BSP job to the Hama
- * BSP framework for execution.
- */
-public class BSPJob extends BSPJobContext {
- public static enum JobState {
- DEFINE, RUNNING
- };
-
- private JobState state = JobState.DEFINE;
- private BSPJobClient jobClient;
- private RunningJob info;
-
- public BSPJob() throws IOException {
- this(new HamaConfiguration());
- }
-
- public BSPJob(HamaConfiguration conf) throws IOException {
- super(conf, null);
- jobClient = new BSPJobClient(conf);
- }
-
- public BSPJob(HamaConfiguration conf, String jobName) throws IOException {
- this(conf);
- setJobName(jobName);
- }
-
- public BSPJob(BSPJobID jobID, String jobFile) throws IOException {
- super(new Path(jobFile), jobID);
- }
-
- public BSPJob(HamaConfiguration conf, Class> exampleClass)
- throws IOException {
- this(conf);
- setJarByClass(exampleClass);
- }
-
- public BSPJob(HamaConfiguration conf, int numPeer) {
- super(conf, null);
- this.setNumBspTask(numPeer);
- }
-
- private void ensureState(JobState state) throws IllegalStateException {
- if (state != this.state) {
- throw new IllegalStateException("Job in state " + this.state
- + " instead of " + state);
- }
- }
-
- // /////////////////////////////////////
- // Setter for Job Submission
- // /////////////////////////////////////
- public void setWorkingDirectory(Path dir) throws IOException {
- ensureState(JobState.DEFINE);
- dir = new Path(getWorkingDirectory(), dir);
- conf.set(WORKING_DIR, dir.toString());
- }
-
- /**
- * Set the BSP algorithm class for the job.
- *
- * @param cls
- * @throws IllegalStateException
- */
- public void setBspClass(Class extends BSP> cls)
- throws IllegalStateException {
- ensureState(JobState.DEFINE);
- conf.setClass(WORK_CLASS_ATTR, cls, BSP.class);
- }
-
- @SuppressWarnings("unchecked")
- public Class extends BSP> getBspClass() {
- return (Class extends BSP>) conf.getClass(WORK_CLASS_ATTR, BSP.class);
- }
-
- public void setJar(String jar) {
- conf.set("bsp.jar", jar);
- }
-
- public void setJarByClass(Class> cls) {
- String jar = findContainingJar(cls);
- if (jar != null) {
- conf.set("bsp.jar", jar);
- }
- }
-
- private static String findContainingJar(Class> my_class) {
- ClassLoader loader = my_class.getClassLoader();
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
- try {
- for (Enumeration itr = loader.getResources(class_file); itr
- .hasMoreElements();) {
-
- URL url = itr.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- return toReturn.replaceAll("!.*$", "");
- }
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return null;
- }
-
- public void setJobName(String name) throws IllegalStateException {
- ensureState(JobState.DEFINE);
- conf.set("bsp.job.name", name);
- }
-
- public void setInputPath(HamaConfiguration conf, Path iNPUTPATH) {
-
- }
-
- public void setUser(String user) {
- conf.set("user.name", user);
- }
-
- // /////////////////////////////////////
- // Methods for Job Control
- // /////////////////////////////////////
- public long progress() throws IOException {
- ensureState(JobState.RUNNING);
- return info.progress();
- }
-
- public boolean isComplete() throws IOException {
- ensureState(JobState.RUNNING);
- return info.isComplete();
- }
-
- public boolean isSuccessful() throws IOException {
- ensureState(JobState.RUNNING);
- return info.isSuccessful();
- }
-
- public void killJob() throws IOException {
- ensureState(JobState.RUNNING);
- info.killJob();
- }
-
- public void killTask(TaskAttemptID taskId) throws IOException {
- ensureState(JobState.RUNNING);
- info.killTask(taskId, false);
- }
-
- public void failTask(TaskAttemptID taskId) throws IOException {
- ensureState(JobState.RUNNING);
- info.killTask(taskId, true);
- }
-
- public void submit() throws IOException, InterruptedException {
- ensureState(JobState.DEFINE);
- info = jobClient.submitJobInternal(this);
- state = JobState.RUNNING;
- }
-
- public boolean waitForCompletion(boolean verbose) throws IOException,
- InterruptedException, ClassNotFoundException {
- if (state == JobState.DEFINE) {
- submit();
- }
- if (verbose) {
- jobClient.monitorAndPrintJob(this, info);
- } else {
- info.waitForCompletion();
- }
- return isSuccessful();
- }
-
- public void set(String name, String value) {
- conf.set(name, value);
- }
-
- public void setNumBspTask(int tasks) {
- conf.setInt("bsp.peers.num", tasks);
- }
-
- public int getNumBspTask() {
- return conf.getInt("bsp.peers.num", 0);
- }
-}
Index: src/java/org/apache/hama/bsp/PeerNames.java
===================================================================
--- src/java/org/apache/hama/bsp/PeerNames.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/PeerNames.java (working copy)
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-
-/**
- *
- */
-public class PeerNames implements Writable {
- Collection allPeers;
-
- public PeerNames() {
- this.allPeers = new ArrayList();
- }
-
- public PeerNames(Collection allPeers) {
- this.allPeers = allPeers;
- }
-
- public Collection getAllPeerNames() {
- return allPeers;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(allPeers.size());
- for (String peerName : allPeers) {
- Text.writeString(out, peerName);
- }
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- int peersNum = in.readInt();
- for (int i = 0; i < peersNum; i++) {
- allPeers.add(Text.readString(in));
- }
- }
-
-}
Index: src/java/org/apache/hama/bsp/TaskID.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskID.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskID.java (working copy)
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-/**
- * TaskID represents the immutable and unique identifier for a BSP Task.
- */
-public class TaskID extends ID {
- protected static final String TASK = "task";
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(6);
- }
-
- private BSPJobID jobId;
-
- public TaskID(BSPJobID jobId, int id) {
- super(id);
- if (jobId == null) {
- throw new IllegalArgumentException("jobId cannot be null");
- }
- this.jobId = jobId;
- }
-
- public TaskID(String jtIdentifier, int jobId, int id) {
- this(new BSPJobID(jtIdentifier, jobId), id);
- }
-
- public TaskID() {
- jobId = new BSPJobID();
- }
-
- /** Returns the {@link BSPJobID} object that this tip belongs to */
- public BSPJobID getJobID() {
- return jobId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- TaskID that = (TaskID) o;
- return this.jobId.equals(that.jobId);
- }
-
- @Override
- public int compareTo(ID o) {
- TaskID that = (TaskID) o;
- int jobComp = this.jobId.compareTo(that.jobId);
- if (jobComp == 0) {
- return this.id - that.id;
- } else {
- return jobComp;
- }
- }
-
- @Override
- public String toString() {
- return appendTo(new StringBuilder(TASK)).toString();
- }
-
- protected StringBuilder appendTo(StringBuilder builder) {
- return jobId.appendTo(builder).append(SEPARATOR)
- .append(idFormat.format(id));
- }
-
- @Override
- public int hashCode() {
- return jobId.hashCode() * 524287 + id;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- jobId.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- jobId.write(out);
- }
-
- public static TaskID forName(String str) throws IllegalArgumentException {
- if (str == null)
- return null;
- try {
- String[] parts = str.split("_");
- if (parts.length == 5) {
- if (parts[0].equals(TASK)) {
- return new TaskID(parts[1], Integer.parseInt(parts[2]), Integer
- .parseInt(parts[4]));
- }
- }
- } catch (Exception ex) {
- }
- throw new IllegalArgumentException("TaskId string : " + str
- + " is not properly formed");
- }
-}
Index: src/java/org/apache/hama/bsp/BSPPeerInterface.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPPeerInterface.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPPeerInterface.java (working copy)
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.hama.Constants;
-import org.apache.hama.ipc.HamaRPCProtocolVersion;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * BSP communication interface.
- */
-public interface BSPPeerInterface extends HamaRPCProtocolVersion, Closeable,
- Constants {
-
- /**
- * Send a data with a tag to another BSPSlave corresponding to hostname.
- * Messages sent by this method are not guaranteed to be received in a sent
- * order.
- *
- * @param peerName
- * @param msg
- * @throws IOException
- */
- public void send(String peerName, BSPMessage msg) throws IOException;
-
- /**
- * Puts a message to local queue.
- *
- * @param msg
- * @throws IOException
- */
- public void put(BSPMessage msg) throws IOException;
-
- /**
- * Puts a bundle of messages to local queue.
- *
- * @param messages
- * @throws IOException
- */
- public void put(BSPMessageBundle messages) throws IOException;
-
- /**
- * @return A message from the peer's received messages queue (a FIFO).
- * @throws IOException
- */
- public BSPMessage getCurrentMessage() throws IOException;
-
- /**
- * @return The number of messages in the peer's received messages queue.
- */
- public int getNumCurrentMessages();
-
- /**
- * Barrier Synchronization.
- *
- * Sends all the messages in the outgoing message queues to the corresponding
- * remote peers.
- *
- * @throws InterruptedException
- * @throws KeeperException
- */
- public void sync() throws IOException, KeeperException, InterruptedException;
-
- /**
- * @return the count of current super-step
- */
- public long getSuperstepCount();
-
- /**
- * @return The name of this peer in the format "hostname:port".
- */
- public String getPeerName();
-
- /**
- * @return The names of all the peers executing tasks from the same job
- * (including this peer).
- */
- public String[] getAllPeerNames();
-
- /**
- * Clears all queues entries.
- */
- public void clear();
-}
Index: src/java/org/apache/hama/bsp/JobChangeEvent.java
===================================================================
--- src/java/org/apache/hama/bsp/JobChangeEvent.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobChangeEvent.java (working copy)
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * {@link JobChangeEvent} is used to capture state changes in a job. A job can
- * change its state w.r.t priority, progress, run-state etc.
- */
-abstract class JobChangeEvent {
- private JobInProgress jip;
-
- JobChangeEvent(JobInProgress jip) {
- this.jip = jip;
- }
-
- /**
- * Get the job object for which the change is reported
- */
- JobInProgress getJobInProgress() {
- return jip;
- }
-}
Index: src/java/org/apache/hama/bsp/BSPMessageBundle.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPMessageBundle.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPMessageBundle.java (working copy)
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * BSPMessageBundle stores a group of BSPMessages so that they can be sent in
- * batch rather than individually.
- *
- */
-public class BSPMessageBundle implements Writable {
-
- public static final Log LOG = LogFactory.getLog(BSPMessageBundle.class);
-
- private HashMap> messages = new HashMap>();
- private HashMap> classCache = new HashMap>();
-
- public BSPMessageBundle() {
- }
-
- /**
- * Add message to this bundle.
- *
- * @param message BSPMessage to add.
- */
- public void addMessage(BSPMessage message) {
- String className = message.getClass().getName();
- if (!messages.containsKey(className)) {
- // use linked list because we're just iterating over them
- LinkedList list = new LinkedList();
- list.add(message);
- messages.put(className, list);
- } else {
- messages.get(className).add(message);
- }
- }
-
- public List getMessages() {
- // here we use an arraylist, because we know the size and outside may need
- // random access
- List mergeList = new ArrayList(messages.size());
- for (LinkedList c : messages.values()) {
- mergeList.addAll(c);
- }
- return mergeList;
- }
-
- public void write(DataOutput out) throws IOException {
- // writes the k/v mapping size
- out.writeInt(messages.size());
- if (messages.size() > 0) {
- for (Entry> entry : messages.entrySet()) {
- out.writeUTF(entry.getKey());
- LinkedList messageList = entry.getValue();
- out.writeInt(messageList.size());
- for (BSPMessage msg : messageList) {
- msg.write(out);
- }
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- public void readFields(DataInput in) throws IOException {
- if (messages == null) {
- messages = new HashMap>();
- }
- int numMessages = in.readInt();
- if (numMessages > 0) {
- for (int entries = 0; entries < numMessages; entries++) {
- String className = in.readUTF();
- int size = in.readInt();
- LinkedList msgList = new LinkedList();
- messages.put(className, msgList);
-
- Class extends BSPMessage> clazz = null;
- if ((clazz = classCache.get(className)) == null) {
- try {
- clazz = (Class extends BSPMessage>) Class.forName(className);
- classCache.put(className, clazz);
- } catch (ClassNotFoundException e) {
- LOG.error("Class was not found.",e);
- }
- }
-
- for (int i = 0; i < size; i++) {
- BSPMessage msg = ReflectionUtils.newInstance(clazz, null);
- msg.readFields(in);
- msgList.add(msg);
- }
-
- }
- }
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSP.java
===================================================================
--- src/java/org/apache/hama/bsp/BSP.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSP.java (working copy)
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * This class provides an abstract implementation of the BSP interface.
- */
-public abstract class BSP implements BSPInterface {
-}
Index: src/java/org/apache/hama/bsp/KillJobAction.java
===================================================================
--- src/java/org/apache/hama/bsp/KillJobAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/KillJobAction.java (working copy)
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the
- * {@link org.apache.hama.bsp.GroomServer} to kill the task of a job and cleanup
- * resources.
- */
-class KillJobAction extends GroomServerAction {
- String jobId;
-
- public KillJobAction() {
- super(ActionType.KILL_JOB);
- jobId = new String();
- }
-
- public KillJobAction(String killJobId) {
- super(ActionType.KILL_JOB);
- this.jobId = killJobId;
- }
-
- public String getJobID() {
- return jobId;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, jobId);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- jobId = Text.readString(in);
- }
-
-}
Index: src/java/org/apache/hama/bsp/Queue.java
===================================================================
--- src/java/org/apache/hama/bsp/Queue.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/Queue.java (working copy)
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.util.Collection;
-
-/**
- * Job Queue interface.
- *
- * @param
- */
-public interface Queue{
-
- /**
- * The queue name.
- * @return the name of current queue.
- */
- String getName();
-
- /**
- * Add a job to a queue.
- * @param job to be added to the queue.
- */
- void addJob(T job);
-
- /**
- * Remove a job from the queue.
- * @param job to be removed from the queue.
- */
- void removeJob(T job);
-
- /**
- * Get a job
- * @return job that is removed from the queue.
- */
- T removeJob();
-
- /**
- * Return all data stored in this queue.
- * @return Collection of jobs.
- */
- public Collection jobs();
-
-}
Index: src/java/org/apache/hama/bsp/LocalBSPRunner.java
===================================================================
--- src/java/org/apache/hama/bsp/LocalBSPRunner.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/LocalBSPRunner.java (working copy)
@@ -1,356 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadPoolExecutor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hama.bsp.BSPMaster.State;
-import org.apache.hama.ipc.JobSubmissionProtocol;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * A multithreaded local BSP runner that can be used for debugging BSP's. It
- * uses the working directory "/user/hama/bsp/" and starts runners based on the
- * number of the machines core.
- *
- */
-public class LocalBSPRunner implements JobSubmissionProtocol {
- public static final Log LOG = LogFactory.getLog(LocalBSPRunner.class);
-
- private static final String IDENTIFIER = "localrunner";
- private static String WORKING_DIR = "/user/hama/bsp/";
- protected static volatile ThreadPoolExecutor threadPool;
- protected static int threadPoolSize;
- protected static final LinkedList> futureList = new LinkedList>();
- protected static CyclicBarrier barrier;
-
- static {
- threadPoolSize = Runtime.getRuntime().availableProcessors();
- barrier = new CyclicBarrier(threadPoolSize);
- threadPool = (ThreadPoolExecutor) Executors
- .newFixedThreadPool(threadPoolSize);
- }
-
- protected HashMap localGrooms = new HashMap();
- protected String jobFile;
- protected String jobName;
-
- protected JobStatus currentJobStatus;
-
- protected Configuration conf;
- protected FileSystem fs;
-
- public LocalBSPRunner(Configuration conf) throws IOException {
- super();
- this.conf = conf;
- this.fs = FileSystem.get(conf);
- String path = conf.get("bsp.local.dir");
- if (path != null && !path.isEmpty())
- WORKING_DIR = path;
-
- threadPoolSize = conf.getInt("bsp.local.tasks.maximum", 20);
- threadPool = (ThreadPoolExecutor) Executors
- .newFixedThreadPool(threadPoolSize);
- barrier = new CyclicBarrier(threadPoolSize);
-
- for (int i = 0; i < threadPoolSize; i++) {
- String name = IDENTIFIER + " " + i;
- localGrooms.put(name, new LocalGroom(name));
- }
-
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return 3;
- }
-
- @Override
- public BSPJobID getNewJobId() throws IOException {
- return new BSPJobID(IDENTIFIER, 1);
- }
-
- @Override
- public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException {
- this.jobFile = jobFile;
- BSPJob job = new BSPJob(jobID, jobFile);
- job.setNumBspTask(threadPoolSize);
- this.jobName = job.getJobName();
- currentJobStatus = new JobStatus(jobID, System.getProperty("user.name"), 0,
- JobStatus.RUNNING);
- for (int i = 0; i < threadPoolSize; i++) {
- String name = IDENTIFIER + " " + i;
- LocalGroom localGroom = new LocalGroom(name);
- localGrooms.put(name, localGroom);
- futureList.add(threadPool.submit(new BSPRunner(conf, job, ReflectionUtils
- .newInstance(job.getBspClass(), conf), localGroom)));
- }
- new Thread(new ThreadObserver(currentJobStatus)).start();
- return currentJobStatus;
- }
-
- @Override
- public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
- Map map = new HashMap();
- for (Entry entry : localGrooms.entrySet()) {
- map.put(entry.getKey(), entry.getValue().getPeerName());
- }
- return new ClusterStatus(map, threadPoolSize, threadPoolSize, State.RUNNING);
- }
-
- @Override
- public JobProfile getJobProfile(BSPJobID jobid) throws IOException {
- return new JobProfile(System.getProperty("user.name"), jobid, jobFile,
- jobName);
- }
-
- @Override
- public JobStatus getJobStatus(BSPJobID jobid) throws IOException {
- if (currentJobStatus == null) {
- currentJobStatus = new JobStatus(jobid, System.getProperty("user.name"),
- 0L, JobStatus.RUNNING);
- }
- return currentJobStatus;
- }
-
- @Override
- public String getFilesystemName() throws IOException {
- return fs.getUri().toString();
- }
-
- @Override
- public JobStatus[] jobsToComplete() throws IOException {
- return null;
- }
-
- @Override
- public JobStatus[] getAllJobs() throws IOException {
- return null;
- }
-
- @Override
- public String getSystemDir() {
- return WORKING_DIR;
- }
-
- @Override
- public void killJob(BSPJobID jobid) throws IOException {
- return;
- }
-
- @Override
- public boolean killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException {
- return false;
- }
-
- // this class will spawn a new thread and executes the BSP
- class BSPRunner implements Callable {
-
- Configuration conf;
- BSPJob job;
- BSP bsp;
- LocalGroom groom;
-
- public BSPRunner(Configuration conf, BSPJob job, BSP bsp, LocalGroom groom) {
- super();
- this.conf = conf;
- this.job = job;
- this.bsp = bsp;
- this.groom = groom;
- }
-
- public void run() {
- bsp.setConf(conf);
- try {
- bsp.bsp(groom);
- } catch (Exception e) {
- LOG.error("Exception during BSP execution!", e);
- }
- }
-
- @Override
- public BSP call() throws Exception {
- run();
- return bsp;
- }
- }
-
- // this thread observes the status of the runners.
- class ThreadObserver implements Runnable {
-
- JobStatus status;
-
- public ThreadObserver(JobStatus currentJobStatus) {
- this.status = currentJobStatus;
- }
-
- @Override
- public void run() {
- boolean success = true;
- for (Future future : futureList) {
- try {
- future.get();
- } catch (InterruptedException e) {
- LOG.error("Exception during BSP execution!", e);
- success = false;
- } catch (ExecutionException e) {
- LOG.error("Exception during BSP execution!", e);
- success = false;
- }
- }
- if (success) {
- currentJobStatus.setState(JobStatus.State.SUCCEEDED);
- currentJobStatus.setRunState(JobStatus.SUCCEEDED);
- } else {
- currentJobStatus.setState(JobStatus.State.FAILED);
- currentJobStatus.setRunState(JobStatus.FAILED);
- }
- threadPool.shutdownNow();
- }
-
- }
-
- class LocalGroom extends BSPPeer {
- private long superStepCount = 0;
- private final ConcurrentLinkedQueue localMessageQueue = new ConcurrentLinkedQueue();
- // outgoing queue
- private final Map> outgoingQueues = new ConcurrentHashMap>();
- private final String peerName;
-
- public LocalGroom(String peerName) throws IOException {
- this.peerName = peerName;
- }
-
- @Override
- public void send(String peerName, BSPMessage msg) throws IOException {
- if (this.peerName.equals(peerName)) {
- put(msg);
- } else {
- // put this into a outgoing queue
- if (outgoingQueues.get(peerName) == null) {
- outgoingQueues.put(peerName, new ConcurrentLinkedQueue());
- }
- outgoingQueues.get(peerName).add(msg);
- }
- }
-
- @Override
- public void put(BSPMessage msg) throws IOException {
- localMessageQueue.add(msg);
- }
-
- @Override
- public BSPMessage getCurrentMessage() throws IOException {
- return localMessageQueue.poll();
- }
-
- @Override
- public int getNumCurrentMessages() {
- return localMessageQueue.size();
- }
-
- @Override
- public void sync() throws IOException, KeeperException,
- InterruptedException {
- // wait until all threads reach this barrier
- barrierSync();
- // send the messages
- for (Entry> entry : outgoingQueues
- .entrySet()) {
- String peerName = entry.getKey();
- for (BSPMessage msg : entry.getValue())
- localGrooms.get(peerName).put(msg);
- }
- // clear the local outgoing queue
- outgoingQueues.clear();
- // sync again to avoid data inconsistency
- barrierSync();
- incrementSuperSteps();
- }
-
- private void barrierSync() throws InterruptedException {
- try {
- barrier.await();
- } catch (BrokenBarrierException e) {
- throw new InterruptedException("Barrier has been broken!" + e);
- }
- }
-
- private void incrementSuperSteps() {
- currentJobStatus.setprogress(superStepCount++);
- currentJobStatus.setSuperstepCount(currentJobStatus.progress());
- }
-
- @Override
- public long getSuperstepCount() {
- return superStepCount;
- }
-
- @Override
- public String getPeerName() {
- return peerName;
- }
-
- @Override
- public String[] getAllPeerNames() {
- return localGrooms.keySet().toArray(
- new String[localGrooms.keySet().size()]);
- }
-
- @Override
- public void clear() {
- localMessageQueue.clear();
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return 3;
- }
-
- @Override
- public void close() throws IOException {
-
- }
-
- @Override
- public void put(BSPMessageBundle messages) throws IOException {
- }
-
- }
-}
Index: src/java/org/apache/hama/bsp/JobProfile.java
===================================================================
--- src/java/org/apache/hama/bsp/JobProfile.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobProfile.java (working copy)
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-
-/**
- * A JobProfile tracks job's status.
- */
-public class JobProfile implements Writable {
-
- static { // register actor
- WritableFactories.setFactory(JobProfile.class, new WritableFactory() {
- public Writable newInstance() {
- return new JobProfile();
- }
- });
- }
-
- String user;
- final BSPJobID jobid;
- String jobFile;
- String name;
-
- /**
- * Construct an empty {@link JobProfile}.
- */
- public JobProfile() {
- jobid = new BSPJobID();
- }
-
- /**
- * Construct a {@link JobProfile} the userid, jobid, job config-file,
- * job-details url and job name.
- *
- * @param user userid of the person who submitted the job.
- * @param jobid id of the job.
- * @param jobFile job configuration file.
- * @param name user-specified job name.
- */
- public JobProfile(String user, BSPJobID jobid, String jobFile, String name) {
- this.user = user;
- this.jobid = jobid;
- this.jobFile = jobFile;
- this.name = name;
- }
-
- /**
- * Get the user id.
- */
- public String getUser() {
- return user;
- }
-
- /**
- * Get the job id.
- */
- public BSPJobID getJobID() {
- return jobid;
- }
-
- /**
- * Get the configuration file for the job.
- */
- public String getJobFile() {
- return jobFile;
- }
-
- /**
- * Get the user-specified job name.
- */
- public String getJobName() {
- return name;
- }
-
- public void write(DataOutput out) throws IOException {
- jobid.write(out);
- Text.writeString(out, jobFile);
- Text.writeString(out, user);
- Text.writeString(out, name);
- }
-
- public void readFields(DataInput in) throws IOException {
- jobid.readFields(in);
- this.jobFile = Text.readString(in);
- this.user = Text.readString(in);
- this.name = Text.readString(in);
- }
-}
Index: src/java/org/apache/hama/bsp/LaunchTaskAction.java
===================================================================
--- src/java/org/apache/hama/bsp/LaunchTaskAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/LaunchTaskAction.java (working copy)
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the
- * {@link org.apache.hama.bsp.GroomServer} to launch a new task.
- */
-class LaunchTaskAction extends GroomServerAction {
- private Task task;
-
- public LaunchTaskAction() {
- super(ActionType.LAUNCH_TASK);
- }
-
- public LaunchTaskAction(Task task) {
- super(ActionType.LAUNCH_TASK);
- this.task = task;
- }
-
- public Task getTask() {
- return task;
- }
-
- public void write(DataOutput out) throws IOException {
- task.write(out);
- }
-
- public void readFields(DataInput in) throws IOException {
- task = new BSPTask();
- task.readFields(in);
- }
-
-}
Index: src/java/org/apache/hama/bsp/BSPJobID.java
===================================================================
--- src/java/org/apache/hama/bsp/BSPJobID.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/BSPJobID.java (working copy)
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-import org.apache.hadoop.io.Text;
-
-/**
- * BSPJobID represents the immutable and unique identifier for the job.
- */
-public class BSPJobID extends ID implements Comparable {
- protected static final String JOB = "job";
- private final Text jtIdentifier;
-
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(4);
- }
-
- public BSPJobID(String jtIdentifier, int id) {
- super(id);
- this.jtIdentifier = new Text(jtIdentifier);
- }
-
- public BSPJobID() {
- jtIdentifier = new Text();
- }
-
- public String getJtIdentifier() {
- return jtIdentifier.toString();
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- BSPJobID that = (BSPJobID) o;
- return this.jtIdentifier.equals(that.jtIdentifier);
- }
-
- @Override
- public int compareTo(ID o) {
- BSPJobID that = (BSPJobID) o;
- int jtComp = this.jtIdentifier.compareTo(that.jtIdentifier);
- if (jtComp == 0) {
- return this.id - that.id;
- } else
- return jtComp;
- }
-
- public StringBuilder appendTo(StringBuilder builder) {
- builder.append(SEPARATOR);
- builder.append(jtIdentifier);
- builder.append(SEPARATOR);
- builder.append(idFormat.format(id));
- return builder;
- }
-
- @Override
- public int hashCode() {
- return jtIdentifier.hashCode() + id;
- }
-
- @Override
- public String toString() {
- return appendTo(new StringBuilder(JOB)).toString();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- this.jtIdentifier.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- jtIdentifier.write(out);
- }
-
- public static BSPJobID forName(String str) throws IllegalArgumentException {
- if (str == null)
- return null;
- try {
- String[] parts = str.split("_");
- if (parts.length == 3) {
- if (parts[0].equals(JOB)) {
- return new BSPJobID(parts[1], Integer.parseInt(parts[2]));
- }
- }
- } catch (Exception ex) {
- }
- throw new IllegalArgumentException("JobId string : " + str
- + " is not properly formed");
- }
-}
Index: src/java/org/apache/hama/bsp/JobInProgressListener.java
===================================================================
--- src/java/org/apache/hama/bsp/JobInProgressListener.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/JobInProgressListener.java (working copy)
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.IOException;
-
-/**
- * A listener for changes in a {@link JobInProgress job}'s lifecycle in the
- * {@link BSPMaster}.
- */
-abstract class JobInProgressListener {
-
- /**
- * Invoked when a new job has been added to the {@link BSPMaster}.
- * @param job The job to be added.
- * @throws IOException
- */
- public abstract void jobAdded(JobInProgress job) throws IOException;
-
- /**
- * Invoked when a job has been removed from the {@link BSPMaster}.
- * @param job The job to be removed .
- * @throws IOException
- */
- public abstract void jobRemoved(JobInProgress job) throws IOException;
-
-}
Index: src/java/org/apache/hama/bsp/ReinitGroomAction.java
===================================================================
--- src/java/org/apache/hama/bsp/ReinitGroomAction.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/ReinitGroomAction.java (working copy)
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the
- * {@link org.apache.hama.bsp.GroomServer} to reinitialize itself.
- */
-class ReinitGroomAction extends GroomServerAction {
-
- public ReinitGroomAction() {
- super(ActionType.REINIT_GROOM);
- }
-
- public void write(DataOutput out) throws IOException {
- }
-
- public void readFields(DataInput in) throws IOException {
- }
-
-}
Index: src/java/org/apache/hama/bsp/TaskRunner.java
===================================================================
--- src/java/org/apache/hama/bsp/TaskRunner.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/TaskRunner.java (working copy)
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.InetSocketAddress;
-import java.util.Vector;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.RunJar;
-
-/**
- * Base class that runs a task in a separate process.
- */
-public class TaskRunner extends Thread {
-
- public static final Log LOG = LogFactory.getLog(TaskRunner.class);
-
- boolean killed = false;
- private Process process;
- private Task task;
- private BSPJob conf;
- private GroomServer groomServer;
-
- public TaskRunner(BSPTask bspTask, GroomServer groom, BSPJob conf) {
- this.task = bspTask;
- this.conf = conf;
- this.groomServer = groom;
- }
-
- public Task getTask() {
- return task;
- }
-
- /**
- * Called to assemble this task's input. This method is run in the parent
- * process before the child is spawned. It should not execute user code, only
- * system code.
- */
- public boolean prepare() throws IOException {
- return true;
- }
-
- public void run() {
- try {
- String sep = System.getProperty("path.separator");
- File workDir = new File(new File(task.getJobFile()).getParent(), "work");
- boolean isCreated = workDir.mkdirs();
- if(!isCreated) {
- LOG.debug("TaskRunner.workDir : " + workDir);
- }
-
- StringBuffer classPath = new StringBuffer();
- // start with same classpath as parent process
- classPath.append(System.getProperty("java.class.path"));
- classPath.append(sep);
-
- String jar = conf.getJar();
- if (jar != null) { // if jar exists, it into workDir
- RunJar.unJar(new File(jar), workDir);
- File[] libs = new File(workDir, "lib").listFiles();
- if (libs != null) {
- for (int i = 0; i < libs.length; i++) {
- classPath.append(sep); // add libs from jar to classpath
- classPath.append(libs[i]);
- }
- }
- classPath.append(sep);
- classPath.append(new File(workDir, "classes"));
- classPath.append(sep);
- classPath.append(workDir);
- }
-
- // Build exec child jmv args.
- Vector vargs = new Vector();
- File jvm = // use same jvm as parent
- new File(new File(System.getProperty("java.home"), "bin"), "java");
- vargs.add(jvm.toString());
-
- // bsp.child.java.opts
- String javaOpts = conf.getConf().get("bsp.child.java.opts", "-Xmx200m");
- javaOpts = javaOpts.replace("@taskid@", task.getTaskID().toString());
-
- String[] javaOptsSplit = javaOpts.split(" ");
- for (int i = 0; i < javaOptsSplit.length; i++) {
- vargs.add(javaOptsSplit[i]);
- }
-
- // Add classpath.
- vargs.add("-classpath");
- vargs.add(classPath.toString());
- // Add main class and its arguments
- vargs.add(GroomServer.Child.class.getName()); // main of Child
-
- InetSocketAddress addr = groomServer.getTaskTrackerReportAddress();
- vargs.add(addr.getHostName());
- vargs.add(Integer.toString(addr.getPort()));
- vargs.add(task.getTaskID().toString());
- vargs.add(groomServer.groomHostName);
-
- // Run java
- runChild((String[]) vargs.toArray(new String[0]), workDir);
- } catch (IOException e) {
- LOG.error(e);
- }
- }
-
- /**
- * Run the child process
- */
- private void runChild(String[] args, File dir) throws IOException {
- this.process = Runtime.getRuntime().exec(args, null, dir);
- try {
- new Thread() {
- public void run() {
- logStream(process.getErrorStream()); // copy log output
- }
- }.start();
-
- logStream(process.getInputStream()); // normally empty
-
- int exit_code = process.waitFor();
- if (!killed && exit_code != 0) {
- throw new IOException("Task process exit with nonzero status of "
- + exit_code + ".");
- }
-
- } catch (InterruptedException e) {
- throw new IOException(e.toString());
- } finally {
- kill();
- }
- }
-
- /**
- * Kill the child process
- */
- public void kill() {
- if (process != null) {
- process.destroy();
- }
- killed = true;
- }
-
- /**
- */
- private void logStream(InputStream output) {
- try {
- BufferedReader in = new BufferedReader(new InputStreamReader(output));
- String line;
- while ((line = in.readLine()) != null) {
- LOG.info(task.getTaskID() + " " + line);
- }
- } catch (IOException e) {
- LOG.warn(task.getTaskID() + " Error reading child output", e);
- } finally {
- try {
- output.close();
- } catch (IOException e) {
- LOG.warn(task.getTaskID() + " Error closing child output", e);
- }
- }
- }
-
-}
Index: src/java/org/apache/hama/bsp/DirectiveException.java
===================================================================
--- src/java/org/apache/hama/bsp/DirectiveException.java (revision 1152138)
+++ src/java/org/apache/hama/bsp/DirectiveException.java (working copy)
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.bsp;
-
-/**
- * A custom exception class for Directive.
- */
-public class DirectiveException extends RuntimeException{
- private static final long serialVersionUID = -8052582046894492822L;
-
- public DirectiveException(){
- super();
- }
-
- public DirectiveException(String message){
- super(message);
- }
-
- public DirectiveException(String message, Throwable t){
- super(message, t);
- }
-}
Index: src/java/org/apache/hama/BSPMasterRunner.java
===================================================================
--- src/java/org/apache/hama/BSPMasterRunner.java (revision 1152138)
+++ src/java/org/apache/hama/BSPMasterRunner.java (working copy)
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hama.bsp.BSPMaster;
-
-/**
- * This class starts and runs the BSPMaster.
- */
-public class BSPMasterRunner extends Configured implements Tool {
-
- public static final Log LOG = LogFactory.getLog(BSPMasterRunner.class);
-
- @Override
- public int run(String[] args) throws Exception {
- StringUtils.startupShutdownMessage(BSPMaster.class, args, LOG);
-
- if (args.length != 0) {
- System.out.println("usage: BSPMasterRunner");
- System.exit(-1);
- }
-
- try {
- HamaConfiguration conf = new HamaConfiguration(getConf());
- BSPMaster master = BSPMaster.startMaster(conf);
- master.offerService();
- } catch (Throwable e) {
- LOG.fatal(StringUtils.stringifyException(e));
- return -1;
- }
- return 0;
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new BSPMasterRunner(), args);
- System.exit(exitCode);
- }
-
-}
Index: src/java/org/apache/hama/ipc/BSPPeerProtocol.java
===================================================================
--- src/java/org/apache/hama/ipc/BSPPeerProtocol.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/BSPPeerProtocol.java (working copy)
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.hama.Constants;
-import org.apache.hama.bsp.PeerNames;
-import org.apache.hama.bsp.Task;
-import org.apache.hama.bsp.TaskAttemptID;
-
-/**
- * Protocol that task child process uses to contact its parent process.
- */
-public interface BSPPeerProtocol extends HamaRPCProtocolVersion, Closeable,
- Constants {
-
- /** Called when a child task process starts, to get its task. */
- Task getTask(TaskAttemptID taskid) throws IOException;
-
- /**
- * Periodically called by child to check if parent is still alive.
- *
- * @return True if the task is known
- */
- boolean ping(TaskAttemptID taskid) throws IOException;
-
- /**
- * Report that the task is successfully completed. Failure is assumed if the
- * task process exits without calling this.
- *
- * @param taskid task's id
- * @param shouldBePromoted whether to promote the task's output or not
- */
- void done(TaskAttemptID taskid, boolean shouldBePromoted) throws IOException;
-
- /** Report that the task encounted a local filesystem error. */
- void fsError(TaskAttemptID taskId, String message) throws IOException;
-
- void incrementSuperstepCount(TaskAttemptID taskid) throws IOException;
-
- /**
- * @return the all BSPPeer names.
- */
- PeerNames getAllPeerNames();
-
-}
Index: src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java
===================================================================
--- src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java (working copy)
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import org.apache.hadoop.ipc.VersionedProtocol;
-
-/**
- * There is one version id for all the RPC interfaces. If any interface is
- * changed, the versionID must be changed here.
- */
-public interface HamaRPCProtocolVersion extends VersionedProtocol {
- public static final long versionID = 1L;
-}
Index: src/java/org/apache/hama/ipc/GroomProtocol.java
===================================================================
--- src/java/org/apache/hama/ipc/GroomProtocol.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/GroomProtocol.java (working copy)
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-
-import org.apache.hama.bsp.Directive;
-
-/**
- * A protocol for BSPMaster talks to GroomServer.
- */
-public interface GroomProtocol extends HamaRPCProtocolVersion {
-
- /**
- * Instruct GroomServer performaning tasks.
- *
- * @param directive instructs a GroomServer performing necessary
- * execution.
- * @throws IOException
- */
- void dispatch(Directive directive) throws IOException;
-
-}
Index: src/java/org/apache/hama/ipc/JobSubmissionProtocol.java
===================================================================
--- src/java/org/apache/hama/ipc/JobSubmissionProtocol.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/JobSubmissionProtocol.java (working copy)
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-
-import org.apache.hama.bsp.ClusterStatus;
-import org.apache.hama.bsp.BSPJobID;
-import org.apache.hama.bsp.JobProfile;
-import org.apache.hama.bsp.JobStatus;
-import org.apache.hama.bsp.TaskAttemptID;
-
-/**
- * Protocol that a groom server and the central BSP Master use to communicate.
- * This interface will contains several methods: submitJob, killJob, and
- * killTask.
- */
-public interface JobSubmissionProtocol extends HamaRPCProtocolVersion {
-
- /**
- * Allocate a new id for the job.
- *
- * @return job id
- * @throws IOException
- */
- public BSPJobID getNewJobId() throws IOException;
-
- /**
- * Submit a Job for execution. Returns the latest profile for that job. The
- * job files should be submitted in system-dir /jobName .
- *
- * @param jobID
- * @param jobFile
- * @return jobStatus
- * @throws IOException
- */
- // public JobStatus submitJob(BSPJobID jobName) throws IOException;
-
- public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException;
-
- /**
- * Get the current status of the cluster
- *
- * @param detailed if true then report groom names as well
- * @return summary of the state of the cluster
- */
- public ClusterStatus getClusterStatus(boolean detailed) throws IOException;
-
- /**
- * Grab a handle to a job that is already known to the BSPMaster.
- *
- * @return Profile of the job, or null if not found.
- */
- public JobProfile getJobProfile(BSPJobID jobid) throws IOException;
-
- /**
- * Grab a handle to a job that is already known to the BSPMaster.
- *
- * @return Status of the job, or null if not found.
- */
- public JobStatus getJobStatus(BSPJobID jobid) throws IOException;
-
- /**
- * A BSP system always operates on a single filesystem. This function returns
- * the fs name. ('local' if the localfs; 'addr:port' if dfs). The client can
- * then copy files into the right locations prior to submitting the job.
- */
- public String getFilesystemName() throws IOException;
-
- /**
- * Get the jobs that are not completed and not failed
- *
- * @return array of JobStatus for the running/to-be-run jobs.
- */
- public JobStatus[] jobsToComplete() throws IOException;
-
- /**
- * Get all the jobs submitted.
- *
- * @return array of JobStatus for the submitted jobs
- */
- public JobStatus[] getAllJobs() throws IOException;
-
- /**
- * Grab the bspmaster system directory path where job-specific files are to be
- * placed.
- *
- * @return the system directory where job-specific files are to be placed.
- */
- public String getSystemDir();
-
- /**
- * Kill the indicated job
- */
- public void killJob(BSPJobID jobid) throws IOException;
-
- /**
- * Kill indicated task attempt.
- *
- * @param taskId the id of the task to kill.
- * @param shouldFail if true the task is failed and added to failed tasks
- * list, otherwise it is just killed, w/o affecting job failure
- * status.
- */
- public boolean killTask(TaskAttemptID taskId, boolean shouldFail)
- throws IOException;
-
-}
Index: src/java/org/apache/hama/ipc/MasterProtocol.java
===================================================================
--- src/java/org/apache/hama/ipc/MasterProtocol.java (revision 1152138)
+++ src/java/org/apache/hama/ipc/MasterProtocol.java (working copy)
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.ipc;
-
-import java.io.IOException;
-
-import org.apache.hama.bsp.GroomServerStatus;
-import org.apache.hama.bsp.Directive;
-
-/**
- * A new protocol for GroomServers communicate with BSPMaster. This
- * protocol paired with WorkerProtocl, let GroomServers enrol with
- * BSPMaster, so that BSPMaster can dispatch tasks to GroomServers.
- */
-public interface MasterProtocol extends HamaRPCProtocolVersion {
-
- /**
- * A GroomServer register with its status to BSPMaster, which will update
- * GroomServers cache.
- *
- * @param status to be updated in cache.
- * @return true if successfully register with BSPMaster; false if fail.
- */
- boolean register(GroomServerStatus status) throws IOException;
-
- /**
- * A GroomServer (periodically) reports task statuses back to the BSPMaster.
- * @param directive
- */
- boolean report(Directive directive) throws IOException;
-
- public String getSystemDir();
-
-}
Index: src/java/org/apache/hama/ipc/package.html
===================================================================
--- src/java/org/apache/hama/ipc/package.html (revision 1152138)
+++ src/java/org/apache/hama/ipc/package.html (working copy)
@@ -1,23 +0,0 @@
-
-
-
-
-
-Tools to help define network clients and servers.
-
-
Index: src/java/org/apache/hama/Constants.java
===================================================================
--- src/java/org/apache/hama/Constants.java (revision 1152138)
+++ src/java/org/apache/hama/Constants.java (working copy)
@@ -1,95 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-/**
- * Some constants used in the Hama.
- */
-public interface Constants {
-
- public static final String GROOM_RPC_HOST = "bsp.groom.rpc.hostname";
-
- public static final String DEFAULT_GROOM_RPC_HOST = "0.0.0.0";
-
- public static final String GROOM_RPC_PORT = "bsp.groom.rpc.port";
-
- /** Default port region rpc server listens on. */
- public static final int DEFAULT_GROOM_RPC_PORT = 50000;
-
-
- ///////////////////////////////////////
- // Constants for BSP Package
- ///////////////////////////////////////
- /** default host address */
- public static final String PEER_HOST = "bsp.peer.hostname";
- /** default host address */
- public static final String DEFAULT_PEER_HOST = "0.0.0.0";
-
- public static final String PEER_PORT = "bsp.peer.port";
- /** Default port region server listens on. */
- public static final int DEFAULT_PEER_PORT = 61000;
-
- public static final String PEER_ID = "bsp.peer.id";
-
- /** Parameter name for what groom server implementation to use. */
- public static final String GROOM_SERVER_IMPL= "hama.groomserver.impl";
-
- /** When we encode strings, we always specify UTF8 encoding */
- static final String UTF8_ENCODING = "UTF-8";
-
- ///////////////////////////////////////
- // Constants for ZooKeeper
- ///////////////////////////////////////
- /** zookeeper root */
- public static final String ZOOKEEPER_ROOT = "bsp.zookeeper.root";
- /** zookeeper default root */
- public static final String DEFAULT_ZOOKEEPER_ROOT = "/bsp";
-
- /** zookeeper server address */
- public static final String ZOOKEEPER_SERVER_ADDRS = "zookeeper.server";
- /** zookeeper default server address */
- static final String DEFAULT_ZOOKEEPER_SERVER_ADDR = "localhost:21810";
- /** Parameter name for number of times to retry writes to ZooKeeper. */
- public static final String ZOOKEEPER_RETRIES = "zookeeper.retries";
- /** Default number of times to retry writes to ZooKeeper. */
- public static final int DEFAULT_ZOOKEEPER_RETRIES = 5;
- /** Parameter name for ZooKeeper pause between retries. In milliseconds. */
- public static final String ZOOKEEPER_PAUSE = "zookeeper.pause";
- /** Default ZooKeeper pause value. In milliseconds. */
- public static final int DEFAULT_ZOOKEEPER_PAUSE = 2 * 1000;
-
- static final String ZOOKEEPER_CONFIG_NAME = "zoo.cfg";
- static final String ZOOKEEPER_CLIENT_PORT = "hama.zookeeper.property.clientPort";
- static final String ZOOKEEPER_SESSION_TIMEOUT = "hama.zookeeper.session.timeout";
- static final int DEFAULT_ZOOKEEPER_CLIENT_PORT = 21810;
- static final String ZOOKEEPER_QUORUM = "hama.zookeeper.quorum";
- /** Cluster is in distributed mode or not */
- static final String CLUSTER_DISTRIBUTED = "hama.cluster.distributed";
- /** Cluster is fully-distributed */
- static final String CLUSTER_IS_DISTRIBUTED = "true";
-
-
- // Other constants
-
- /**
- * An empty instance.
- */
- static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
-}
Index: src/java/org/apache/hama/ZooKeeperRunner.java
===================================================================
--- src/java/org/apache/hama/ZooKeeperRunner.java (revision 1152138)
+++ src/java/org/apache/hama/ZooKeeperRunner.java (working copy)
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hama.zookeeper.QuorumPeer;
-
-/**
- * This class starts and runs the ZooKeeperServer.
- */
-public class ZooKeeperRunner extends Configured implements Tool {
-
- public static final Log LOG = LogFactory.getLog(ZooKeeperRunner.class);
-
- @Override
- public int run(String[] args) throws Exception {
- QuorumPeer.run(getConf());
- return 0;
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new ZooKeeperRunner(), args);
- System.exit(exitCode);
- }
-
-}
Index: src/java/org/apache/hama/util/ClusterUtil.java
===================================================================
--- src/java/org/apache/hama/util/ClusterUtil.java (revision 1152138)
+++ src/java/org/apache/hama/util/ClusterUtil.java (working copy)
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hama.HamaConfiguration;
-import org.apache.hama.bsp.BSPMaster;
-import org.apache.hama.bsp.GroomServer;
-
-public class ClusterUtil {
- private static final Log LOG = LogFactory.getLog(ClusterUtil.class);
-
- /**
- * Data Structure to hold GroomServer Thread and GroomServer instance
- */
- public static class GroomServerThread extends Thread {
- private final GroomServer groomServer;
-
- public GroomServerThread(final GroomServer r, final int index) {
- super(r, "GroomServer:" + index);
- this.groomServer = r;
- }
-
- /** @return the groom server */
- public GroomServer getGroomServer() {
- return this.groomServer;
- }
-
- /**
- * Block until the groom server has come online, indicating it is ready
- * to be used.
- */
- public void waitForServerOnline() {
- while (!groomServer.isRunning()) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // continue waiting
- }
- }
- }
- }
-
- /**
- * Creates a {@link GroomServerThread}.
- * Call 'start' on the returned thread to make it run.
- * @param c Configuration to use.
- * @param hrsc Class to create.
- * @param index Used distingushing the object returned.
- * @throws IOException
- * @return Groom server added.
- */
- public static ClusterUtil.GroomServerThread createGroomServerThread(final Configuration c,
- final Class extends GroomServer> hrsc, final int index)
- throws IOException {
- GroomServer server;
- try {
- server = hrsc.getConstructor(Configuration.class).newInstance(c);
- } catch (Exception e) {
- IOException ioe = new IOException();
- ioe.initCause(e);
- throw ioe;
- }
- return new ClusterUtil.GroomServerThread(server, index);
- }
-
- /**
- * Start the cluster.
- * @param m
- * @param conf
- * @param groomservers
- * @return Address to use contacting master.
- * @throws InterruptedException
- * @throws IOException
- */
- public static String startup(final BSPMaster m,
- final List groomservers, Configuration conf) throws IOException, InterruptedException {
- if (m != null) {
- BSPMaster.startMaster((HamaConfiguration) conf);
- }
-
- if (groomservers != null) {
- for (ClusterUtil.GroomServerThread t: groomservers) {
- t.start();
- }
- }
-
- return m == null? null: BSPMaster.getAddress(conf).getHostName();
- }
-
- public static void shutdown(BSPMaster master,
- List groomThreads, Configuration conf) {
- LOG.debug("Shutting down HAMA Cluster");
- // TODO:
- }
-}
Index: src/java/org/apache/hama/util/RunJar.java
===================================================================
--- src/java/org/apache/hama/util/RunJar.java (revision 1152138)
+++ src/java/org/apache/hama/util/RunJar.java (working copy)
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.lang.reflect.Array;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-import java.util.jar.Manifest;
-
-import org.apache.hadoop.fs.FileUtil;
-
-/**
- * Run a Hama job jar.
- */
-public class RunJar {
-
- /** Unpack a jar file into a directory. */
- public static void unJar(File jarFile, File toDir) throws IOException {
- JarFile jar = new JarFile(jarFile);
- try {
- Enumeration entries = jar.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = (JarEntry) entries.nextElement();
- if (!entry.isDirectory()) {
- InputStream in = jar.getInputStream(entry);
- try {
- File file = new File(toDir, entry.getName());
- file.getParentFile().mkdirs();
- OutputStream out = new FileOutputStream(file);
- try {
- byte[] buffer = new byte[8192];
- int i;
- while ((i = in.read(buffer)) != -1) {
- out.write(buffer, 0, i);
- }
- } finally {
- out.close();
- }
- } finally {
- in.close();
- }
- }
- }
- } finally {
- jar.close();
- }
- }
-
- /**
- * Run a Hama job jar. If the main class is not in the jar's manifest, then
- * it must be provided on the command line.
- */
- public static void main(String[] args) throws Throwable {
- String usage = "Usage: hama jar [mainClass] args...";
-
- if (args.length < 1) {
- System.err.println(usage);
- System.exit(-1);
- }
-
- int firstArg = 0;
- String fileName = args[firstArg++];
- File file = new File(fileName);
- String mainClassName = null;
-
- JarFile jarFile = new JarFile(fileName);
- Manifest manifest = jarFile.getManifest();
- if (manifest != null) {
- mainClassName = manifest.getMainAttributes().getValue("Main-Class");
- }
- jarFile.close();
-
- if (mainClassName == null) {
- if (args.length < 2) {
- System.err.println(usage);
- System.exit(-1);
- }
- mainClassName = args[firstArg++];
- }
- mainClassName = mainClassName.replaceAll("/", ".");
-
- final File workDir = File.createTempFile("hama-unjar", "");
- workDir.delete();
- workDir.mkdirs();
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- public void run() {
- try {
- FileUtil.fullyDelete(workDir);
- } catch (IOException e) {
- }
- }
- });
-
- unJar(file, workDir);
-
- List classPath = new ArrayList();
- classPath.add(new File(workDir + "/").toURI().toURL());
- classPath.add(file.toURI().toURL());
- classPath.add(new File(workDir, "classes/").toURI().toURL());
- File[] libs = new File(workDir, "lib").listFiles();
- if (libs != null) {
- for (int i = 0; i < libs.length; i++) {
- classPath.add(libs[i].toURI().toURL());
- }
- }
- ClassLoader loader = new URLClassLoader((URL[]) classPath
- .toArray(new URL[0]));
-
- Thread.currentThread().setContextClassLoader(loader);
- Class> mainClass = loader.loadClass(mainClassName);
- Method main = mainClass.getMethod("main", new Class[] { Array.newInstance(
- String.class, 0).getClass() });
- String[] newArgs = (String[]) Arrays.asList(args).subList(firstArg,
- args.length).toArray(new String[0]);
- try {
- main.invoke(null, new Object[] { newArgs });
- } catch (InvocationTargetException e) {
- throw e.getTargetException();
- }
- }
-
-}
Index: src/java/org/apache/hama/util/VersionInfo.java
===================================================================
--- src/java/org/apache/hama/util/VersionInfo.java (revision 1152138)
+++ src/java/org/apache/hama/util/VersionInfo.java (working copy)
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-/**
- * A version information class.
- */
-public class VersionInfo {
-
- public static void main(String[] args) {
- System.out.println("Apache Hama - 0.4");
- }
-}
Index: src/java/org/apache/hama/util/Bytes.java
===================================================================
--- src/java/org/apache/hama/util/Bytes.java (revision 1152138)
+++ src/java/org/apache/hama/util/Bytes.java (working copy)
@@ -1,1226 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.Comparator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.hama.Constants;
-
-/**
- * Utility class that handles byte arrays, conversions to/from other types,
- * comparisons, hash code generation, manufacturing keys for HashMaps or
- * HashSets, etc.
- */
-public class Bytes {
-
- private static final Log LOG = LogFactory.getLog(Bytes.class);
-
- /**
- * Size of boolean in bytes
- */
- public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE;
-
- /**
- * Size of byte in bytes
- */
- public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN;
-
- /**
- * Size of char in bytes
- */
- public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE;
-
- /**
- * Size of double in bytes
- */
- public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE;
-
- /**
- * Size of float in bytes
- */
- public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE;
-
- /**
- * Size of int in bytes
- */
- public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE;
-
- /**
- * Size of long in bytes
- */
- public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE;
-
- /**
- * Size of short in bytes
- */
- public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
-
- /**
- * Estimate of size cost to pay beyond payload in jvm for instance of byte [].
- * Estimate based on study of jhat and jprofiler numbers.
- */
- // JHat says BU is 56 bytes.
- // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
- public static final int ESTIMATED_HEAP_TAX = 16;
-
- /**
- * Byte array comparator class.
- */
- public static class ByteArrayComparator implements RawComparator {
- /**
- * Constructor
- */
- public ByteArrayComparator() {
- super();
- }
-
- public int compare(byte[] left, byte[] right) {
- return compareTo(left, right);
- }
-
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- return compareTo(b1, s1, l1, b2, s2, l2);
- }
- }
-
- /**
- * Pass this to TreeMaps where byte [] are keys.
- */
- public static Comparator BYTES_COMPARATOR = new ByteArrayComparator();
-
- /**
- * Use comparing byte arrays, byte-by-byte
- */
- public static RawComparator BYTES_RAWCOMPARATOR = new ByteArrayComparator();
-
- /**
- * Read byte-array written with a WritableableUtils.vint prefix.
- *
- * @param in Input to read from.
- * @return byte array read off in
- * @throws IOException e
- */
- public static byte[] readByteArray(final DataInput in) throws IOException {
- int len = WritableUtils.readVInt(in);
- if (len < 0) {
- throw new NegativeArraySizeException(Integer.toString(len));
- }
- byte[] result = new byte[len];
- in.readFully(result, 0, len);
- return result;
- }
-
- /**
- * Read byte-array written with a WritableableUtils.vint prefix. IOException
- * is converted to a RuntimeException.
- *
- * @param in Input to read from.
- * @return byte array read off in
- */
- public static byte[] readByteArrayThrowsRuntime(final DataInput in) {
- try {
- return readByteArray(in);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Write byte-array with a WritableableUtils.vint prefix.
- *
- * @param out output stream to be written to
- * @param b array to write
- * @throws IOException e
- */
- public static void writeByteArray(final DataOutput out, final byte[] b)
- throws IOException {
- if (b == null) {
- WritableUtils.writeVInt(out, 0);
- } else {
- writeByteArray(out, b, 0, b.length);
- }
- }
-
- /**
- * Write byte-array to out with a vint length prefix.
- *
- * @param out output stream
- * @param b array
- * @param offset offset into array
- * @param length length past offset
- * @throws IOException e
- */
- public static void writeByteArray(final DataOutput out, final byte[] b,
- final int offset, final int length) throws IOException {
- WritableUtils.writeVInt(out, length);
- out.write(b, offset, length);
- }
-
- /**
- * Write byte-array from src to tgt with a vint length prefix.
- *
- * @param tgt target array
- * @param tgtOffset offset into target array
- * @param src source array
- * @param srcOffset source offset
- * @param srcLength source length
- * @return New offset in src array.
- */
- public static int writeByteArray(final byte[] tgt, final int tgtOffset,
- final byte[] src, final int srcOffset, final int srcLength) {
- byte[] vint = vintToBytes(srcLength);
- System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
- int offset = tgtOffset + vint.length;
- System.arraycopy(src, srcOffset, tgt, offset, srcLength);
- return offset + srcLength;
- }
-
- /**
- * Put bytes at the specified byte array position.
- *
- * @param tgtBytes the byte array
- * @param tgtOffset position in the array
- * @param srcBytes array to write out
- * @param srcOffset source offset
- * @param srcLength source length
- * @return incremented offset
- */
- public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
- int srcOffset, int srcLength) {
- System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
- return tgtOffset + srcLength;
- }
-
- /**
- * Write a single byte out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param b byte to write out
- * @return incremented offset
- */
- public static int putByte(byte[] bytes, int offset, byte b) {
- bytes[offset] = b;
- return offset + 1;
- }
-
- /**
- * Returns a new byte array, copied from the passed ByteBuffer.
- *
- * @param bb A ByteBuffer
- * @return the byte array
- */
- public static byte[] toBytes(ByteBuffer bb) {
- int length = bb.limit();
- byte[] result = new byte[length];
- System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length);
- return result;
- }
-
- /**
- * @param b Presumed UTF-8 encoded byte array.
- * @return String made from b
- */
- public static String toString(final byte[] b) {
- if (b == null) {
- return null;
- }
- return toString(b, 0, b.length);
- }
-
- /**
- * Joins two byte arrays together using a separator.
- *
- * @param b1 The first byte array.
- * @param sep The separator to use.
- * @param b2 The second byte array.
- */
- public static String toString(final byte[] b1, String sep, final byte[] b2) {
- return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length);
- }
-
- /**
- * This method will convert utf8 encoded bytes into a string. If an
- * UnsupportedEncodingException occurs, this method will eat it and return
- * null instead.
- *
- * @param b Presumed UTF-8 encoded byte array.
- * @param off offset into array
- * @param len length of utf-8 sequence
- * @return String made from b or null
- */
- public static String toString(final byte[] b, int off, int len) {
- if (b == null) {
- return null;
- }
- if (len == 0) {
- return "";
- }
- try {
- return new String(b, off, len, Constants.UTF8_ENCODING);
- } catch (UnsupportedEncodingException e) {
- LOG.error("UTF-8 not supported?", e);
- return null;
- }
- }
-
- /**
- * Write a printable representation of a byte array.
- *
- * @param b byte array
- * @return string
- * @see #toStringBinary(byte[], int, int)
- */
- public static String toStringBinary(final byte[] b) {
- return toStringBinary(b, 0, b.length);
- }
-
- /**
- * Write a printable representation of a byte array. Non-printable characters
- * are hex escaped in the format \\x%02X, eg: \x00 \x05 etc
- *
- * @param b array to write out
- * @param off offset to start at
- * @param len length to write
- * @return string output
- */
- public static String toStringBinary(final byte[] b, int off, int len) {
- StringBuilder result = new StringBuilder();
- try {
- String first = new String(b, off, len, "ISO-8859-1");
- for (int i = 0; i < first.length(); ++i) {
- int ch = first.charAt(i) & 0xFF;
- if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z')
- || (ch >= 'a' && ch <= 'z') || ch == ',' || ch == '_' || ch == '-'
- || ch == ':' || ch == ' ' || ch == '<' || ch == '>' || ch == '='
- || ch == '/' || ch == '.') {
- result.append(first.charAt(i));
- } else {
- result.append(String.format("\\x%02X", ch));
- }
- }
- } catch (UnsupportedEncodingException e) {
- LOG.error("ISO-8859-1 not supported?", e);
- }
- return result.toString();
- }
-
- private static boolean isHexDigit(char c) {
- return (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9');
- }
-
- /**
- * Takes a ASCII digit in the range A-F0-9 and returns the corresponding
- * integer/ordinal value.
- *
- * @param ch The hex digit.
- * @return The converted hex value as a byte.
- */
- public static byte toBinaryFromHex(byte ch) {
- if (ch >= 'A' && ch <= 'F')
- return (byte) ((byte) 10 + (byte) (ch - 'A'));
- // else
- return (byte) (ch - '0');
- }
-
- public static byte[] toBytesBinary(String in) {
- // this may be bigger than we need, but lets be safe.
- byte[] b = new byte[in.length()];
- int size = 0;
- for (int i = 0; i < in.length(); ++i) {
- char ch = in.charAt(i);
- if (ch == '\\') {
- // begin hex escape:
- char next = in.charAt(i + 1);
- if (next != 'x') {
- // invalid escape sequence, ignore this one.
- b[size++] = (byte) ch;
- continue;
- }
- // ok, take next 2 hex digits.
- char hd1 = in.charAt(i + 2);
- char hd2 = in.charAt(i + 3);
-
- // they need to be A-F0-9:
- if (!isHexDigit(hd1) || !isHexDigit(hd2)) {
- // bogus escape code, ignore:
- continue;
- }
- // turn hex ASCII digit -> number
- byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2));
-
- b[size++] = d;
- i += 3; // skip 3
- } else {
- b[size++] = (byte) ch;
- }
- }
- // resize:
- byte[] b2 = new byte[size];
- System.arraycopy(b, 0, b2, 0, size);
- return b2;
- }
-
- /**
- * Converts a string to a UTF-8 byte array.
- *
- * @param s string
- * @return the byte array
- */
- public static byte[] toBytes(String s) {
- try {
- return s.getBytes(Constants.UTF8_ENCODING);
- } catch (UnsupportedEncodingException e) {
- LOG.error("UTF-8 not supported?", e);
- return null;
- }
- }
-
- /**
- * Convert a boolean to a byte array. True becomes -1 and false becomes 0.
- *
- * @param b value
- * @return b encoded in a byte array.
- */
- public static byte[] toBytes(final boolean b) {
- return new byte[] { b ? (byte) -1 : (byte) 0 };
- }
-
- /**
- * Reverses {@link #toBytes(boolean)}
- *
- * @param b array
- * @return True or false.
- */
- public static boolean toBoolean(final byte[] b) {
- if (b.length != 1) {
- throw new IllegalArgumentException("Array has wrong size: " + b.length);
- }
- return b[0] != (byte) 0;
- }
-
- /**
- * Convert a long value to a byte array using big-endian.
- *
- * @param val value to convert
- * @return the byte array
- */
- public static byte[] toBytes(long val) {
- byte[] b = new byte[8];
- for (int i = 7; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to a long value. Reverses {@link #toBytes(long)}
- *
- * @param bytes array
- * @return the long value
- */
- public static long toLong(byte[] bytes) {
- return toLong(bytes, 0, SIZEOF_LONG);
- }
-
- /**
- * Converts a byte array to a long value. Assumes there will be
- * {@link #SIZEOF_LONG} bytes available.
- *
- * @param bytes bytes
- * @param offset offset
- * @return the long value
- */
- public static long toLong(byte[] bytes, int offset) {
- return toLong(bytes, offset, SIZEOF_LONG);
- }
-
- /**
- * Converts a byte array to a long value.
- *
- * @param bytes array of bytes
- * @param offset offset into array
- * @param length length of data (must be {@link #SIZEOF_LONG})
- * @return the long value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or
- * if there's not enough room in the array at the offset indicated.
- */
- public static long toLong(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_LONG || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
- }
- long l = 0;
- for (int i = offset; i < offset + length; i++) {
- l <<= 8;
- l ^= bytes[i] & 0xFF;
- }
- return l;
- }
-
- private static IllegalArgumentException explainWrongLengthOrOffset(
- final byte[] bytes, final int offset, final int length,
- final int expectedLength) {
- String reason;
- if (length != expectedLength) {
- reason = "Wrong length: " + length + ", expected " + expectedLength;
- } else {
- reason = "offset (" + offset + ") + length (" + length + ") exceed the"
- + " capacity of the array: " + bytes.length;
- }
- return new IllegalArgumentException(reason);
- }
-
- /**
- * Put a long value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val long to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putLong(byte[] bytes, int offset, long val) {
- if (bytes.length - offset < SIZEOF_LONG) {
- throw new IllegalArgumentException("Not enough room to put a long at"
- + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- for (int i = offset + 7; i > offset; i--) {
- bytes[i] = (byte) val;
- val >>>= 8;
- }
- bytes[offset] = (byte) val;
- return offset + SIZEOF_LONG;
- }
-
- /**
- * Presumes float encoded as IEEE 754 floating-point "single format"
- *
- * @param bytes byte array
- * @return Float made from passed byte array.
- */
- public static float toFloat(byte[] bytes) {
- return toFloat(bytes, 0);
- }
-
- /**
- * Presumes float encoded as IEEE 754 floating-point "single format"
- *
- * @param bytes array to convert
- * @param offset offset into array
- * @return Float made from passed byte array.
- */
- public static float toFloat(byte[] bytes, int offset) {
- return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
- }
-
- /**
- * @param bytes byte array
- * @param offset offset to write to
- * @param f float value
- * @return New offset in bytes
- */
- public static int putFloat(byte[] bytes, int offset, float f) {
- return putInt(bytes, offset, Float.floatToRawIntBits(f));
- }
-
- /**
- * @param f float value
- * @return the float represented as byte []
- */
- public static byte[] toBytes(final float f) {
- // Encode it as int
- return Bytes.toBytes(Float.floatToRawIntBits(f));
- }
-
- /**
- * @param bytes byte array
- * @return Return double made from passed bytes.
- */
- public static double toDouble(final byte[] bytes) {
- return toDouble(bytes, 0);
- }
-
- /**
- * @param bytes byte array
- * @param offset offset where double is
- * @return Return double made from passed bytes.
- */
- public static double toDouble(final byte[] bytes, final int offset) {
- return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
- }
-
- /**
- * @param bytes byte array
- * @param offset offset to write to
- * @param d value
- * @return New offset into array bytes
- */
- public static int putDouble(byte[] bytes, int offset, double d) {
- return putLong(bytes, offset, Double.doubleToLongBits(d));
- }
-
- /**
- * Serialize a double as the IEEE 754 double format output. The resultant
- * array will be 8 bytes long.
- *
- * @param d value
- * @return the double represented as byte []
- */
- public static byte[] toBytes(final double d) {
- // Encode it as a long
- return Bytes.toBytes(Double.doubleToRawLongBits(d));
- }
-
- /**
- * Convert an int value to a byte array
- *
- * @param val value
- * @return the byte array
- */
- public static byte[] toBytes(int val) {
- byte[] b = new byte[4];
- for (int i = 3; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @return the int value
- */
- public static int toInt(byte[] bytes) {
- return toInt(bytes, 0, SIZEOF_INT);
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the int value
- */
- public static int toInt(byte[] bytes, int offset) {
- return toInt(bytes, offset, SIZEOF_INT);
- }
-
- /**
- * Converts a byte array to an int value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @param length length of int (has to be {@link #SIZEOF_INT})
- * @return the int value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or if
- * there's not enough room in the array at the offset indicated.
- */
- public static int toInt(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_INT || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
- }
- int n = 0;
- for (int i = offset; i < (offset + length); i++) {
- n <<= 8;
- n ^= bytes[i] & 0xFF;
- }
- return n;
- }
-
- /**
- * Put an int value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val int to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putInt(byte[] bytes, int offset, int val) {
- if (bytes.length - offset < SIZEOF_INT) {
- throw new IllegalArgumentException("Not enough room to put an int at"
- + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- for (int i = offset + 3; i > offset; i--) {
- bytes[i] = (byte) val;
- val >>>= 8;
- }
- bytes[offset] = (byte) val;
- return offset + SIZEOF_INT;
- }
-
- /**
- * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long.
- *
- * @param val value
- * @return the byte array
- */
- public static byte[] toBytes(short val) {
- byte[] b = new byte[SIZEOF_SHORT];
- b[1] = (byte) val;
- val >>= 8;
- b[0] = (byte) val;
- return b;
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @return the short value
- */
- public static short toShort(byte[] bytes) {
- return toShort(bytes, 0, SIZEOF_SHORT);
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @return the short value
- */
- public static short toShort(byte[] bytes, int offset) {
- return toShort(bytes, offset, SIZEOF_SHORT);
- }
-
- /**
- * Converts a byte array to a short value
- *
- * @param bytes byte array
- * @param offset offset into array
- * @param length length, has to be {@link #SIZEOF_SHORT}
- * @return the short value
- * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} or
- * if there's not enough room in the array at the offset indicated.
- */
- public static short toShort(byte[] bytes, int offset, final int length) {
- if (length != SIZEOF_SHORT || offset + length > bytes.length) {
- throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
- }
- short n = 0;
- n ^= bytes[offset] & 0xFF;
- n <<= 8;
- n ^= bytes[offset + 1] & 0xFF;
- return n;
- }
-
- /**
- * Put a short value out to the specified byte array position.
- *
- * @param bytes the byte array
- * @param offset position in the array
- * @param val short to write out
- * @return incremented offset
- * @throws IllegalArgumentException if the byte array given doesn't have
- * enough room at the offset specified.
- */
- public static int putShort(byte[] bytes, int offset, short val) {
- if (bytes.length - offset < SIZEOF_SHORT) {
- throw new IllegalArgumentException("Not enough room to put a short at"
- + " offset " + offset + " in a " + bytes.length + " byte array");
- }
- bytes[offset + 1] = (byte) val;
- val >>= 8;
- bytes[offset] = (byte) val;
- return offset + SIZEOF_SHORT;
- }
-
- /**
- * @param vint Integer to make a vint of.
- * @return Vint as bytes array.
- */
- public static byte[] vintToBytes(final long vint) {
- long i = vint;
- int size = WritableUtils.getVIntSize(i);
- byte[] result = new byte[size];
- int offset = 0;
- if (i >= -112 && i <= 127) {
- result[offset] = (byte) i;
- return result;
- }
-
- int len = -112;
- if (i < 0) {
- i ^= -1L; // take one's complement'
- len = -120;
- }
-
- long tmp = i;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- result[offset++] = (byte) len;
-
- len = (len < -120) ? -(len + 120) : -(len + 112);
-
- for (int idx = len; idx != 0; idx--) {
- int shiftbits = (idx - 1) * 8;
- long mask = 0xFFL << shiftbits;
- result[offset++] = (byte) ((i & mask) >> shiftbits);
- }
- return result;
- }
-
- /**
- * @param buffer buffer to convert
- * @return vint bytes as an integer.
- */
- public static long bytesToVint(final byte[] buffer) {
- int offset = 0;
- byte firstByte = buffer[offset++];
- int len = WritableUtils.decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = buffer[offset++];
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
- }
-
- /**
- * Reads a zero-compressed encoded long from input stream and returns it.
- *
- * @param buffer Binary array
- * @param offset Offset into array at which vint begins.
- * @throws java.io.IOException e
- * @return deserialized long from stream.
- */
- public static long readVLong(final byte[] buffer, final int offset)
- throws IOException {
- byte firstByte = buffer[offset];
- int len = WritableUtils.decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len - 1; idx++) {
- byte b = buffer[offset + 1 + idx];
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
- }
-
- /**
- * @param left left operand
- * @param right right operand
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- public static int compareTo(final byte[] left, final byte[] right) {
- return compareTo(left, 0, left.length, right, 0, right.length);
- }
-
- /**
- * Lexographically compare two arrays.
- *
- * @param b1 left operand
- * @param b2 right operand
- * @param s1 Where to start comparing in the left buffer
- * @param s2 Where to start comparing in the right buffer
- * @param l1 How much to compare from the left buffer
- * @param l2 How much to compare from the right buffer
- * @return 0 if equal, < 0 if left is less than right, etc.
- */
- public static int compareTo(byte[] b1, int s1, int l1, byte[] b2, int s2,
- int l2) {
- // Bring WritableComparator code local
- int end1 = s1 + l1;
- int end2 = s2 + l2;
- for (int i = s1, j = s2; i < end1 && j < end2; i++, j++) {
- int a = (b1[i] & 0xff);
- int b = (b2[j] & 0xff);
- if (a != b) {
- return a - b;
- }
- }
- return l1 - l2;
- }
-
- /**
- * @param left left operand
- * @param right right operand
- * @return True if equal
- */
- public static boolean equals(final byte[] left, final byte[] right) {
- // Could use Arrays.equals?
- // noinspection SimplifiableConditionalExpression
- if (left == null && right == null) {
- return true;
- }
- return (left == null || right == null || (left.length != right.length) ? false
- : compareTo(left, right) == 0);
- }
-
- /**
- * @param b bytes to hash
- * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
- * passed in array. This method is what
- * {@link org.apache.hadoop.io.Text} use calculating hash code.
- */
- public static int hashCode(final byte[] b) {
- return hashCode(b, b.length);
- }
-
- /**
- * @param b value
- * @param length length of the value
- * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
- * passed in array. This method is what
- * {@link org.apache.hadoop.io.Text} use calculating hash code.
- */
- public static int hashCode(final byte[] b, final int length) {
- return WritableComparator.hashBytes(b, length);
- }
-
- /**
- * @param b bytes to hash
- * @return A hash of b as an Integer that can be used as key in
- * Maps.
- */
- public static Integer mapKey(final byte[] b) {
- return hashCode(b);
- }
-
- /**
- * @param b bytes to hash
- * @param length length to hash
- * @return A hash of b as an Integer that can be used as key in
- * Maps.
- */
- public static Integer mapKey(final byte[] b, final int length) {
- return hashCode(b, length);
- }
-
- /**
- * @param a lower half
- * @param b upper half
- * @return New array that has a in lower half and b in upper half.
- */
- public static byte[] add(final byte[] a, final byte[] b) {
- return add(a, b, Constants.EMPTY_BYTE_ARRAY);
- }
-
- /**
- * @param a first third
- * @param b second third
- * @param c third third
- * @return New array made from a, b and c
- */
- public static byte[] add(final byte[] a, final byte[] b, final byte[] c) {
- byte[] result = new byte[a.length + b.length + c.length];
- System.arraycopy(a, 0, result, 0, a.length);
- System.arraycopy(b, 0, result, a.length, b.length);
- System.arraycopy(c, 0, result, a.length + b.length, c.length);
- return result;
- }
-
- /**
- * @param a array
- * @param length amount of bytes to grab
- * @return First length bytes from a
- */
- public static byte[] head(final byte[] a, final int length) {
- if (a.length < length) {
- return null;
- }
- byte[] result = new byte[length];
- System.arraycopy(a, 0, result, 0, length);
- return result;
- }
-
- /**
- * @param a array
- * @param length amount of bytes to snarf
- * @return Last length bytes from a
- */
- public static byte[] tail(final byte[] a, final int length) {
- if (a.length < length) {
- return null;
- }
- byte[] result = new byte[length];
- System.arraycopy(a, a.length - length, result, 0, length);
- return result;
- }
-
- /**
- * @param a array
- * @param length new array size
- * @return Value in a plus length prepended 0 bytes
- */
- public static byte[] padHead(final byte[] a, final int length) {
- byte[] padding = new byte[length];
- for (int i = 0; i < length; i++) {
- padding[i] = 0;
- }
- return add(padding, a);
- }
-
- /**
- * @param a array
- * @param length new array size
- * @return Value in a plus length appended 0 bytes
- */
- public static byte[] padTail(final byte[] a, final int length) {
- byte[] padding = new byte[length];
- for (int i = 0; i < length; i++) {
- padding[i] = 0;
- }
- return add(a, padding);
- }
-
- /**
- * Split passed range. Expensive operation relatively. Uses BigInteger math.
- * Useful splitting ranges for MapReduce jobs.
- *
- * @param a Beginning of range
- * @param b End of range
- * @param num Number of times to split range. Pass 1 if you want to split the
- * range in two; i.e. one split.
- * @return Array of dividing values
- */
- public static byte[][] split(final byte[] a, final byte[] b, final int num) {
- byte[] aPadded;
- byte[] bPadded;
- if (a.length < b.length) {
- aPadded = padTail(a, b.length - a.length);
- bPadded = b;
- } else if (b.length < a.length) {
- aPadded = a;
- bPadded = padTail(b, a.length - b.length);
- } else {
- aPadded = a;
- bPadded = b;
- }
- if (compareTo(aPadded, bPadded) >= 0) {
- throw new IllegalArgumentException("b <= a");
- }
- if (num <= 0) {
- throw new IllegalArgumentException("num cannot be < 0");
- }
- byte[] prependHeader = { 1, 0 };
- BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
- BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
- BigInteger diffBI = stopBI.subtract(startBI);
- BigInteger splitsBI = BigInteger.valueOf(num + 1);
- if (diffBI.compareTo(splitsBI) < 0) {
- return null;
- }
- BigInteger intervalBI;
- try {
- intervalBI = diffBI.divide(splitsBI);
- } catch (Exception e) {
- LOG.error("Exception caught during division", e);
- return null;
- }
-
- byte[][] result = new byte[num + 2][];
- result[0] = a;
-
- for (int i = 1; i <= num; i++) {
- BigInteger curBI = startBI
- .add(intervalBI.multiply(BigInteger.valueOf(i)));
- byte[] padded = curBI.toByteArray();
- if (padded[1] == 0)
- padded = tail(padded, padded.length - 2);
- else
- padded = tail(padded, padded.length - 1);
- result[i] = padded;
- }
- result[num + 1] = b;
- return result;
- }
-
- /**
- * @param t operands
- * @return Array of byte arrays made from passed array of Text
- */
- public static byte[][] toByteArrays(final String[] t) {
- byte[][] result = new byte[t.length][];
- for (int i = 0; i < t.length; i++) {
- result[i] = Bytes.toBytes(t[i]);
- }
- return result;
- }
-
- /**
- * @param column operand
- * @return A byte array of a byte array where first and only entry is
- * column
- */
- public static byte[][] toByteArrays(final String column) {
- return toByteArrays(toBytes(column));
- }
-
- /**
- * @param column operand
- * @return A byte array of a byte array where first and only entry is
- * column
- */
- public static byte[][] toByteArrays(final byte[] column) {
- byte[][] result = new byte[1][];
- result[0] = column;
- return result;
- }
-
- /**
- * Binary search for keys in indexes.
- *
- * @param arr array of byte arrays to search for
- * @param key the key you want to find
- * @param offset the offset in the key you want to find
- * @param length the length of the key
- * @param comparator a comparator to compare.
- * @return index of key
- */
- public static int binarySearch(byte[][] arr, byte[] key, int offset,
- int length, RawComparator comparator) {
- int low = 0;
- int high = arr.length - 1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- // we have to compare in this order, because the comparator order
- // has special logic when the 'left side' is a special key.
- int cmp = comparator.compare(key, offset, length, arr[mid], 0,
- arr[mid].length);
- // key lives above the midpoint
- if (cmp > 0)
- low = mid + 1;
- // key lives below the midpoint
- else if (cmp < 0)
- high = mid - 1;
- // BAM. how often does this really happen?
- else
- return mid;
- }
- return -(low + 1);
- }
-
- /**
- * Bytewise binary increment/deincrement of long contained in byte array on
- * given amount.
- *
- * @param value - array of bytes containing long (length <= SIZEOF_LONG)
- * @param amount value will be incremented on (deincremented if negative)
- * @return array of bytes containing incremented long (length == SIZEOF_LONG)
- * @throws IOException - if value.length > SIZEOF_LONG
- */
- public static byte[] incrementBytes(byte[] value, long amount)
- throws IOException {
- byte[] val = value;
- if (val.length < SIZEOF_LONG) {
- // Hopefully this doesn't happen too often.
- byte[] newvalue;
- if (val[0] < 0) {
- newvalue = new byte[] { -1, -1, -1, -1, -1, -1, -1, -1 };
- } else {
- newvalue = new byte[SIZEOF_LONG];
- }
- System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
- val.length);
- val = newvalue;
- } else if (val.length > SIZEOF_LONG) {
- throw new IllegalArgumentException("Increment Bytes - value too big: "
- + val.length);
- }
- if (amount == 0)
- return val;
- if (val[0] < 0) {
- return binaryIncrementNeg(val, amount);
- }
- return binaryIncrementPos(val, amount);
- }
-
- /* increment/deincrement for positive value */
- private static byte[] binaryIncrementPos(byte[] value, long amount) {
- long amo = amount;
- int sign = 1;
- if (amount < 0) {
- amo = -amount;
- sign = -1;
- }
- for (int i = 0; i < value.length; i++) {
- int cur = ((int) amo % 256) * sign;
- amo = (amo >> 8);
- int val = value[value.length - i - 1] & 0x0ff;
- int total = val + cur;
- if (total > 255) {
- amo += sign;
- total %= 256;
- } else if (total < 0) {
- amo -= sign;
- }
- value[value.length - i - 1] = (byte) total;
- if (amo == 0)
- return value;
- }
- return value;
- }
-
- /* increment/deincrement for negative value */
- private static byte[] binaryIncrementNeg(byte[] value, long amount) {
- long amo = amount;
- int sign = 1;
- if (amount < 0) {
- amo = -amount;
- sign = -1;
- }
- for (int i = 0; i < value.length; i++) {
- int cur = ((int) amo % 256) * sign;
- amo = (amo >> 8);
- int val = ((~value[value.length - i - 1]) & 0x0ff) + 1;
- int total = cur - val;
- if (total >= 0) {
- amo += sign;
- } else if (total < -256) {
- amo -= sign;
- total %= 256;
- }
- value[value.length - i - 1] = (byte) total;
- if (amo == 0)
- return value;
- }
- return value;
- }
-
-}
Index: src/java/org/apache/hama/util/RandomVariable.java
===================================================================
--- src/java/org/apache/hama/util/RandomVariable.java (revision 1152138)
+++ src/java/org/apache/hama/util/RandomVariable.java (working copy)
@@ -1,227 +0,0 @@
-/**
- * Copyright 2007 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-/**
- * The RandomVaraibale Class provides static methods for generating random
- * numbers.
- */
-public class RandomVariable {
-
- /**
- * Generate a random number between 0 and 1.
- *
- * @return a double between 0 and 1.
- */
- public static double rand() {
- double x = Math.random();
- return x;
- }
-
- /**
- * Generate a random integer.
- *
- * @param i0 min of the random variable.
- * @param i1 max of the random variable.
- * @return an int between i0 and i1.
- */
- public static int randInt(int i0, int i1) {
- double x = rand();
- int i = i0 + (int) Math.floor((i1 - i0 + 1) * x);
- return i;
- }
-
- /**
- * Generate a random string using the specified prefix and a fixed length.
- * @param prefix
- * the specified string prefix.
- * @param length
- * the length of the string to be appended.
- * @return random string.
- */
- public static String randString(String prefix, int length) {
- StringBuilder result = new StringBuilder(prefix);
- for (int i = 0; i < length; i++) {
- char ch = (char) ((Math.random() * 26) + 97);
- result.append(ch);
- }
-
- return result.toString();
- }
-
- /**
- * Generate a random number from a uniform random variable.
- *
- * @param min min of the random variable.
- * @param max max of the random variable.
- * @return a double.
- */
- public static double uniform(double min, double max) {
- double x = min + (max - min) * rand();
- return x;
- }
-
- /**
- * Generate a random number from a discrete random variable.
- *
- * @param values discrete values.
- * @param prob probability of each value.
- * @return a double.
- */
- public static double dirac(double[] values, double[] prob) {
- double[] prob_cumul = new double[values.length];
- prob_cumul[0] = prob[0];
- for (int i = 1; i < values.length; i++) {
- prob_cumul[i] = prob_cumul[i - 1] + prob[i];
- }
- double y = rand();
- double x = 0;
- for (int i = 0; i < values.length - 1; i++) {
- if ((y > prob_cumul[i]) && (y < prob_cumul[i + 1])) {
- x = values[i];
- }
- }
- return x;
- }
-
- /**
- * Generate a random number from a Gaussian (Normal) random variable.
- *
- * @param mu mean of the random variable.
- * @param sigma standard deviation of the random variable.
- * @return a double.
- */
- public static double normal(double mu, double sigma) {
- double x = mu + sigma * Math.cos(2 * Math.PI * rand())
- * Math.sqrt(-2 * Math.log(rand()));
- return x;
- }
-
- /**
- * Generate a random number from a Chi-2 random variable.
- *
- * @param n degrees of freedom of the chi2 random variable.
- * @return a double.
- */
- public static double chi2(int n) {
- double x = 0;
- for (int i = 0; i < n; i++) {
- double norm = normal(0, 1);
- x += norm * norm;
- }
- return x;
- }
-
- /**
- * Generate a random number from a LogNormal random variable.
- *
- * @param mu mean of the Normal random variable.
- * @param sigma standard deviation of the Normal random variable.
- * @return a double.
- */
- public static double logNormal(double mu, double sigma) {
- double x = mu + sigma * Math.cos(2 * Math.PI * rand())
- * Math.sqrt(-2 * Math.log(rand()));
- return x;
- }
-
- /**
- * Generate a random number from an exponantial random variable (Mean =
- * 1/lambda, variance = 1/lambda^2).
- *
- * @param lambda parmaeter of the exponential random variable.
- * @return a double.
- */
- public static double exponential(double lambda) {
- double x = -1 / lambda * Math.log(rand());
- return x;
- }
-
- /**
- * Generate a random number from a symetric triangular random variable.
- *
- * @param min min of the random variable.
- * @param max max of the random variable.
- * @return a double.
- */
- public static double triangular(double min, double max) {
- double x = min / 2 + (max - min) * rand() / 2 + min / 2 + (max - min)
- * rand() / 2;
- return x;
- }
-
- /**
- * Generate a random number from a non-symetric triangular random variable.
- *
- * @param min min of the random variable.
- * @param med value of the random variable with max density.
- * @param max max of the random variable.
- * @return a double.
- */
- public static double triangular(double min, double med, double max) {
- double y = rand();
- double x = (y < ((med - min) / (max - min))) ? (min + Math.sqrt(y
- * (max - min) * (med - min))) : (max - Math.sqrt((1 - y) * (max - min)
- * (max - med)));
- return x;
- }
-
- /**
- * Generate a random number from a beta random variable.
- *
- * @param a first parameter of the Beta random variable.
- * @param b second parameter of the Beta random variable.
- * @return a double.
- */
- public static double beta(double a, double b) {
- double try_x;
- double try_y;
- do {
- try_x = Math.pow(rand(), 1 / a);
- try_y = Math.pow(rand(), 1 / b);
- } while ((try_x + try_y) > 1);
- return try_x / (try_x + try_y);
- }
-
- /**
- * Generate a random number from a Cauchy random variable (Mean = Inf, and
- * Variance = Inf).
- *
- * @param mu median of the Weibull random variable
- * @param sigma second parameter of the Cauchy random variable.
- * @return a double.
- */
- public static double cauchy(double mu, double sigma) {
- double x = sigma * Math.tan(Math.PI * (rand() - 0.5)) + mu;
- return x;
- }
-
- /**
- * Generate a random number from a Weibull random variable.
- *
- * @param lambda first parameter of the Weibull random variable.
- * @param c second parameter of the Weibull random variable.
- * @return a double.
- */
- public static double weibull(double lambda, double c) {
- double x = Math.pow(-Math.log(1 - rand()), 1 / c) / lambda;
- return x;
- }
-}
Index: src/java/org/apache/hama/util/BSPServletUtil.java
===================================================================
--- src/java/org/apache/hama/util/BSPServletUtil.java (revision 1152138)
+++ src/java/org/apache/hama/util/BSPServletUtil.java (working copy)
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hama.util;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.Calendar;
-
-import org.apache.hadoop.util.ServletUtil;
-import org.apache.hama.bsp.JobStatus;
-
-public class BSPServletUtil extends ServletUtil {
-
- public static final String HTML_TAIL = " \n"
- + "Hama , "
- + Calendar.getInstance().get(Calendar.YEAR) + ".\n" + "