Property changes on: . ___________________________________________________________________ Modified: svn:ignore - .project .classpath build .fbprefs src-gen hama-0.1.0-dev.jar hama-0.1.0-dev-examples.jar hama-0.1.0-dev-test.jar logs .settings + .project .classpath build .fbprefs src-gen hama-0.1.0-dev.jar hama-0.1.0-dev-examples.jar hama-0.1.0-dev-test.jar logs .settings lib target *.ipr *.iml Property changes on: site ___________________________________________________________________ Added: svn:ignore + *.iml Index: site/src/docs/forrest.properties =================================================================== --- site/src/docs/forrest.properties (revision 0) +++ site/src/docs/forrest.properties (revision 0) @@ -0,0 +1,104 @@ +# Copyright 2002-2004 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############## +# Properties used by forrest.build.xml for building the website +# These are the defaults, un-comment them if you need to change them. +############## + +# Prints out a summary of Forrest settings for this project +#forrest.echo=true + +# Project name (used to name .war file) +#project.name=my-project + +# Specifies name of Forrest skin to use +#project.skin=tigris +#project.skin=pelt + +# comma separated list, file:// is supported +#forrest.skins.descriptors=http://forrest.apache.org/skins/skins.xml,file:///c:/myskins/skins.xml + +############## +# behavioural properties +#project.menu-scheme=tab_attributes +#project.menu-scheme=directories + +############## +# layout properties + +# Properties that can be set to override the default locations +# +# Parent properties must be set. This usually means uncommenting +# project.content-dir if any other property using it is uncommented + +#project.status=status.xml +#project.content-dir=src/documentation +#project.raw-content-dir=${project.content-dir}/content +#project.conf-dir=${project.content-dir}/conf +#project.sitemap-dir=${project.content-dir} +#project.xdocs-dir=${project.content-dir}/content/xdocs +#project.resources-dir=${project.content-dir}/resources +#project.stylesheets-dir=${project.resources-dir}/stylesheets +#project.images-dir=${project.resources-dir}/images +#project.schema-dir=${project.resources-dir}/schema +#project.skins-dir=${project.content-dir}/skins +#project.skinconf=${project.content-dir}/skinconf.xml +#project.lib-dir=${project.content-dir}/lib +#project.classes-dir=${project.content-dir}/classes +#project.translations-dir=${project.content-dir}/translations + +############## +# validation properties + +# This set of properties determine if validation is performed +# Values are inherited unless overridden. +# e.g. if forrest.validate=false then all others are false unless set to true. +forrest.validate=false +#forrest.validate.xdocs=${forrest.validate} +#forrest.validate.skinconf=${forrest.validate} +#forrest.validate.sitemap=${forrest.validate} +#forrest.validate.stylesheets=${forrest.validate} +#forrest.validate.skins=${forrest.validate} +#forrest.validate.skins.stylesheets=${forrest.validate.skins} + +# *.failonerror=(true|false) - stop when an XML file is invalid +#forrest.validate.failonerror=true + +# *.excludes=(pattern) - comma-separated list of path patterns to not validate +# e.g. +#forrest.validate.xdocs.excludes=samples/subdir/**, samples/faq.xml +#forrest.validate.xdocs.excludes= + + +############## +# General Forrest properties + +# The URL to start crawling from +#project.start-uri=linkmap.html +# Set logging level for messages printed to the console +# (DEBUG, INFO, WARN, ERROR, FATAL_ERROR) +#project.debuglevel=ERROR +# Max memory to allocate to Java +#forrest.maxmemory=64m +# Any other arguments to pass to the JVM. For example, to run on an X-less +# server, set to -Djava.awt.headless=true +#forrest.jvmargs= +# The bugtracking URL - the issue number will be appended +#project.bugtracking-url=http://issues.apache.org/bugzilla/show_bug.cgi?id= +#project.bugtracking-url=http://issues.apache.org/jira/browse/ +# The issues list as rss +#project.issues-rss-url= +#I18n Property only works for the "forrest run" target. +#project.i18n=true Index: site/src/docs/status.xml =================================================================== --- site/src/docs/status.xml (revision 0) +++ site/src/docs/status.xml (revision 0) @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + Initial Import + + + + + + + + + Customize this template project with your project's details. This + TODO list is generated from 'status.xml'. + + + Add lots of content. XML content goes in + src/documentation/content/xdocs, or wherever the + ${project.xdocs-dir} property (set in + forrest.properties) points. + + + Mail forrest-dev@xml.apache.org + with feedback. + + + + + + Index: site/src/docs/src/documentation/skinconf.xml =================================================================== --- site/src/docs/src/documentation/skinconf.xml (revision 0) +++ site/src/docs/src/documentation/skinconf.xml (revision 0) @@ -0,0 +1,354 @@ + + + + + + + + + + + + + true + + false + + true + + true + + + true + + + true + + + true + + + false + + + true + + + Apache Hama + + index.html + images/hama_paint_logo_120x120.png + + + Apache Incubator + + http://incubator.apache.org/ + images/apache-incubator-logo.png + + + + + + + images/favicon.ico + + + 2008 + The Apache Software Foundation. + http://www.apache.org/licenses/ + + + + + + + + + + + + + + + + + + + + + #footer a { color: black; } + img.logo { + float: left; + margin-right: 0.5em; + } + + + + + + + + + + + + + + + + + + + + + + + + 1in + 1in + 1.25in + 1in + + + + false + + + false + + + + + + Built with Apache Forrest + http://forrest.apache.org/ + images/built-with-forrest-button.png + 88 + 31 + + + + + + Index: site/src/docs/src/documentation/content/xdocs/site.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/site.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/site.xml (revision 0) @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: site/src/docs/src/documentation/content/xdocs/irc_channel.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/irc_channel.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/irc_channel.xml (revision 0) @@ -0,0 +1,46 @@ + + + + + + + +
+ IRC Channel +
+ + + +
+ IRC Channel +

+ There is an IRC channel dedicated to Hama at irc.freenode.org. + The name of the channel is #hama. +

+ The IRC channel can be used for online discussion about hama related stuff, + but developers should be careful to transfer all the official decisions + or useful discussions to the issue tracking + system. +

+ +
+ +
Index: site/src/docs/src/documentation/content/xdocs/index.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/index.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/index.xml (revision 0) @@ -0,0 +1,91 @@ + + + + + + + +
+ Welcome to Hama project +
+ +
+ Overview +

+ Hama is a distributed computing framework based on + BSP (Bulk Synchronous Parallel) computing techniques for massive scientific computations, such as Google's Pregel, + Currently being incubated as one of the incubator project by the Apache Software Foundation. +

+
+ +
+ Getting Started +

+ Start by installing Hama on a Hadoop cluster. +
+

+ +
+ +
+ Getting Involved +

+ Hama is an open source volunteer project under the Apache Software Foundation. We encourage you to learn about the project and contribute your expertise. + Here are some starter links:
+

+ +
+ +
+ Recent News +
+ July 28, 2011: release 0.3.0 available +
    +
  • Added LockBSPRunner
  • +
  • Added web UI for BSP cluster and job monitoring
  • +
  • Added more practical examples e.g., Shortest Path Problem, PageRank
  • +
  • Performance has improved by more than 25% with BSPMessageBundle
  • +
  • Switched from Ant to Maven
  • +
+
+
+ June 2, 2011: release 0.2.0 available +
    +
  • BSP computing framework and its examples +
    • Pi Estimator and Serialize Printing
  • +
  • CLI-based managing and monitoring tool of BSP job
  • +
+
+
+ Apr 30, 2010: Introduced BSP Worldwide +

We're introduced in the BSP Worldwide.

+
+
+ May 20, 2008: Accept Hama to be a Apache incubator project +

The Incubator PMC have voted to accept Hama to be a Apache incubator project.

+
+
+ + +
Index: site/src/docs/src/documentation/content/xdocs/developers.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/developers.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/developers.xml (revision 0) @@ -0,0 +1,68 @@ + + + + + + + +
+ Developers +
+ + +
+ Contribution +

+ We are always looking for contributions. See the + Wiki for details on contributing. +

+
+
+ Source Code +

+ The source files are now stored on Subversion : + https://svn.apache.org/repos/asf/incubator/hama/trunk +
+ And, git mirros : git://git.apache.org/hama.git + [ http, github ] +

+
+
+ Code Style +

+ The following sections show how to set up the code style for Hama in Eclipse. + It is strongly preferred that patches use this style before they are supplied. +

+ + Download hama-eclipse-formatter.xml. +

+ After this, select Window > Preferences, and open up the configuration for + Java > Code Style > Code Formatter. Click on the button labeled Import... + and select the file you downloaded. Give the style a name, and click OK. +

+
+
+ Issues +

+ All bugs, improvements, patches, etc. should be logged in + JIRA +

+
+ + +
Index: site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml (revision 0) @@ -0,0 +1,37 @@ + + + + + + + +
+ Welcome to Hama project +
+ + +
+ Documentations + +
+ + +
\ No newline at end of file Index: site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/ApacheHama-0.2_UserGuide.pdf =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/content/xdocs/docs/r0.2.0/ApacheHama-0.2_UserGuide.pdf ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml (revision 0) @@ -0,0 +1,36 @@ + + + + + + + +
+ Welcome to Hama project +
+ + +
+ Documentations + +
+ + +
\ No newline at end of file Index: site/src/docs/src/documentation/content/xdocs/tabs.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/tabs.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/tabs.xml (revision 0) @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + Index: site/src/docs/src/documentation/content/xdocs/downloads.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/downloads.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/downloads.xml (revision 0) @@ -0,0 +1,64 @@ + + + + + + + +
+ Apache Hama Downloads +
+ + +
+ Download +

Use the links below to download Apache Hama releases from one of our mirrors. + You should verify the integrity of the files using the signatures and checksums available from this page. +

+ Apache Hama releases are available under the + Apache License, Version 2.0. + See the NOTICE.txt file contained in each release artifact for applicable copyright attribution notices. +

+ Download a release now! +

+
+ +
+ PGP Signatures +

All of the release distribution package have been digitally signed (using PGP or GPG) by + the Apache Group members that constructed them. There will be an accompanying distribution.asc file + in the same directory as the distribution. + The PGP keys can be found in the MIT key repository (hkp://pgp.mit.edu), + the OpenPGP Public Key repository (hkp://subkeys.pgp.net), and within this project's KEYS file, + which is located within each download directory. +

+ Always use the signature files to verify the authenticity of the distribution, e.g., +

  % pgpk -a KEYS
+  % pgpv hama-x.x.x.tar.gz.asc
+ or, +
  % gpg --import KEYS
+  % gpg --verify hama-x.x.x.tar.gz.asc
+ We offer MD5 hashes as an alternative to validate the integrity of the downloaded files. + A unix program called md5 or md5sum is included in many unix distributions. + It is also available as part of GNU Core Utilities package. +

+
+ + + +
Index: site/src/docs/src/documentation/content/xdocs/issue_tracking.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/issue_tracking.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/issue_tracking.xml (revision 0) @@ -0,0 +1,44 @@ + + + + + + + +
+ Issue Tracking +
+ + + +
+ Issue Tracking +

+ Hama tracks both bugs and enhancement requests + here using Apache JIRA. + We welcome input, however, before filing a request, + please make sure you do the following: +

+ +
+ +
Index: site/src/docs/src/documentation/content/xdocs/privacy_policy.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/privacy_policy.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/privacy_policy.xml (revision 0) @@ -0,0 +1,61 @@ + + + + + + + +
+ Privacy Policy +
+ + + +
+ Privacy Policy +

+ Information about your use of this website is collected using + server access logs and a tracking cookie. + The collected information consists of the following:

+

+ +



+ Part of this information is gathered using a tracking cookie set by the + Google Analytics service + and handled by Google as described in their + privacy policy. + See your browser documentation for instructions on how to disable + the cookie if you prefer not to share this data with Google.

+ We use the gathered information to help us make our site more + useful to visitors and to better understand how and when + our site is used. We do not track or collect personally + identifiable information or associate gathered data + with any personally identifying information from other sources.

+ By using this website, you consent to the collection of this data + in the manner and for the purpose described above. +

+
+ + +
Index: site/src/docs/src/documentation/content/xdocs/mailing_lists.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/mailing_lists.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/mailing_lists.xml (revision 0) @@ -0,0 +1,70 @@ + + + + + + + +
+ Mailing Lists +
+ + + +
+ Hama User List +

+ If you use Hama, please subscribe to the Hama user mailing list.
+ The Hama user mailing list is : + hama-user@incubator.apache.org. +

+ +
+ +
+ Hama Developer List +

+ If you'd like to contribute to Hama, please subscribe to the Hama developer mailing list.
+ The Hama developer mailing list is : + hama-dev@incubator.apache.org. +

+ +
+ +
+ Hama Commit List +

+ If you'd like to see changes made in Hama's version control system then subscribe + to the Hama commit mailing list.
+ The Hama commit mailing list is : + hama-commits@incubator.apache.org. +

+ +
+ + +
Index: site/src/docs/src/documentation/content/xdocs/who_we_are.xml =================================================================== --- site/src/docs/src/documentation/content/xdocs/who_we_are.xml (revision 0) +++ site/src/docs/src/documentation/content/xdocs/who_we_are.xml (revision 0) @@ -0,0 +1,80 @@ + + + + + + + +
+ Who we are +
+ + + +
+ Active Committers + +

Hama's active committers are:

+ + + + + + + + + + + + + + + + + + + + + + + +
usernamenameorganizationrolestimezone
edwardyoonEdward J. Yoon+9
chanwitChanwit KaewkasiThe University of Manchester+1
+
+ +
+ Mentors +

Mentors while in incubation include:

+ +
+ +
+ Contributors +

Contributors are:

+ +
+ + +
Index: site/src/docs/src/documentation/resources/images/hama_art__arthur_300x220.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/hama_art__arthur_300x220.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/hama_paint_logo_120x120.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/hama_paint_logo_120x120.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/hama_art_arthur.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/hama_art_arthur.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/hama_paint_logo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/hama_paint_logo.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/apache-incubator-logo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/apache-incubator-logo.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/MapReduce.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/MapReduce.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/hama_logo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/hama_logo.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/apache-incubator-hama-white.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/apache-incubator-hama-white.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/apache-incubator-hama.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/apache-incubator-hama.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/headerlogo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/headerlogo.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/block.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: site/src/docs/src/documentation/resources/images/block.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: site/src/docs/src/documentation/resources/images/favicon.ico =================================================================== Index: site/src/docs/src/documentation/classes/CatalogManager.properties =================================================================== --- site/src/docs/src/documentation/classes/CatalogManager.properties (revision 0) +++ site/src/docs/src/documentation/classes/CatalogManager.properties (revision 0) @@ -0,0 +1,37 @@ +# Copyright 2002-2004 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#======================================================================= +# CatalogManager.properties +# +# This is the default properties file for Apache Forrest. +# This facilitates local configuration of application-specific catalogs. +# +# See the Apache Forrest documentation: +# http://forrest.apache.org/docs/your-project.html +# http://forrest.apache.org/docs/validation.html + +# verbosity ... level of messages for status/debug +# See forrest/src/core/context/WEB-INF/cocoon.xconf + +# catalogs ... list of additional catalogs to load +# (Note that Apache Forrest will automatically load its own default catalog +# from src/core/context/resources/schema/catalog.xcat) +# use full pathnames +# pathname separator is always semi-colon (;) regardless of operating system +# directory separator is always slash (/) regardless of operating system +# +#catalogs=/home/me/forrest/my-site/src/documentation/resources/schema/catalog.xcat +catalogs= + Index: site/pom.xml =================================================================== --- site/pom.xml (revision 0) +++ site/pom.xml (revision 0) @@ -0,0 +1,60 @@ + + + + + + org.apache.hama + hama-parent + 0.4.0-incubating-SNAPSHOT + + + 4.0.0 + org.apache.hama + hama-site + 0.4.0-incubating-SNAPSHOT + pom + + + + + maven-antrun-plugin + + + package + + + + + + + + + + + + run + + + + + + + + Index: conf/hama-env.sh.template =================================================================== --- conf/hama-env.sh.template (revision 1152138) +++ conf/hama-env.sh.template (working copy) @@ -1,37 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Set environment variables here. - -# The java implementation to use. Required. -# export JAVA_HOME=/usr/lib/jvm/java-6-sun - -# Where log files are stored. $HAMA_HOME/logs by default. -# export HAMA_LOG_DIR=${HAMA_HOME}/logs - -# The maximum amount of heap to use, in MB. Default is 1000. -# export HAMA_HEAPSIZE=1000 - -# Extra ssh options. Empty by default. -# export HAMA_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HAMA_CONF_DIR" - -# Tell Hama whether it should manage it's own instance of Zookeeper or not. -# export HAMA_MANAGES_ZK=true Index: conf/hama-site.xml.template =================================================================== --- conf/hama-site.xml.template (revision 1152138) +++ conf/hama-site.xml.template (working copy) @@ -1,25 +0,0 @@ - - - - - Index: conf/groomservers.template =================================================================== --- conf/groomservers.template (revision 1152138) +++ conf/groomservers.template (working copy) @@ -1 +0,0 @@ -localhost \ No newline at end of file Index: conf/hama-default.xml =================================================================== --- conf/hama-default.xml (revision 1152138) +++ conf/hama-default.xml (working copy) @@ -1,165 +0,0 @@ - - - - - - bsp.master.address - local - The address of the bsp master server. Either the - literal string "local" or a host[:port] (where host is a name or - IP address) for distributed mode. - - - - bsp.master.port - 40000 - The port master should bind to. - - - bsp.peer.port - 61000 - The port an groom server binds to. - - - bsp.groom.rpc.port - 50000 - The port an groom rpc binds to. - - - bsp.http.infoserver.port - 40013 - The port where the web-interface can be seen. - - - - bsp.groom.report.address - 127.0.0.1:0 - The interface and port that groom server listens on. - Since it is only connected to by the tasks, it uses the local interface. - EXPERT ONLY. Should only be changed if your host does not have the loopback - interface. - - - bsp.system.dir - ${hadoop.tmp.dir}/bsp/system - The shared directory where BSP stores control files. - - - - bsp.local.dir - ${hadoop.tmp.dir}/bsp/local - local directory for temporal store. - - - hama.tmp.dir - /tmp/hama-${user.name} - Temporary directory on the local filesystem. - - - bsp.child.java.opts - -Xmx1024m - Java opts for the groom server child processes. - The following symbol, if present, will be interpolated: @taskid@ is replaced - by current TaskID. Any other occurrences of '@' will go unchanged. - For example, to enable verbose gc logging to a file named for the taskid in - /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: - -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc - The configuration variable bsp.child.ulimit can be used to control the - maximum virtual memory of the child processes. - - - - bsp.local.tasks.maximum - 20 - Number of tasks that run in parallel when in local mode. - - - - - hama.zookeeper.property.initLimit - 10 - Property from ZooKeeper's config zoo.cfg. - The number of ticks that the initial synchronization phase can take. - - - - hama.zookeeper.property.syncLimit - 5 - Property from ZooKeeper's config zoo.cfg. - The number of ticks that can pass between sending a request and getting an - acknowledgment. - - - - hama.zookeeper.property.dataDir - ${hama.tmp.dir}/zookeeper - Property from ZooKeeper's config zoo.cfg. - The directory where the snapshot is stored. - - - - hama.zookeeper.property.clientPort - 21810 - Property from ZooKeeper's config zoo.cfg. - The port at which the clients will connect. - - - - hama.zookeeper.property.tickTime - 3000 - - - - hama.zookeeper.property.maxClientCnxns - 30 - Property from ZooKeeper's config zoo.cfg. - Limit on number of concurrent connections (at the socket level) that a - single client, identified by IP address, may make to a single member of - the ZooKeeper ensemble. Set high to avoid zk connection issues running - standalone and pseudo-distributed. - - - - - hama.zookeeper.quorum - localhost - Comma separated list of servers in the ZooKeeper quorum. - For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". - By default this is set to localhost for local and pseudo-distributed modes - of operation. For a fully-distributed setup, this should be set to a full - list of ZooKeeper quorum servers. If HAMA_MANAGES_ZK is set in hama-env.sh - this is the list of servers which we will start/stop ZooKeeper on. - - - - - Index: conf/log4j.properties =================================================================== --- conf/log4j.properties (revision 1152138) +++ conf/log4j.properties (working copy) @@ -1,81 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Define some default values that can be overridden by system properties -hama.root.logger=INFO,console -hama.log.dir=. -hama.log.file=hama.log - -# Define the root logger to the system property "hama.root.logger". -log4j.rootLogger=${hama.root.logger} - -# Logging Threshold -log4j.threshhold=ALL - -# -# Daily Rolling File Appender -# -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hama.log.dir}/${hama.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# Debugging Pattern format -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - -# -# TaskLog Appender -# - -#Default values -hama.tasklog.taskid=null -hama.tasklog.noKeepSplits=4 -hama.tasklog.totalLogFileSize=100 -hama.tasklog.purgeLogSplits=true -hama.tasklog.logsRetainHours=12 - -log4j.appender.TLA=org.apache.hama.bsp.TaskLogAppender -log4j.appender.TLA.taskId=${hama.tasklog.taskid} -log4j.appender.TLA.totalLogFileSize=${hama.tasklog.totalLogFileSize} - -log4j.appender.TLA.layout=org.apache.log4j.PatternLayout -log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n - -# Custom Logging levels - -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG -#log4j.logger.org.apache.hadoop.dfs=DEBUG -#log4j.logger.org.apache.hama=DEBUG Index: src/test/org/apache/hama/HamaTestCase.java =================================================================== --- src/test/org/apache/hama/HamaTestCase.java (revision 1152138) +++ src/test/org/apache/hama/HamaTestCase.java (working copy) @@ -1,179 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import java.io.File; -import java.io.IOException; - -import junit.framework.AssertionFailedError; -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hama.util.Bytes; - -public abstract class HamaTestCase extends TestCase { - private static Log LOG = LogFactory.getLog(HamaTestCase.class); - - /** configuration parameter name for test directory */ - public static final String TEST_DIRECTORY_KEY = "test.build.data"; - - private boolean localfs = false; - protected Path testDir = null; - protected FileSystem fs = null; - - static { - initialize(); - } - - public volatile HamaConfiguration conf; - - /** constructor */ - public HamaTestCase() { - super(); - init(); - } - - /** - * @param name - */ - public HamaTestCase(String name) { - super(name); - init(); - } - - private void init() { - conf = new HamaConfiguration(); - conf.setStrings("bsp.local.dir", "/tmp/hama-test"); - conf.set("bsp.master.address", "localhost"); - conf.set("bsp.groom.report.address", "127.0.0.1:0"); - } - - /** - * Note that this method must be called after the mini hdfs cluster has - * started or we end up with a local file system. - */ - @Override - protected void setUp() throws Exception { - super.setUp(); - localfs = - (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0); - - if (fs == null) { - this.fs = FileSystem.get(conf); - } - try { - if (localfs) { - this.testDir = getUnitTestdir(getName()); - if (fs.exists(testDir)) { - fs.delete(testDir, true); - } - } else { - this.testDir = - this.fs.makeQualified(new Path("/tmp/hama-test")); - } - } catch (Exception e) { - LOG.fatal("error during setup", e); - throw e; - } - } - - @Override - protected void tearDown() throws Exception { - try { - if (localfs) { - if (this.fs.exists(testDir)) { - this.fs.delete(testDir, true); - } - } - } catch (Exception e) { - LOG.fatal("error during tear down", e); - } - super.tearDown(); - } - - protected Path getUnitTestdir(String testName) { - return new Path( - conf.get(TEST_DIRECTORY_KEY, "/tmp/hama-test/build/data"), testName); - } - - /** - * Initializes parameters used in the test environment: - * - * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set. - * Sets the boolean debugging if "DEBUGGING" is set in the environment. - * If debugging is enabled, reconfigures logging so that the root log level is - * set to WARN and the logging level for the package is set to DEBUG. - */ - public static void initialize() { - if (System.getProperty(TEST_DIRECTORY_KEY) == null) { - System.setProperty(TEST_DIRECTORY_KEY, new File( - "build/hama/test").getAbsolutePath()); - } - } - - /** - * Common method to close down a MiniDFSCluster and the associated file system - * - * @param cluster - */ - public static void shutdownDfs(MiniDFSCluster cluster) { - if (cluster != null) { - LOG.info("Shutting down Mini DFS "); - try { - cluster.shutdown(); - } catch (Exception e) { - /// Can get a java.lang.reflect.UndeclaredThrowableException thrown - // here because of an InterruptedException. Don't let exceptions in - // here be cause of test failure. - } - try { - FileSystem fs = cluster.getFileSystem(); - if (fs != null) { - LOG.info("Shutting down FileSystem"); - fs.close(); - } - FileSystem.closeAll(); - } catch (IOException e) { - LOG.error("error closing file system", e); - } - } - } - - public void assertByteEquals(byte[] expected, - byte[] actual) { - if (Bytes.compareTo(expected, actual) != 0) { - throw new AssertionFailedError("expected:<" + - Bytes.toString(expected) + "> but was:<" + - Bytes.toString(actual) + ">"); - } - } - - public static void assertEquals(byte[] expected, - byte[] actual) { - if (Bytes.compareTo(expected, actual) != 0) { - throw new AssertionFailedError("expected:<" + - Bytes.toStringBinary(expected) + "> but was:<" + - Bytes.toStringBinary(actual) + ">"); - } - } - -} Index: src/test/org/apache/hama/MiniBSPCluster.java =================================================================== --- src/test/org/apache/hama/MiniBSPCluster.java (revision 1152138) +++ src/test/org/apache/hama/MiniBSPCluster.java (working copy) @@ -1,252 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import java.io.IOException; - -import java.net.ServerSocket; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import static java.util.concurrent.TimeUnit.*; - -import static junit.framework.Assert.*; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hama.bsp.BSPMaster; -import org.apache.hama.bsp.GroomServer; -import org.apache.hama.HamaConfiguration; - - -public class MiniBSPCluster { - - public static final Log LOG = LogFactory.getLog(MiniBSPCluster.class); - - private ScheduledExecutorService scheduler; - - private HamaConfiguration configuration; - private BSPMasterRunner master; - private List groomServerList = - new CopyOnWriteArrayList(); - private int grooms; - - public class BSPMasterRunner implements Runnable{ - BSPMaster bspm; - HamaConfiguration conf; - - public BSPMasterRunner(HamaConfiguration conf){ - this.conf = conf; - if(null == this.conf) - throw new NullPointerException("No Configuration for BSPMaster."); - } - - public void run(){ - try{ - LOG.info("Starting BSP Master."); - this.bspm = BSPMaster.startMaster(this.conf); - this.bspm.offerService(); - }catch(IOException ioe){ - LOG.error("Fail to startup BSP Master.", ioe); - }catch(InterruptedException ie){ - LOG.error("BSP Master fails in offerService().", ie); - Thread.currentThread().interrupt(); - } - } - - public void shutdown(){ - if(null != this.bspm) this.bspm.shutdown(); - } - - public boolean isRunning(){ - if(null == this.bspm) return false; - - if(this.bspm.currentState().equals(BSPMaster.State.RUNNING)){ - return true; - } - return false; - } - - public BSPMaster getMaster(){ - return this.bspm; - } - } - - public class GroomServerRunner implements Runnable{ - GroomServer gs; - HamaConfiguration conf; - - public GroomServerRunner(HamaConfiguration conf){ - this.conf = conf; - } - - public void run(){ - try{ - this.gs = GroomServer.constructGroomServer(GroomServer.class, conf); - GroomServer.startGroomServer(this.gs).join(); - }catch(InterruptedException ie){ - LOG.error("Fail to start GroomServer. ", ie); - Thread.currentThread().interrupt(); - } - } - - public void shutdown(){ - try{ - if(null != this.gs) this.gs.shutdown(); - }catch(IOException ioe){ - LOG.info("Fail to shutdown GroomServer.", ioe); - } - } - - public boolean isRunning(){ - if(null == this.gs) return false; - return this.gs.isRunning(); - } - - public GroomServer getGroomServer(){ - return this.gs; - } - } - - public MiniBSPCluster(HamaConfiguration conf, int groomServers) { - this.configuration = conf; - this.grooms = groomServers; - if(1 > this.grooms) { - this.grooms = 2; - } - LOG.info("Groom server number "+this.grooms); - int threadpool = conf.getInt("bsp.test.threadpool", 10); - LOG.info("Thread pool value "+threadpool); - scheduler = Executors.newScheduledThreadPool(threadpool); - } - - public void startBSPCluster(){ - startMaster(); - startGroomServers(); - } - - public void shutdownBSPCluster(){ - if(null != this.master && this.master.isRunning()) - this.master.shutdown(); - if(0 < groomServerList.size()){ - for(GroomServerRunner groom: groomServerList){ - if(groom.isRunning()) groom.shutdown(); - } - } - } - - - public void startMaster(){ - if(null == this.scheduler) - throw new NullPointerException("No ScheduledExecutorService exists."); - this.master = new BSPMasterRunner(this.configuration); - scheduler.schedule(this.master, 0, SECONDS); - } - - public void startGroomServers(){ - if(null == this.scheduler) - throw new NullPointerException("No ScheduledExecutorService exists."); - if(null == this.master) - throw new NullPointerException("No BSPMaster exists."); - int cnt=0; - while(!this.master.isRunning()){ - LOG.info("Waiting BSPMaster up."); - try{ - Thread.sleep(1000); - cnt++; - if(100 < cnt){ - fail("Fail to launch BSPMaster."); - } - }catch(InterruptedException ie){ - LOG.error("Fail to check BSP Master's state.", ie); - Thread.currentThread().interrupt(); - } - } - for(int i=0; i < this.grooms; i++){ - HamaConfiguration c = new HamaConfiguration(this.configuration); - randomPort(c); - GroomServerRunner gsr = new GroomServerRunner(c); - groomServerList.add(gsr); - scheduler.schedule(gsr, 0, SECONDS); - cnt = 0; - while(!gsr.isRunning()){ - LOG.info("Waitin for GroomServer up."); - try{ - Thread.sleep(1000); - cnt++; - if(10 < cnt){ - fail("Fail to launch groom server."); - } - }catch(InterruptedException ie){ - LOG.error("Fail to check Groom Server's state.", ie); - Thread.currentThread().interrupt(); - } - } - } - - } - - private void randomPort(HamaConfiguration conf){ - try{ - ServerSocket skt = new ServerSocket(0); - int p = skt.getLocalPort(); - skt.close(); - conf.set(Constants.PEER_PORT, new Integer(p).toString()); - conf.setInt(Constants.GROOM_RPC_PORT, p+100); - }catch(IOException ioe){ - LOG.error("Can not find a free port for BSPPeer.", ioe); - } - } - - public void shutdown() { - shutdownBSPCluster(); - scheduler.shutdown(); - } - - public List getGroomServerThreads() { - List list = new ArrayList(); - for(GroomServerRunner gsr: groomServerList){ - list.add(new Thread(gsr)); - } - return list; - } - - public Thread getMaster() { - return new Thread(this.master); - } - - public List getGroomServers(){ - List list = new ArrayList(); - for(GroomServerRunner gsr: groomServerList){ - list.add(gsr.getGroomServer()); - } - return list; - } - - public BSPMaster getBSPMaster(){ - if(null != this.master) - return this.master.getMaster(); - return null; - } - - public ScheduledExecutorService getScheduler(){ - return this.scheduler; - } -} Index: src/test/org/apache/hama/zookeeper/TestZKTools.java =================================================================== --- src/test/org/apache/hama/zookeeper/TestZKTools.java (revision 1152138) +++ src/test/org/apache/hama/zookeeper/TestZKTools.java (working copy) @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.zookeeper; - -import org.apache.hama.Constants; -import org.apache.hama.HamaConfiguration; - -import junit.framework.TestCase; - -public class TestZKTools extends TestCase { - - public void testZKProps() { - HamaConfiguration conf = new HamaConfiguration(); - conf.set(Constants.ZOOKEEPER_QUORUM, "test.com:123"); - conf.set(Constants.ZOOKEEPER_CLIENT_PORT, "2222"); - - assertEquals("test.com:2222", QuorumPeer.getZKQuorumServersString(conf)); - } -} Index: src/test/org/apache/hama/HamaClusterTestCase.java =================================================================== --- src/test/org/apache/hama/HamaClusterTestCase.java (revision 1152138) +++ src/test/org/apache/hama/HamaClusterTestCase.java (working copy) @@ -1,107 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import java.io.File; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; - -public abstract class HamaClusterTestCase extends HamaTestCase { - public static final Log LOG = LogFactory.getLog(HamaClusterTestCase.class); - protected MiniDFSCluster dfsCluster; - protected MiniBSPCluster bspCluster; - protected MiniZooKeeperCluster zooKeeperCluster; - protected boolean startDfs; - protected int numOfGroom = 2; - - /** default constructor */ - public HamaClusterTestCase() { - this(false); - } - - public HamaClusterTestCase(boolean startDfs) { - super(); - this.startDfs = startDfs; - } - - /** - * Actually start the MiniBSP instance. - */ - protected void hamaClusterSetup() throws Exception { - File testDir = new File(getUnitTestdir(getName()).toString()); - - // Note that this is done before we create the MiniHamaCluster because we - // need to edit the config to add the ZooKeeper servers. - this.zooKeeperCluster = new MiniZooKeeperCluster(); - int clientPort = this.zooKeeperCluster.startup(testDir); - conf.set("hama.zookeeper.property.clientPort", Integer.toString(clientPort)); - bspCluster = new MiniBSPCluster(this.conf, numOfGroom); - bspCluster.startBSPCluster(); - } - - @Override - protected void setUp() throws Exception { - try { - if (this.startDfs) { - // This spews a bunch of warnings about missing scheme. TODO: fix. - this.dfsCluster = new MiniDFSCluster(0, this.conf, 2, true, true, true, - null, null, null, null); - - // mangle the conf so that the fs parameter points to the minidfs we - // just started up - FileSystem filesystem = dfsCluster.getFileSystem(); - conf.set("fs.defaultFS", filesystem.getUri().toString()); - Path parentdir = filesystem.getHomeDirectory(); - - filesystem.mkdirs(parentdir); - } - - // do the super setup now. if we had done it first, then we would have - // gotten our conf all mangled and a local fs started up. - super.setUp(); - - // start the instance - hamaClusterSetup(); - } catch (Exception e) { - if (zooKeeperCluster != null) { - zooKeeperCluster.shutdown(); - } - if (dfsCluster != null) { - shutdownDfs(dfsCluster); - } - throw e; - } - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - try { - if (startDfs) { - shutdownDfs(dfsCluster); - } - bspCluster.shutdown(); - } catch (Exception e) { - LOG.error(e); - } - } -} Index: src/test/org/apache/hama/bsp/TestBSPMessageBundle.java =================================================================== --- src/test/org/apache/hama/bsp/TestBSPMessageBundle.java (revision 1152138) +++ src/test/org/apache/hama/bsp/TestBSPMessageBundle.java (working copy) @@ -1,81 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.Arrays; -import junit.framework.TestCase; - -public class TestBSPMessageBundle extends TestCase { - - public void testEmpty() throws IOException { - BSPMessageBundle bundle = new BSPMessageBundle(); - // Serialize it. - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - bundle.write(new DataOutputStream(baos)); - baos.close(); - // Deserialize it. - BSPMessageBundle readBundle = new BSPMessageBundle(); - readBundle.readFields(new DataInputStream(new ByteArrayInputStream(baos - .toByteArray()))); - assertEquals(0, readBundle.getMessages().size()); - } - - public void testSerializationDeserialization() throws IOException { - BSPMessageBundle bundle = new BSPMessageBundle(); - ByteMessage[] testMessages = new ByteMessage[16]; - for (int i = 0; i < testMessages.length; ++i) { - // Create a one byte tag containing the number of the message. - byte[] tag = new byte[1]; - tag[0] = (byte) i; - // Create a four bytes data part containing serialized number of the - // message. - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - baos.write(i); - baos.close(); - byte[] data = baos.toByteArray(); - testMessages[i] = new ByteMessage(tag, data); - bundle.addMessage(testMessages[i]); - } - // Serialize it. - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - bundle.write(new DataOutputStream(baos)); - baos.close(); - // Deserialize it. - BSPMessageBundle readBundle = new BSPMessageBundle(); - readBundle.readFields(new DataInputStream(new ByteArrayInputStream(baos - .toByteArray()))); - // Check contents. - int messageNumber = 0; - for (BSPMessage message : readBundle.getMessages()) { - ByteMessage byteMessage = (ByteMessage) message; - assertTrue(Arrays.equals(testMessages[messageNumber].getTag(), - byteMessage.getTag())); - assertTrue(Arrays.equals(testMessages[messageNumber].getData(), - byteMessage.getData())); - ++messageNumber; - } - assertEquals(testMessages.length, messageNumber); - } -} Index: src/test/org/apache/hama/bsp/TestClusterStatus.java =================================================================== --- src/test/org/apache/hama/bsp/TestClusterStatus.java (revision 1152138) +++ src/test/org/apache/hama/bsp/TestClusterStatus.java (working copy) @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import junit.framework.TestCase; - -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hama.bsp.BSPMaster; -import org.apache.hama.bsp.ClusterStatus; - -public class TestClusterStatus extends TestCase { - Random rnd = new Random(); - - protected void setUp() throws Exception { - super.setUp(); - } - - public final void testWriteAndReadFields() throws IOException { - DataOutputBuffer out = new DataOutputBuffer(); - DataInputBuffer in = new DataInputBuffer(); - - ClusterStatus status1; - Map grooms = new HashMap(); - - for (int i = 0; i < 10; i++) { - int num = rnd.nextInt(); - String groomName = "groom_" + num; - String peerName = "peerhost:" + num; - grooms.put(groomName, peerName); - } - - int tasks = rnd.nextInt(100); - int maxTasks = rnd.nextInt(100); - BSPMaster.State state = BSPMaster.State.RUNNING; - - status1 = new ClusterStatus(grooms, tasks, maxTasks, state); - status1.write(out); - - in.reset(out.getData(), out.getLength()); - - ClusterStatus status2 = new ClusterStatus(); - status2.readFields(in); - - Map grooms_s = new HashMap(status1 - .getActiveGroomNames()); - Map grooms_o = new HashMap(status2 - .getActiveGroomNames()); - - assertEquals(status1.getGroomServers(), status2.getGroomServers()); - - assertTrue(grooms_s.entrySet().containsAll(grooms_o.entrySet())); - assertTrue(grooms_o.entrySet().containsAll(grooms_s.entrySet())); - - assertEquals(status1.getTasks(), status2.getTasks()); - assertEquals(status1.getMaxTasks(), status2.getMaxTasks()); - } -} Index: src/test/org/apache/hama/bsp/TestMessages.java =================================================================== --- src/test/org/apache/hama/bsp/TestMessages.java (revision 1152138) +++ src/test/org/apache/hama/bsp/TestMessages.java (working copy) @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import junit.framework.TestCase; - -import org.apache.hama.util.Bytes; - -public class TestMessages extends TestCase { - - public void testByteMessage() { - int dataSize = (int) (Runtime.getRuntime().maxMemory() * 0.60); - ByteMessage msg = new ByteMessage(Bytes.toBytes("tag"), new byte[dataSize]); - assertEquals(msg.getData().length, dataSize); - msg = null; - - byte[] dummyData = new byte[1024]; - ByteMessage msg2 = new ByteMessage(Bytes.tail(dummyData, 128), dummyData); - assertEquals( - Bytes.compareTo(msg2.getTag(), 0, 128, msg2.getData(), - msg2.getData().length - 128, 128), 0); - } -} Index: src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java =================================================================== --- src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java (revision 1152138) +++ src/test/org/apache/hama/bsp/TestBSPMasterGroomServer.java (working copy) @@ -1,94 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hama.Constants; -import org.apache.hama.HamaCluster; -import org.apache.hama.HamaConfiguration; - -public class TestBSPMasterGroomServer extends HamaCluster { - - private static Log LOG = LogFactory.getLog(TestBSPMasterGroomServer.class); - private static String TMP_OUTPUT = "/tmp/test-example/"; - private HamaConfiguration configuration; - private String TEST_JOB = "src/test/testjar/testjob.jar"; - - public TestBSPMasterGroomServer() { - configuration = new HamaConfiguration(); - configuration.set("bsp.master.address", "localhost"); - assertEquals("Make sure master addr is set to localhost:", "localhost", - configuration.get("bsp.master.address")); - configuration.setStrings("bsp.local.dir", "/tmp/hama-test"); - configuration.set(Constants.ZOOKEEPER_QUORUM, "localhost"); - configuration.setInt(Constants.ZOOKEEPER_CLIENT_PORT, 21810); - } - - public void setUp() throws Exception { - super.setUp(); - } - - public void testSubmitJob() throws Exception { - BSPJob bsp = new BSPJob(configuration); - bsp.setJobName("Test Serialize Printing"); - bsp.setBspClass(testjar.ClassSerializePrinting.HelloBSP.class); - bsp.setJar(TEST_JOB); - - // Set the task size as a number of GroomServer - BSPJobClient jobClient = new BSPJobClient(configuration); - configuration.setInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 600); - ClusterStatus cluster = jobClient.getClusterStatus(false); - assertEquals(this.numOfGroom, cluster.getMaxTasks()); - - // TODO test the multi-tasks - bsp.setNumBspTask(1); - - FileSystem fileSys = FileSystem.get(conf); - - if (bsp.waitForCompletion(true)) { - checkOutput(fileSys, cluster, conf); - } - LOG.info("Client finishes execution job."); - } - - private static void checkOutput(FileSystem fileSys, ClusterStatus cluster, - HamaConfiguration conf) throws Exception { - for (int i = 0; i < 1; i++) { // TODO test the multi-tasks - SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, new Path( - TMP_OUTPUT + i), conf); - LongWritable timestamp = new LongWritable(); - Text message = new Text(); - reader.next(timestamp, message); - assertTrue("Check if `Hello BSP' gets printed.", message.toString() - .indexOf("Hello BSP from") >= 0); - reader.close(); - } - } - - public void tearDown() throws Exception { - super.tearDown(); - } -} Index: src/test/org/apache/hama/HamaCluster.java =================================================================== --- src/test/org/apache/hama/HamaCluster.java (revision 1152138) +++ src/test/org/apache/hama/HamaCluster.java (working copy) @@ -1,47 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Forming up the miniDfs and miniZooKeeper - */ -public abstract class HamaCluster extends HamaClusterTestCase { - public static final Log LOG = LogFactory.getLog(HamaCluster.class); - protected final static HamaConfiguration conf = new HamaConfiguration(); - - public HamaCluster(){ - super(); - } - - public HamaCluster(boolean startDfs) { - super(startDfs); - } - - protected void setUp() throws Exception { - super.setUp(); - } - - protected static HamaConfiguration getConf() { - return conf; - } -} Index: src/test/org/apache/hama/ipc/TestRPC.java =================================================================== --- src/test/org/apache/hama/ipc/TestRPC.java (revision 1152138) +++ src/test/org/apache/hama/ipc/TestRPC.java (working copy) @@ -1,173 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ipc; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.net.InetSocketAddress; -import java.util.Arrays; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.ipc.VersionedProtocol; - -import junit.framework.TestCase; - -public class TestRPC extends TestCase { - private static final int PORT = 1234; - private static final String ADDRESS = "0.0.0.0"; - - public static final Log LOG = LogFactory - .getLog("org.apache.hadoop.ipc.TestRPC"); - - private static Configuration conf = new Configuration(); - - public TestRPC(String name) { - super(name); - } - - public interface TestProtocol extends VersionedProtocol { - public static final long versionID = 1L; - - void ping() throws IOException; - - String echo(String value) throws IOException; - - String[] echo(String[] value) throws IOException; - - Writable echo(Writable value) throws IOException; - - int add(int v1, int v2) throws IOException; - - int add(int[] values) throws IOException; - - int error() throws IOException; - - void testServerGet() throws IOException; - } - - public class TestImpl implements TestProtocol { - - public long getProtocolVersion(String protocol, long clientVersion) { - return TestProtocol.versionID; - } - - public void ping() { - } - - public String echo(String value) throws IOException { - return value; - } - - public String[] echo(String[] values) throws IOException { - return values; - } - - public Writable echo(Writable writable) { - return writable; - } - - public int add(int v1, int v2) { - return v1 + v2; - } - - public int add(int[] values) { - int sum = 0; - for (int i = 0; i < values.length; i++) { - sum += values[i]; - } - return sum; - } - - public int error() throws IOException { - throw new IOException("bobo"); - } - - public void testServerGet() throws IOException { - if (!(Server.get() instanceof RPC.Server)) { - throw new IOException("Server.get() failed"); - } - } - - } - - public void testCalls() throws Exception { - Server server = RPC.getServer(new TestImpl(), ADDRESS, PORT, conf); - server.start(); - - InetSocketAddress addr = new InetSocketAddress(PORT); - TestProtocol proxy = (TestProtocol) RPC.getProxy(TestProtocol.class, - TestProtocol.versionID, addr, conf); - - proxy.ping(); - - String stringResult = proxy.echo("foo"); - assertEquals(stringResult, "foo"); - - stringResult = proxy.echo((String) null); - assertEquals(stringResult, null); - - String[] stringResults = proxy.echo(new String[] { "foo", "bar" }); - assertTrue(Arrays.equals(stringResults, new String[] { "foo", "bar" })); - - stringResults = proxy.echo((String[]) null); - assertTrue(Arrays.equals(stringResults, null)); - - int intResult = proxy.add(1, 2); - assertEquals(intResult, 3); - - intResult = proxy.add(new int[] { 1, 2 }); - assertEquals(intResult, 3); - - boolean caught = false; - try { - proxy.error(); - } catch (IOException e) { - LOG.debug("Caught " + e); - caught = true; - } - assertTrue(caught); - - proxy.testServerGet(); - - // try some multi-calls - Method echo = TestProtocol.class.getMethod("echo", - new Class[] { String.class }); - String[] strings = (String[]) RPC.call(echo, new String[][] { { "a" }, - { "b" } }, new InetSocketAddress[] { addr, addr }, null, conf); - assertTrue(Arrays.equals(strings, new String[] { "a", "b" })); - - Method ping = TestProtocol.class.getMethod("ping", new Class[] {}); - Object[] voids = (Object[]) RPC.call(ping, new Object[][] { {}, {} }, - new InetSocketAddress[] { addr, addr }, null, conf); - assertEquals(voids, null); - - server.stop(); - } - - public static void main(String[] args) throws Exception { - - new TestRPC("test").testCalls(); - - } - -} Index: src/test/org/apache/hama/ipc/TestIPC.java =================================================================== --- src/test/org/apache/hama/ipc/TestIPC.java (revision 1152138) +++ src/test/org/apache/hama/ipc/TestIPC.java (working copy) @@ -1,232 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ipc; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Random; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.ipc.Client; -import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.util.StringUtils; - -public class TestIPC extends TestCase { - public static final Log LOG = LogFactory.getLog(TestIPC.class); - - final private static Configuration conf = new Configuration(); - final static private int PING_INTERVAL = 1000; - - static { - Client.setPingInterval(conf, PING_INTERVAL); - } - - public TestIPC(String name) { - super(name); - } - - private static final Random RANDOM = new Random(); - - private static final String ADDRESS = "0.0.0.0"; - - private static class TestServer extends Server { - private boolean sleep; - - public TestServer(int handlerCount, boolean sleep) throws IOException { - super(ADDRESS, 0, LongWritable.class, handlerCount, conf); - this.sleep = sleep; - } - - @Override - public Writable call(Class protocol, Writable param, long receiveTime) - throws IOException { - if (sleep) { - try { - Thread.sleep(RANDOM.nextInt(2 * PING_INTERVAL)); // sleep a bit - } catch (InterruptedException e) { - } - } - return param; // echo param as result - } - } - - private static class SerialCaller extends Thread { - private Client client; - private InetSocketAddress server; - private int count; - private boolean failed; - - public SerialCaller(Client client, InetSocketAddress server, int count) { - this.client = client; - this.server = server; - this.count = count; - } - - public void run() { - for (int i = 0; i < count; i++) { - try { - LongWritable param = new LongWritable(RANDOM.nextLong()); - LongWritable value = (LongWritable) client.call(param, server, null, - null); - if (!param.equals(value)) { - LOG.fatal("Call failed!"); - failed = true; - break; - } - } catch (Exception e) { - LOG.fatal("Caught: " + StringUtils.stringifyException(e)); - failed = true; - } - } - } - } - - private static class ParallelCaller extends Thread { - private Client client; - private int count; - private InetSocketAddress[] addresses; - private boolean failed; - - public ParallelCaller(Client client, InetSocketAddress[] addresses, - int count) { - this.client = client; - this.addresses = addresses; - this.count = count; - } - - public void run() { - for (int i = 0; i < count; i++) { - try { - Writable[] params = new Writable[addresses.length]; - for (int j = 0; j < addresses.length; j++) - params[j] = new LongWritable(RANDOM.nextLong()); - Writable[] values = client.call(params, addresses, null, null); - for (int j = 0; j < addresses.length; j++) { - if (!params[j].equals(values[j])) { - LOG.fatal("Call failed!"); - failed = true; - break; - } - } - } catch (Exception e) { - LOG.fatal("Caught: " + StringUtils.stringifyException(e)); - failed = true; - } - } - } - } - - public void testSerial() throws Exception { - testSerial(3, false, 2, 5, 100); - } - - public void testSerial(int handlerCount, boolean handlerSleep, - int clientCount, int callerCount, int callCount) throws Exception { - Server server = new TestServer(handlerCount, handlerSleep); - InetSocketAddress addr = NetUtils.getConnectAddress(server); - server.start(); - - Client[] clients = new Client[clientCount]; - for (int i = 0; i < clientCount; i++) { - clients[i] = new Client(LongWritable.class, conf); - } - - SerialCaller[] callers = new SerialCaller[callerCount]; - for (int i = 0; i < callerCount; i++) { - callers[i] = new SerialCaller(clients[i % clientCount], addr, callCount); - callers[i].start(); - } - for (int i = 0; i < callerCount; i++) { - callers[i].join(); - assertFalse(callers[i].failed); - } - for (int i = 0; i < clientCount; i++) { - clients[i].stop(); - } - server.stop(); - } - - public void testParallel() throws Exception { - testParallel(10, false, 2, 4, 2, 4, 100); - } - - public void testParallel(int handlerCount, boolean handlerSleep, - int serverCount, int addressCount, int clientCount, int callerCount, - int callCount) throws Exception { - Server[] servers = new Server[serverCount]; - for (int i = 0; i < serverCount; i++) { - servers[i] = new TestServer(handlerCount, handlerSleep); - servers[i].start(); - } - - InetSocketAddress[] addresses = new InetSocketAddress[addressCount]; - for (int i = 0; i < addressCount; i++) { - addresses[i] = NetUtils.getConnectAddress(servers[i % serverCount]); - } - - Client[] clients = new Client[clientCount]; - for (int i = 0; i < clientCount; i++) { - clients[i] = new Client(LongWritable.class, conf); - } - - ParallelCaller[] callers = new ParallelCaller[callerCount]; - for (int i = 0; i < callerCount; i++) { - callers[i] = new ParallelCaller(clients[i % clientCount], addresses, - callCount); - callers[i].start(); - } - for (int i = 0; i < callerCount; i++) { - callers[i].join(); - assertFalse(callers[i].failed); - } - for (int i = 0; i < clientCount; i++) { - clients[i].stop(); - } - for (int i = 0; i < serverCount; i++) { - servers[i].stop(); - } - } - - public void testStandAloneClient() throws Exception { - testParallel(10, false, 2, 4, 2, 4, 100); - Client client = new Client(LongWritable.class, conf); - InetSocketAddress address = new InetSocketAddress("127.0.0.1", 10); - try { - client.call(new LongWritable(RANDOM.nextLong()), address, null, null); - fail("Expected an exception to have been thrown"); - } catch (IOException e) { - String message = e.getMessage(); - String addressText = address.toString(); - assertTrue("Did not find " + addressText + " in " + message, message - .contains(addressText)); - Throwable cause = e.getCause(); - assertNotNull("No nested exception in " + e, cause); - String causeText = cause.getMessage(); - assertTrue("Did not find " + causeText + " in " + message, message - .contains(causeText)); - } - } - -} Index: src/test/org/apache/hama/util/TestNumeric.java =================================================================== --- src/test/org/apache/hama/util/TestNumeric.java (revision 1152138) +++ src/test/org/apache/hama/util/TestNumeric.java (working copy) @@ -1,34 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -import junit.framework.TestCase; - -public class TestNumeric extends TestCase { - final static int TEST_INT = 3; - final static double TEST_DOUBLE = 0.4; - - /** - * Double conversion test - */ - public void testDouble() { - assertEquals(Bytes.toDouble(Bytes.toBytes(TEST_DOUBLE)), TEST_DOUBLE); - } -} Index: src/test/org/apache/hama/util/TestBytes.java =================================================================== --- src/test/org/apache/hama/util/TestBytes.java (revision 1152138) +++ src/test/org/apache/hama/util/TestBytes.java (working copy) @@ -1,162 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -import java.io.IOException; -import java.util.Arrays; - -import junit.framework.TestCase; - -public class TestBytes extends TestCase { - public void testNullHashCode() { - byte [] b = null; - Exception ee = null; - try { - Bytes.hashCode(b); - } catch (Exception e) { - ee = e; - } - assertNotNull(ee); - } - - public void testSplit() throws Exception { - byte [] lowest = Bytes.toBytes("AAA"); - byte [] middle = Bytes.toBytes("CCC"); - byte [] highest = Bytes.toBytes("EEE"); - byte [][] parts = Bytes.split(lowest, highest, 1); - for (int i = 0; i < parts.length; i++) { - System.out.println(Bytes.toString(parts[i])); - } - assertEquals(3, parts.length); - assertTrue(Bytes.equals(parts[1], middle)); - // Now divide into three parts. Change highest so split is even. - highest = Bytes.toBytes("DDD"); - parts = Bytes.split(lowest, highest, 2); - for (int i = 0; i < parts.length; i++) { - System.out.println(Bytes.toString(parts[i])); - } - assertEquals(4, parts.length); - // Assert that 3rd part is 'CCC'. - assertTrue(Bytes.equals(parts[2], middle)); - } - - public void testSplit2() throws Exception { - // More split tests. - byte [] lowest = Bytes.toBytes("http://A"); - byte [] highest = Bytes.toBytes("http://z"); - byte [] middle = Bytes.toBytes("http://]"); - byte [][] parts = Bytes.split(lowest, highest, 1); - for (int i = 0; i < parts.length; i++) { - System.out.println(Bytes.toString(parts[i])); - } - assertEquals(3, parts.length); - assertTrue(Bytes.equals(parts[1], middle)); - } - - public void testToLong() throws Exception { - long [] longs = {-1l, 123l, 122232323232l}; - for (int i = 0; i < longs.length; i++) { - byte [] b = Bytes.toBytes(longs[i]); - assertEquals(longs[i], Bytes.toLong(b)); - } - } - - public void testToFloat() throws Exception { - float [] floats = {-1f, 123.123f, Float.MAX_VALUE}; - for (int i = 0; i < floats.length; i++) { - byte [] b = Bytes.toBytes(floats[i]); - assertEquals(floats[i], Bytes.toFloat(b)); - } - } - - public void testToDouble() throws Exception { - double [] doubles = {Double.MIN_VALUE, Double.MAX_VALUE}; - for (int i = 0; i < doubles.length; i++) { - byte [] b = Bytes.toBytes(doubles[i]); - assertEquals(doubles[i], Bytes.toDouble(b)); - } - } - - public void testBinarySearch() throws Exception { - byte [][] arr = { - {1}, - {3}, - {5}, - {7}, - {9}, - {11}, - {13}, - {15}, - }; - byte [] key1 = {3,1}; - byte [] key2 = {4,9}; - byte [] key2_2 = {4}; - byte [] key3 = {5,11}; - - assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1, - Bytes.BYTES_RAWCOMPARATOR)); - assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1, - Bytes.BYTES_RAWCOMPARATOR)); - assertEquals(-(2+1), Arrays.binarySearch(arr, key2_2, - Bytes.BYTES_COMPARATOR)); - assertEquals(-(2+1), Bytes.binarySearch(arr, key2, 0, 1, - Bytes.BYTES_RAWCOMPARATOR)); - assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1, - Bytes.BYTES_RAWCOMPARATOR)); - assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1, - Bytes.BYTES_RAWCOMPARATOR)); - assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1, - Bytes.BYTES_RAWCOMPARATOR)); - } - - public void testIncrementBytes() throws IOException { - - assertTrue(checkTestIncrementBytes(10, 1)); - assertTrue(checkTestIncrementBytes(12, 123435445)); - assertTrue(checkTestIncrementBytes(124634654, 1)); - assertTrue(checkTestIncrementBytes(10005460, 5005645)); - assertTrue(checkTestIncrementBytes(1, -1)); - assertTrue(checkTestIncrementBytes(10, -1)); - assertTrue(checkTestIncrementBytes(10, -5)); - assertTrue(checkTestIncrementBytes(1005435000, -5)); - assertTrue(checkTestIncrementBytes(10, -43657655)); - assertTrue(checkTestIncrementBytes(-1, 1)); - assertTrue(checkTestIncrementBytes(-26, 5034520)); - assertTrue(checkTestIncrementBytes(-10657200, 5)); - assertTrue(checkTestIncrementBytes(-12343250, 45376475)); - assertTrue(checkTestIncrementBytes(-10, -5)); - assertTrue(checkTestIncrementBytes(-12343250, -5)); - assertTrue(checkTestIncrementBytes(-12, -34565445)); - assertTrue(checkTestIncrementBytes(-1546543452, -34565445)); - } - - private static boolean checkTestIncrementBytes(long val, long amount) - throws IOException { - byte[] value = Bytes.toBytes(val); - byte [] testValue = {-1, -1, -1, -1, -1, -1, -1, -1}; - if (value[0] > 0) { - testValue = new byte[Bytes.SIZEOF_LONG]; - } - System.arraycopy(value, 0, testValue, testValue.length - value.length, - value.length); - - long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount)); - - return (Bytes.toLong(testValue) + amount) == incrementResult; - } -} Index: src/test/org/apache/hama/util/TestRandomVariable.java =================================================================== --- src/test/org/apache/hama/util/TestRandomVariable.java (revision 1152138) +++ src/test/org/apache/hama/util/TestRandomVariable.java (working copy) @@ -1,74 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -import org.apache.log4j.Logger; - -import junit.framework.TestCase; - -/** - * Random variable generation test - */ -public class TestRandomVariable extends TestCase { - static final Logger LOG = Logger.getLogger(TestRandomVariable.class); - final static int COUNT = 50; - - /** - * Random object test - * - * @throws Exception - */ - public void testRand() throws Exception { - for (int i = 0; i < COUNT; i++) { - double result = RandomVariable.rand(); - assertTrue(result >= 0.0d && result <= 1.0); - } - } - - /** - * Random integer test - * - * @throws Exception - */ - public void testRandInt() throws Exception { - final int min = 122; - final int max = 561; - - for (int i = 0; i < COUNT; i++) { - int result = RandomVariable.randInt(min, max); - assertTrue(result >= min && result <= max); - } - } - - /** - * Uniform test - * - * @throws Exception - */ - public void testUniform() throws Exception { - final double min = 1.0d; - final double max = 3.0d; - - for (int i = 0; i < COUNT; i++) { - double result = RandomVariable.uniform(min, max); - assertTrue(result >= min && result <= max); - } - } -} Index: src/test/testjar/testjob.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/test/testjar/ClassSerializePrinting.java =================================================================== --- src/test/testjar/ClassSerializePrinting.java (revision 1152138) +++ src/test/testjar/ClassSerializePrinting.java (working copy) @@ -1,86 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package testjar; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.io.Text; -import org.apache.hama.bsp.BSP; -import org.apache.hama.bsp.BSPPeer; -import org.apache.zookeeper.KeeperException; - -public class ClassSerializePrinting { - private static String TMP_OUTPUT = "/tmp/test-example/"; - - public static class HelloBSP extends BSP { - public static final Log LOG = LogFactory.getLog(HelloBSP.class); - private Configuration conf; - private final static int PRINT_INTERVAL = 1000; - private FileSystem fileSys; - private int num; - - public void bsp(BSPPeer bspPeer) throws IOException, - KeeperException, InterruptedException { - - int i = 0; - for (String otherPeer : bspPeer.getAllPeerNames()) { - String peerName = bspPeer.getPeerName(); - if (peerName.equals(otherPeer)) { - writeLogToFile(peerName, i); - } - - Thread.sleep(PRINT_INTERVAL); - bspPeer.sync(); - i++; - } - } - - private void writeLogToFile(String string, int i) throws IOException { - SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, - new Path(TMP_OUTPUT + i), LongWritable.class, Text.class, - CompressionType.NONE); - writer.append(new LongWritable(System.currentTimeMillis()), new Text( - "Hello BSP from " + (i + 1) + " of " + num + ": " + string)); - writer.close(); - } - - public Configuration getConf() { - return conf; - } - - public void setConf(Configuration conf) { - this.conf = conf; - num = Integer.parseInt(conf.get("bsp.peers.num")); - try { - fileSys = FileSystem.get(conf); - } catch (IOException e) { - e.printStackTrace(); - } - } - - } - -} Index: src/java/org/apache/hama/http/HttpServer.java =================================================================== --- src/java/org/apache/hama/http/HttpServer.java (revision 1152138) +++ src/java/org/apache/hama/http/HttpServer.java (working copy) @@ -1,501 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.http; - -import java.io.IOException; -import java.io.PrintWriter; -import java.net.BindException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.log.LogLevel; -import org.apache.hadoop.util.ReflectionUtils; -import org.mortbay.jetty.Connector; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.handler.ContextHandlerCollection; -import org.mortbay.jetty.nio.SelectChannelConnector; -import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.DefaultServlet; -import org.mortbay.jetty.servlet.FilterHolder; -import org.mortbay.jetty.servlet.FilterMapping; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; -import org.mortbay.jetty.webapp.WebAppContext; -import org.mortbay.thread.BoundedThreadPool; -import org.mortbay.util.MultiException; - -/** - * Create a Jetty embedded server to answer http requests. The primary goal is - * to serve up status information for the server. There are three contexts: - * "/logs/" -> points to the log directory "/static/" -> points to common static - * files (src/webapps/static) "/" -> the jsp server code from - * (src/webapps/) - */ -public class HttpServer { - public static final Log LOG = LogFactory.getLog(HttpServer.class); - - static final String FILTER_INITIALIZER_PROPERTY = "hama.http.filter.initializers"; - - protected final Server webServer; - protected final Connector listener; - protected final WebAppContext webAppContext; - protected final boolean findPort; - protected final Map defaultContexts = new HashMap(); - protected final List filterNames = new ArrayList(); - private static final int MAX_RETRIES = 10; - - /** Same as this(name, bindAddress, port, findPort, null); */ - public HttpServer(String name, String bindAddress, int port, boolean findPort) - throws IOException { - this(name, bindAddress, port, findPort, new Configuration()); - } - - /** - * Create a status server on the given port. The jsp scripts are taken from - * src/webapps/. - * - * @param name The name of the server - * @param port The port to use on the server - * @param findPort whether the server should start at the given port and - * increment by 1 until it finds a free port. - * @param conf Configuration - */ - public HttpServer(String name, String bindAddress, int port, - boolean findPort, Configuration conf) throws IOException { - webServer = new Server(); - this.findPort = findPort; - - listener = createBaseListener(conf); - listener.setHost(bindAddress); - listener.setPort(port); - webServer.addConnector(listener); - - webServer.setThreadPool(new BoundedThreadPool()); - - final String appDir = getWebAppsPath(); - ContextHandlerCollection contexts = new ContextHandlerCollection(); - webServer.setHandler(contexts); - webAppContext = new WebAppContext(); - - System.setProperty("java.naming.factory.initial", - "org.mortbay.naming.InitialContextFactory"); - System.setProperty("java.naming.factory.url.pkgs", "org.mortbay.naming"); - - webAppContext.setContextPath("/"); - webAppContext.setWar(appDir + "/" + name); - webServer.addHandler(webAppContext); - - addDefaultApps(contexts, appDir); - - addDefaultServlets(); - } - - /** - * Create a required listener for the Jetty instance listening on the port - * provided. This wrapper and all subclasses must create at least one - * listener. - */ - protected Connector createBaseListener(Configuration conf) throws IOException { - SelectChannelConnector ret = new SelectChannelConnector(); - ret.setLowResourceMaxIdleTime(10000); - ret.setAcceptQueueSize(128); - ret.setResolveNames(false); - ret.setUseDirectBuffers(false); - return ret; - } - - /** - * Add default apps. - * - * @param appDir The application directory - * @throws IOException - */ - protected void addDefaultApps(ContextHandlerCollection parent, - final String appDir) throws IOException { - // set up the context for "/logs/" if "hama.log.dir" property is defined. - String logDir = System.getProperty("hama.log.dir"); - if (logDir != null) { - Context logContext = new Context(parent, "/logs"); - logContext.setResourceBase(logDir); - logContext.addServlet(DefaultServlet.class, "/"); - defaultContexts.put(logContext, true); - } - // set up the context for "/static/*" - Context staticContext = new Context(parent, "/static"); - staticContext.setResourceBase(appDir + "/static"); - staticContext.addServlet(DefaultServlet.class, "/*"); - defaultContexts.put(staticContext, true); - } - - /** - * Add default servlets. - */ - protected void addDefaultServlets() { - // set up default servlets - addServlet("stacks", "/stacks", StackServlet.class); - addServlet("logLevel", "/logLevel", LogLevel.Servlet.class); - } - - public void addContext(Context ctxt, boolean isFiltered) throws IOException { - webServer.addHandler(ctxt); - defaultContexts.put(ctxt, isFiltered); - } - - /** - * Add a context - * - * @param pathSpec The path spec for the context - * @param dir The directory containing the context - * @param isFiltered if true, the servlet is added to the filter path mapping - * @throws IOException - */ - protected void addContext(String pathSpec, String dir, boolean isFiltered) - throws IOException { - if (0 == webServer.getHandlers().length) { - throw new RuntimeException("Couldn't find handler"); - } - WebAppContext webAppCtx = new WebAppContext(); - webAppCtx.setContextPath(pathSpec); - webAppCtx.setWar(dir); - addContext(webAppCtx, true); - } - - /** - * Set a value in the webapp context. These values are available to the jsp - * pages as "application.getAttribute(name)". - * - * @param name The name of the attribute - * @param value The value of the attribute - */ - public void setAttribute(String name, Object value) { - webAppContext.setAttribute(name, value); - } - - /** - * Add a servlet in the server. - * - * @param name The name of the servlet (can be passed as null) - * @param pathSpec The path spec for the servlet - * @param clazz The servlet class - */ - public void addServlet(String name, String pathSpec, - Class clazz) { - addInternalServlet(name, pathSpec, clazz); - addFilterPathMapping(pathSpec, webAppContext); - } - - /** - * Add an internal servlet in the server. - * - * @param name The name of the servlet (can be passed as null) - * @param pathSpec The path spec for the servlet - * @param clazz The servlet class - * @deprecated this is a temporary method - */ - @Deprecated - public void addInternalServlet(String name, String pathSpec, - Class clazz) { - ServletHolder holder = new ServletHolder(clazz); - if (name != null) { - holder.setName(name); - } - webAppContext.addServlet(holder, pathSpec); - } - - /** - * Define a filter for a context and set up default url mappings. - */ - protected void defineFilter(Context ctx, String name, String classname, - Map parameters, String[] urls) { - - FilterHolder holder = new FilterHolder(); - holder.setName(name); - holder.setClassName(classname); - holder.setInitParameters(parameters); - FilterMapping fmap = new FilterMapping(); - fmap.setPathSpecs(urls); - fmap.setDispatches(Handler.ALL); - fmap.setFilterName(name); - ServletHandler handler = ctx.getServletHandler(); - handler.addFilter(holder, fmap); - } - - /** - * Add the path spec to the filter path mapping. - * - * @param pathSpec The path spec - * @param webAppCtx The WebApplicationContext to add to - */ - protected void addFilterPathMapping(String pathSpec, Context webAppCtx) { - ServletHandler handler = webAppCtx.getServletHandler(); - for (String name : filterNames) { - FilterMapping fmap = new FilterMapping(); - fmap.setPathSpec(pathSpec); - fmap.setFilterName(name); - fmap.setDispatches(Handler.ALL); - handler.addFilterMapping(fmap); - } - } - - /** - * Get the value in the webapp context. - * - * @param name The name of the attribute - * @return The value of the attribute - */ - public Object getAttribute(String name) { - return webAppContext.getAttribute(name); - } - - /** - * Get the pathname to the webapps files. - * - * @return the pathname as a URL - * @throws IOException if 'webapps' directory cannot be found on CLASSPATH. - */ - protected String getWebAppsPath() throws IOException { - // URL url = BSPMaster.class.getClassLoader().getResource("webapps"); - // if (url == null) - // throw new IOException("webapps not found in CLASSPATH"); - // return url.toString(); - return "src/webapps"; - } - - /** - * Get the port that the server is on - * - * @return the port - */ - public int getPort() { - return webServer.getConnectors()[0].getLocalPort(); - } - - /** - * Set the min, max number of worker threads (simultaneous connections). - */ - public void setThreads(int min, int max) { - BoundedThreadPool pool = (BoundedThreadPool) webServer.getThreadPool(); - pool.setMinThreads(min); - pool.setMaxThreads(max); - } - - /** - * Configure an ssl listener on the server. - * - * @param addr address to listen on - * @param keystore location of the keystore - * @param storPass password for the keystore - * @param keyPass password for the key - * @deprecated Use - * {@link #addSslListener(InetSocketAddress, Configuration, boolean)} - */ - @Deprecated - public void addSslListener(InetSocketAddress addr, String keystore, - String storPass, String keyPass) throws IOException { - if (webServer.isStarted()) { - throw new IOException("Failed to add ssl listener"); - } - SslSocketConnector sslListener = new SslSocketConnector(); - sslListener.setHost(addr.getHostName()); - sslListener.setPort(addr.getPort()); - sslListener.setKeystore(keystore); - sslListener.setPassword(storPass); - sslListener.setKeyPassword(keyPass); - webServer.addConnector(sslListener); - } - - /** - * Configure an ssl listener on the server. - * - * @param addr address to listen on - * @param sslConf conf to retrieve ssl options - * @param needClientAuth whether client authentication is required - */ - public void addSslListener(InetSocketAddress addr, Configuration sslConf, - boolean needClientAuth) throws IOException { - if (webServer.isStarted()) { - throw new IOException("Failed to add ssl listener"); - } - if (needClientAuth) { - // setting up SSL truststore for authenticating clients - System.setProperty("javax.net.ssl.trustStore", sslConf.get( - "ssl.server.truststore.location", "")); - System.setProperty("javax.net.ssl.trustStorePassword", sslConf.get( - "ssl.server.truststore.password", "")); - System.setProperty("javax.net.ssl.trustStoreType", sslConf.get( - "ssl.server.truststore.type", "jks")); - } - SslSocketConnector sslListener = new SslSocketConnector(); - sslListener.setHost(addr.getHostName()); - sslListener.setPort(addr.getPort()); - sslListener.setKeystore(sslConf.get("ssl.server.keystore.location")); - sslListener.setPassword(sslConf.get("ssl.server.keystore.password", "")); - sslListener.setKeyPassword(sslConf.get("ssl.server.keystore.keypassword", - "")); - sslListener.setKeystoreType(sslConf.get("ssl.server.keystore.type", "jks")); - sslListener.setNeedClientAuth(needClientAuth); - webServer.addConnector(sslListener); - } - - /** - * Start the server. Does not wait for the server to start. - */ - public void start() throws IOException { - try { - int port = 0; - int oriPort = listener.getPort(); // The original requested port - while (true) { - try { - port = webServer.getConnectors()[0].getLocalPort(); - LOG.info("Port returned by webServer.getConnectors()[0]." - + "getLocalPort() before open() is " + port - + ". Opening the listener on " + oriPort); - listener.open(); - port = listener.getLocalPort(); - LOG.info("listener.getLocalPort() returned " - + listener.getLocalPort() - + " webServer.getConnectors()[0].getLocalPort() returned " - + webServer.getConnectors()[0].getLocalPort()); - // Workaround to handle the problem reported in HADOOP-4744 - if (port < 0) { - Thread.sleep(100); - int numRetries = 1; - while (port < 0) { - LOG.warn("listener.getLocalPort returned " + port); - if (numRetries++ > MAX_RETRIES) { - throw new Exception(" listener.getLocalPort is returning " - + "less than 0 even after " + numRetries + " resets"); - } - for (int i = 0; i < 2; i++) { - LOG.info("Retrying listener.getLocalPort()"); - port = listener.getLocalPort(); - if (port > 0) { - break; - } - Thread.sleep(200); - } - if (port > 0) { - break; - } - LOG.info("Bouncing the listener"); - listener.close(); - Thread.sleep(1000); - listener.setPort(oriPort == 0 ? 0 : (oriPort += 1)); - listener.open(); - Thread.sleep(100); - port = listener.getLocalPort(); - } - } // Workaround end - LOG.info("Jetty bound to port " + port); - webServer.start(); - // Workaround for HADOOP-6386 - port = listener.getLocalPort(); - if (port < 0) { - LOG.warn("Bounds port is " + port + " after webserver start"); - for (int i = 0; i < MAX_RETRIES / 2; i++) { - try { - webServer.stop(); - } catch (Exception e) { - LOG.warn("Can't stop web-server", e); - } - Thread.sleep(1000); - - listener.setPort(oriPort == 0 ? 0 : (oriPort += 1)); - listener.open(); - Thread.sleep(100); - webServer.start(); - LOG.info(i + "attempts to restart webserver"); - port = listener.getLocalPort(); - if (port > 0) - break; - } - if (port < 0) - throw new Exception("listener.getLocalPort() is returning " - + "less than 0 even after " + MAX_RETRIES + " resets"); - } - // End of HADOOP-6386 workaround - break; - } catch (IOException ex) { - // if this is a bind exception, - // then try the next port number. - if (ex instanceof BindException) { - if (!findPort) { - throw (BindException) ex; - } - } else { - LOG.info("HttpServer.start() threw a non Bind IOException"); - throw ex; - } - } catch (MultiException ex) { - LOG.info("HttpServer.start() threw a MultiException"); - throw ex; - } - listener.setPort((oriPort += 1)); - } - } catch (IOException e) { - throw e; - } catch (Exception e) { - throw new IOException("Problem starting http server", e); - } - } - - /** - * stop the server - */ - public void stop() throws Exception { - listener.close(); - webServer.stop(); - } - - public void join() throws InterruptedException { - webServer.join(); - } - - /** - * A very simple servlet to serve up a text representation of the current - * stack traces. It both returns the stacks to the caller and logs them. - * Currently the stack traces are done sequentially rather than exactly the - * same data. - */ - public static class StackServlet extends HttpServlet { - private static final long serialVersionUID = -6284183679759467039L; - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - PrintWriter out = new PrintWriter(response.getOutputStream()); - ReflectionUtils.printThreadInfo(out, ""); - out.close(); - ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1); - } - } -} Index: src/java/org/apache/hama/http/package.html =================================================================== --- src/java/org/apache/hama/http/package.html (revision 1152138) +++ src/java/org/apache/hama/http/package.html (working copy) @@ -1,23 +0,0 @@ - - - - - -Contains the administrative web interfaces. - - Index: src/java/org/apache/hama/MiniZooKeeperCluster.java =================================================================== --- src/java/org/apache/hama/MiniZooKeeperCluster.java (revision 1152138) +++ src/java/org/apache/hama/MiniZooKeeperCluster.java (working copy) @@ -1,216 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.Reader; -import java.net.BindException; -import java.net.InetSocketAddress; -import java.net.Socket; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileUtil; -import org.apache.zookeeper.server.NIOServerCnxn; -import org.apache.zookeeper.server.ZooKeeperServer; -import org.apache.zookeeper.server.persistence.FileTxnLog; - -/** - * This class starts and runs the MiniZookeeperCluster. - */ -public class MiniZooKeeperCluster { - private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class); - - private static final int TICK_TIME = 2000; - private static final int CONNECTION_TIMEOUT = 30000; - - private boolean started; - private int clientPort = 21810; // use non-standard port - - private NIOServerCnxn.Factory standaloneServerFactory; - private int tickTime = 0; - - /** Create mini Zookeeper cluster. */ - public MiniZooKeeperCluster() { - this.started = false; - } - - public void setClientPort(int clientPort) { - this.clientPort = clientPort; - } - - public void setTickTime(int tickTime) { - this.tickTime = tickTime; - } - - private static void setupTestEnv() { - // during the tests we run with 100K prealloc in the logs. - // on windows systems prealloc of 64M was seen to take ~15seconds - // resulting in test failure (client timeout on first session). - // set env and directly in order to handle static init/gc issues - System.setProperty("zookeeper.preAllocSize", "100"); - FileTxnLog.setPreallocSize(100); - } - - /** - * @param baseDir - * @return ClientPort server bound to. - * @throws IOException - * @throws InterruptedException - */ - public int startup(File baseDir) throws IOException, - InterruptedException { - - setupTestEnv(); - - shutdown(); - - File dir = new File(baseDir, "zookeeper").getAbsoluteFile(); - recreateDir(dir); - - int tickTimeToUse; - if (this.tickTime > 0) { - tickTimeToUse = this.tickTime; - } else { - tickTimeToUse = TICK_TIME; - } - ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse); - while (true) { - try { - standaloneServerFactory = - new NIOServerCnxn.Factory(new InetSocketAddress(clientPort)); - } catch (BindException e) { - LOG.info("Faild binding ZK Server to client port: " + clientPort); - //this port is already in use. try to use another - clientPort++; - continue; - } - break; - } - standaloneServerFactory.startup(server); - - if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) { - throw new IOException("Waiting for startup of standalone server"); - } - - started = true; - - return clientPort; - } - - private void recreateDir(File dir) throws IOException { - if (dir.exists()) { - FileUtil.fullyDelete(dir); - } - try { - dir.mkdirs(); - } catch (SecurityException e) { - throw new IOException("creating dir: " + dir, e); - } - } - - /** - * @throws IOException - */ - public void shutdown() throws IOException { - if (!started) { - return; - } - - standaloneServerFactory.shutdown(); - if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) { - throw new IOException("Waiting for shutdown of standalone server"); - } - - started = false; - } - - // XXX: From o.a.zk.t.ClientBase - private static boolean waitForServerDown(int port, long timeout) { - long start = System.currentTimeMillis(); - while (true) { - try { - Socket sock = new Socket("localhost", port); - try { - OutputStream outstream = sock.getOutputStream(); - outstream.write("stat".getBytes()); - outstream.flush(); - } finally { - sock.close(); - } - } catch (IOException e) { - return true; - } - - if (System.currentTimeMillis() > start + timeout) { - break; - } - try { - Thread.sleep(250); - } catch (InterruptedException e) { - // ignore - } - } - return false; - } - - // XXX: From o.a.zk.t.ClientBase - private static boolean waitForServerUp(int port, long timeout) { - long start = System.currentTimeMillis(); - while (true) { - try { - Socket sock = new Socket("localhost", port); - BufferedReader reader = null; - try { - OutputStream outstream = sock.getOutputStream(); - outstream.write("stat".getBytes()); - outstream.flush(); - - Reader isr = new InputStreamReader(sock.getInputStream()); - reader = new BufferedReader(isr); - String line = reader.readLine(); - if (line != null && line.startsWith("Zookeeper version:")) { - return true; - } - } finally { - sock.close(); - if (reader != null) { - reader.close(); - } - } - } catch (IOException e) { - // ignore as this is expected - LOG.info("server localhost:" + port + " not up " + e); - } - - if (System.currentTimeMillis() > start + timeout) { - break; - } - try { - Thread.sleep(250); - } catch (InterruptedException e) { - // ignore - } - } - return false; - } -} Index: src/java/org/apache/hama/zookeeper/QuorumPeer.java =================================================================== --- src/java/org/apache/hama/zookeeper/QuorumPeer.java (revision 1152138) +++ src/java/org/apache/hama/zookeeper/QuorumPeer.java (working copy) @@ -1,354 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.zookeeper; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintWriter; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.List; -import java.util.Properties; -import java.util.Map.Entry; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.DNS; -import org.apache.hadoop.util.StringUtils; -import org.apache.hama.Constants; -import org.apache.hama.HamaConfiguration; -import org.apache.zookeeper.server.ServerConfig; -import org.apache.zookeeper.server.ZooKeeperServerMain; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.apache.zookeeper.server.quorum.QuorumPeerMain; - -/** - * This class starts and runs the QuorumPeers - */ -public class QuorumPeer implements Constants { - private static final Log LOG = LogFactory.getLog(QuorumPeer.class); - - private static final String VARIABLE_START = "${"; - private static final int VARIABLE_START_LENGTH = VARIABLE_START.length(); - private static final String VARIABLE_END = "}"; - private static final int VARIABLE_END_LENGTH = VARIABLE_END.length(); - - private static final String ZK_CFG_PROPERTY = "hama.zookeeper.property."; - private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length(); - - /** - * Parse ZooKeeper configuration from Hama XML config and run a QuorumPeer. - * @param baseConf Hadoop Configuration. - */ - public static void run(Configuration baseConf) { - Configuration conf = new HamaConfiguration(baseConf); - try { - Properties zkProperties = makeZKProps(conf); - writeMyID(zkProperties); - QuorumPeerConfig zkConfig = new QuorumPeerConfig(); - zkConfig.parseProperties(zkProperties); - runZKServer(zkConfig); - } catch (Exception e) { - LOG.error("Exception during ZooKeeper startup - exiting...",e); - System.exit(-1); - } - } - - private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException { - if (zkConfig.isDistributed()) { - QuorumPeerMain qp = new QuorumPeerMain(); - qp.runFromConfig(zkConfig); - } else { - ZooKeeperServerMain zk = new ZooKeeperServerMain(); - ServerConfig serverConfig = new ServerConfig(); - serverConfig.readFrom(zkConfig); - zk.runFromConfig(serverConfig); - } - } - - private static boolean addressIsLocalHost(String address) { - return address.equals("localhost") || address.equals("127.0.0.1"); - } - - private static void writeMyID(Properties properties) throws IOException { - long myId = -1; - - Configuration conf = new HamaConfiguration(); - String myAddress = DNS.getDefaultHost( - conf.get("hama.zookeeper.dns.interface","default"), - conf.get("hama.zookeeper.dns.nameserver","default")); - - List ips = new ArrayList(); - - // Add what could be the best (configured) match - ips.add(myAddress.contains(".") ? - myAddress : - StringUtils.simpleHostname(myAddress)); - - // For all nics get all hostnames and IPs - Enumeration nics = NetworkInterface.getNetworkInterfaces(); - while(nics.hasMoreElements()) { - Enumeration rawAdrs = - ((NetworkInterface)nics.nextElement()).getInetAddresses(); - while(rawAdrs.hasMoreElements()) { - InetAddress inet = (InetAddress) rawAdrs.nextElement(); - ips.add(StringUtils.simpleHostname(inet.getHostName())); - ips.add(inet.getHostAddress()); - } - } - - for (Entry entry : properties.entrySet()) { - String key = entry.getKey().toString().trim(); - String value = entry.getValue().toString().trim(); - - if (key.startsWith("server.")) { - int dot = key.indexOf('.'); - long id = Long.parseLong(key.substring(dot + 1)); - String[] parts = value.split(":"); - String address = parts[0]; - if (addressIsLocalHost(address) || ips.contains(address)) { - myId = id; - break; - } - } - } - - if (myId == -1) { - throw new IOException("Could not find my address: " + myAddress + - " in list of ZooKeeper quorum servers"); - } - - String dataDirStr = properties.get("dataDir").toString().trim(); - File dataDir = new File(dataDirStr); - if (!dataDir.isDirectory()) { - if (!dataDir.mkdirs()) { - throw new IOException("Unable to create data dir " + dataDir); - } - } - - File myIdFile = new File(dataDir, "myid"); - PrintWriter w = new PrintWriter(myIdFile); - w.println(myId); - w.close(); - } - - /** - * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg. - * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse - * the corresponding config options from the Hama XML configs and generate - * the appropriate ZooKeeper properties. - * @param conf Configuration to read from. - * @return Properties holding mappings representing ZooKeeper zoo.cfg file. - */ - public static Properties makeZKProps(Configuration conf) { - // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read - // it and grab its configuration properties. - ClassLoader cl = QuorumPeer.class.getClassLoader(); - InputStream inputStream = cl.getResourceAsStream(ZOOKEEPER_CONFIG_NAME); - if (inputStream != null) { - try { - return parseZooCfg(conf, inputStream); - } catch (IOException e) { - LOG.warn("Cannot read " + ZOOKEEPER_CONFIG_NAME + - ", loading from XML files", e); - } - } - - // Otherwise, use the configuration options from Hama's XML files. - Properties zkProperties = new Properties(); - - // Set the max session timeout from the provided client-side timeout - zkProperties.setProperty("maxSessionTimeout", - conf.get(Constants.ZOOKEEPER_SESSION_TIMEOUT, "1200000")); - - // Directly map all of the hama.zookeeper.property.KEY properties. - for (Entry entry : conf) { - String key = entry.getKey(); - if (key.startsWith(ZK_CFG_PROPERTY)) { - String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE); - String value = entry.getValue(); - // If the value has variables substitutions, need to do a get. - if (value.contains(VARIABLE_START)) { - value = conf.get(key); - } - zkProperties.put(zkKey, value); - } - } - - // If clientPort is not set, assign the default - if (zkProperties.getProperty(ZOOKEEPER_CLIENT_PORT) == null) { - zkProperties.put(ZOOKEEPER_CLIENT_PORT, DEFAULT_ZOOKEEPER_CLIENT_PORT); - } - - // Create the server.X properties. - int peerPort = conf.getInt("hama.zookeeper.peerport", 2888); - int leaderPort = conf.getInt("hama.zookeeper.leaderport", 3888); - - String[] serverHosts = conf.getStrings(ZOOKEEPER_QUORUM, "localhost"); - for (int i = 0; i < serverHosts.length; ++i) { - String serverHost = serverHosts[i]; - String address = serverHost + ":" + peerPort + ":" + leaderPort; - String key = "server." + i; - zkProperties.put(key, address); - } - - return zkProperties; - } - - /** - * Parse ZooKeeper's zoo.cfg, injecting Hama Configuration variables in. - * This method is used for testing so we can pass our own InputStream. - * @param conf Configuration to use for injecting variables. - * @param inputStream InputStream to read from. - * @return Properties parsed from config stream with variables substituted. - * @throws IOException if anything goes wrong parsing config - */ - public static Properties parseZooCfg(Configuration conf, - InputStream inputStream) throws IOException { - Properties properties = new Properties(); - try { - properties.load(inputStream); - } catch (IOException e) { - String msg = "fail to read properties from " + ZOOKEEPER_CONFIG_NAME; - LOG.fatal(msg); - throw new IOException(msg, e); - } - for (Entry entry : properties.entrySet()) { - String value = entry.getValue().toString().trim(); - String key = entry.getKey().toString().trim(); - StringBuilder newValue = new StringBuilder(); - int varStart = value.indexOf(VARIABLE_START); - int varEnd = 0; - while (varStart != -1) { - varEnd = value.indexOf(VARIABLE_END, varStart); - if (varEnd == -1) { - String msg = "variable at " + varStart + " has no end marker"; - LOG.fatal(msg); - throw new IOException(msg); - } - String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd); - - String substituteValue = System.getProperty(variable); - if (substituteValue == null) { - substituteValue = conf.get(variable); - } - if (substituteValue == null) { - String msg = "variable " + variable + " not set in system property " - + "or hama configs"; - LOG.fatal(msg); - throw new IOException(msg); - } - - newValue.append(substituteValue); - - varEnd += VARIABLE_END_LENGTH; - varStart = value.indexOf(VARIABLE_START, varEnd); - } - // Special case for 'hama.cluster.distributed' property being 'true' - if (key.startsWith("server.")) { - if (conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) && - value.startsWith("localhost")) { - String msg = "The server in zoo.cfg cannot be set to localhost " + - "in a fully-distributed setup because it won't be reachable. " + - "See \"Getting Started\" for more information."; - LOG.fatal(msg); - throw new IOException(msg); - } - } - newValue.append(value.substring(varEnd)); - properties.setProperty(key, newValue.toString()); - } - return properties; - } - - /** - * Return the ZK Quorum servers string given zk properties returned by - * makeZKProps - * @param properties - * @return Quorum servers String - */ - public static String getZKQuorumServersString(Properties properties) { - String clientPort = null; - List servers = new ArrayList(); - - // The clientPort option may come after the server.X hosts, so we need to - // grab everything and then create the final host:port comma separated list. - boolean anyValid = false; - for (Entry property : properties.entrySet()) { - String key = property.getKey().toString().trim(); - String value = property.getValue().toString().trim(); - if (key.equals("clientPort")) { - clientPort = value; - } - else if (key.startsWith("server.")) { - String host = value.substring(0, value.indexOf(':')); - servers.add(host); - try { - //noinspection ResultOfMethodCallIgnored - InetAddress.getByName(host); - anyValid = true; - } catch (UnknownHostException e) { - LOG.warn(StringUtils.stringifyException(e)); - } - } - } - - if (!anyValid) { - LOG.error("no valid quorum servers found in " + Constants.ZOOKEEPER_CONFIG_NAME); - return null; - } - - if (clientPort == null) { - LOG.error("no clientPort found in " + Constants.ZOOKEEPER_CONFIG_NAME); - return null; - } - - if (servers.isEmpty()) { - LOG.fatal("No server.X lines found in conf/zoo.cfg. Hama must have a " + - "ZooKeeper cluster configured for its operation."); - return null; - } - - StringBuilder hostPortBuilder = new StringBuilder(); - for (int i = 0; i < servers.size(); ++i) { - String host = servers.get(i); - if (i > 0) { - hostPortBuilder.append(','); - } - hostPortBuilder.append(host); - hostPortBuilder.append(':'); - hostPortBuilder.append(clientPort); - } - - return hostPortBuilder.toString(); - } - - /** - * Return the ZK Quorum servers string given the specified configuration. - * @param conf - * @return Quorum servers - */ - public static String getZKQuorumServersString(Configuration conf) { - return getZKQuorumServersString(makeZKProps(conf)); - } -} Index: src/java/org/apache/hama/zookeeper/ZKServerTool.java =================================================================== --- src/java/org/apache/hama/zookeeper/ZKServerTool.java (revision 1152138) +++ src/java/org/apache/hama/zookeeper/ZKServerTool.java (working copy) @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.zookeeper; - -import java.util.Properties; -import java.util.Map.Entry; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hama.HamaConfiguration; - -/** - * A tool class for Zookeeper use. - */ -public class ZKServerTool { - - /** - * Run the tool. - * - * @param args Command line arguments. First arg is path to zookeepers file. - */ - public static void main(String args[]) { - Configuration conf = new HamaConfiguration(); - // Note that we do not simply grab the property ZOOKEEPER_QUORUM from - // the HamaConfiguration because the user may be using a zoo.cfg file. - Properties zkProps = QuorumPeer.makeZKProps(conf); - for (Entry entry : zkProps.entrySet()) { - String key = entry.getKey().toString().trim(); - String value = entry.getValue().toString().trim(); - if (key.startsWith("server.")) { - String[] parts = value.split(":"); - String host = parts[0]; - System.out.println(host); - } - } - } -} Index: src/java/org/apache/hama/zookeeper/package.html =================================================================== --- src/java/org/apache/hama/zookeeper/package.html (revision 1152138) +++ src/java/org/apache/hama/zookeeper/package.html (working copy) @@ -1,23 +0,0 @@ - - - - - -Tools to start the ZooKeeper server and quorum peers. - - Index: src/java/org/apache/hama/GroomServerRunner.java =================================================================== --- src/java/org/apache/hama/GroomServerRunner.java (revision 1152138) +++ src/java/org/apache/hama/GroomServerRunner.java (working copy) @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hama.bsp.GroomServer; - -/** - * This class starts and runs the GroomServer. - */ -public class GroomServerRunner extends Configured implements Tool { - - public static final Log LOG = LogFactory.getLog(GroomServerRunner.class); - - @Override - public int run(String[] args) throws Exception { - StringUtils.startupShutdownMessage(GroomServer.class, args, LOG); - - if (args.length != 0) { - System.out.println("usage: GroomServerRunner"); - System.exit(-1); - } - - try { - Configuration conf = new HamaConfiguration(getConf()); - GroomServer groom = GroomServer.constructGroomServer(GroomServer.class, - conf); - GroomServer.startGroomServer(groom).join(); - } catch (Throwable e) { - LOG.fatal(StringUtils.stringifyException(e)); - return -1; - } - - return 0; - } - - public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new GroomServerRunner(), args); - System.exit(exitCode); - } - -} Index: src/java/org/apache/hama/bsp/BSPMaster.java =================================================================== --- src/java/org/apache/hama/bsp/BSPMaster.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPMaster.java (working copy) @@ -1,730 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.net.InetSocketAddress; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.LinkedBlockingQueue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.http.HttpServer; -import org.apache.hama.ipc.JobSubmissionProtocol; -import org.apache.hama.ipc.MasterProtocol; -import org.apache.hama.ipc.GroomProtocol; - -/** - * BSPMaster is responsible to control all the groom servers and to manage bsp - * jobs. - */ -public class BSPMaster implements JobSubmissionProtocol, MasterProtocol, - GroomServerManager { - public static final Log LOG = LogFactory.getLog(BSPMaster.class); - - private HamaConfiguration conf; - - /** - * Constants for BSPMaster's status. - */ - public static enum State { - INITIALIZING, RUNNING - } - - private static final int FS_ACCESS_RETRY_PERIOD = 10000; - public static final long GROOMSERVER_EXPIRY_INTERVAL = 10 * 60 * 1000; - static long JOBINIT_SLEEP_INTERVAL = 2000; - - // States - State state = State.INITIALIZING; - - // Attributes - String masterIdentifier; - // private Server interServer; - private Server masterServer; - - // host and port - private String host; - private int port; - - // startTime - private long startTime; - - // HTTP server - private HttpServer infoServer; - private int infoPort; - - // Filesystem - static final String SUBDIR = "bspMaster"; - FileSystem fs = null; - Path systemDir = null; - // system directories are world-wide readable and owner readable - final static FsPermission SYSTEM_DIR_PERMISSION = FsPermission - .createImmutable((short) 0733); // rwx-wx-wx - // system files should have 700 permission - final static FsPermission SYSTEM_FILE_PERMISSION = FsPermission - .createImmutable((short) 0700); // rwx------ - - // Jobs' Meta Data - private Integer nextJobId = Integer.valueOf(1); - private int totalSubmissions = 0; // how many jobs has been submitted by - // clients - private int totalTasks = 0; // currnetly running tasks - private int totalTaskCapacity; // max tasks that groom server can run - - private Map jobs = new TreeMap(); - private TaskScheduler taskScheduler; - - // GroomServers cache - protected ConcurrentMap groomServers = new ConcurrentHashMap(); - - private Instructor instructor; - - private final List jobInProgressListeners = new CopyOnWriteArrayList(); - - private class ReportGroomStatusHandler implements DirectiveHandler { - - public void handle(Directive directive) throws DirectiveException { - // update GroomServerStatus held in the groomServers cache. - GroomServerStatus groomStatus = ((ReportGroomStatusDirective) directive) - .getStatus(); - // groomServers cache contains groom server status reported back - if (groomServers.containsKey(groomStatus)) { - GroomServerStatus tmpStatus = null; - for (GroomServerStatus old : groomServers.keySet()) { - if (old.equals(groomStatus)) { - tmpStatus = groomStatus; - updateGroomServersKey(old, tmpStatus); - break; - } - }// for - if (null != tmpStatus) { - List tlist = tmpStatus.getTaskReports(); - for (TaskStatus ts : tlist) { - JobInProgress jip = whichJob(ts.getJobId()); - TaskInProgress tip = jip.findTaskInProgress(((TaskAttemptID) ts - .getTaskId()).getTaskID()); - - if (ts.getRunState() == TaskStatus.State.SUCCEEDED) { - jip.completedTask(tip, ts); - } else if (ts.getRunState() == TaskStatus.State.RUNNING) { - // do nothing - } else if (ts.getRunState() == TaskStatus.State.FAILED) { - jip.status.setRunState(JobStatus.FAILED); - jip.failedTask(tip, ts); - } - if (jip.getStatus().getRunState() == JobStatus.SUCCEEDED) { - for (JobInProgressListener listener : jobInProgressListeners) { - try { - listener.jobRemoved(jip); - } catch (IOException ioe) { - LOG.error("Fail to alter scheduler a job is moved.", ioe); - } - } - } else if (jip.getStatus().getRunState() == JobStatus.RUNNING) { - jip.getStatus().setprogress(ts.getSuperstepCount()); - } else if (jip.getStatus().getRunState() == JobStatus.KILLED) { - GroomProtocol worker = findGroomServer(tmpStatus); - Directive d1 = new DispatchTasksDirective( - currentGroomServerPeers(), - new GroomServerAction[] { new KillTaskAction(ts.getTaskId()) }); - try { - worker.dispatch(d1); - } catch (IOException ioe) { - throw new DirectiveException("Error when dispatching kill task" - + " action.", ioe); - } - } - } - } else { - throw new RuntimeException("BSPMaster contains GroomServerSatus, " - + "but fail to retrieve it."); - } - } else { - throw new RuntimeException("GroomServer not found." - + groomStatus.getGroomName()); - } - } - } - - private class Instructor extends Thread { - private final BlockingQueue buffer = new LinkedBlockingQueue(); - private final ConcurrentMap, DirectiveHandler> handlers = new ConcurrentHashMap, DirectiveHandler>(); - - public void bind(Class instruction, - DirectiveHandler handler) { - handlers.putIfAbsent(instruction, handler); - } - - public void put(Directive directive) { - try { - buffer.put(directive); - } catch (InterruptedException ie) { - LOG.error("Fail to put directive into queue.", ie); - } - } - - public void run() { - while (true) { - try { - Directive directive = this.buffer.take(); - if (directive instanceof ReportGroomStatusDirective) { - ((DirectiveHandler) handlers.get(ReportGroomStatusDirective.class)) - .handle(directive); - } else { - throw new RuntimeException("Directive is not supported." - + directive); - } - } catch (InterruptedException ie) { - LOG.error("Unable to retrieve directive from the queue.", ie); - Thread.currentThread().interrupt(); - } catch (Exception e) { - LOG.error("Fail to execute directive command.", e); - } - } - } - } - - /** - * Start the BSPMaster process, listen on the indicated hostname/port - */ - public BSPMaster(HamaConfiguration conf) throws IOException, - InterruptedException { - this(conf, generateNewIdentifier()); - } - - BSPMaster(HamaConfiguration conf, String identifier) throws IOException, - InterruptedException { - this.conf = conf; - this.masterIdentifier = identifier; - - // Create the scheduler and init scheduler services - Class schedulerClass = conf.getClass( - "bsp.master.taskscheduler", SimpleTaskScheduler.class, - TaskScheduler.class); - this.taskScheduler = (TaskScheduler) ReflectionUtils.newInstance( - schedulerClass, conf); - - host = getAddress(conf).getHostName(); - port = getAddress(conf).getPort(); - LOG.info("RPC BSPMaster: host " + host + " port " + port); - - startTime = System.currentTimeMillis(); - this.masterServer = RPC.getServer(this, host, port, conf); - - infoPort = conf.getInt("bsp.http.infoserver.port", 40013); - - infoServer = new HttpServer("bspmaster", host, infoPort, true, conf); - infoServer.setAttribute("bsp.master", this); - - // starting webserver - infoServer.start(); - - while (!Thread.currentThread().isInterrupted()) { - try { - if (fs == null) { - fs = FileSystem.get(conf); - } - // clean up the system dir, which will only work if hdfs is out of - // safe mode - if (systemDir == null) { - systemDir = new Path(getSystemDir()); - } - - LOG.info("Cleaning up the system directory"); - LOG.info(systemDir); - fs.delete(systemDir, true); - if (FileSystem.mkdirs(fs, systemDir, new FsPermission( - SYSTEM_DIR_PERMISSION))) { - break; - } - LOG.error("Mkdirs failed to create " + systemDir); - LOG.info(SUBDIR); - - } catch (AccessControlException ace) { - LOG.warn("Failed to operate on bsp.system.dir (" + systemDir - + ") because of permissions."); - LOG.warn("Manually delete the bsp.system.dir (" + systemDir - + ") and then start the BSPMaster."); - LOG.warn("Bailing out ... "); - throw ace; - } catch (IOException ie) { - LOG.info("problem cleaning system directory: " + systemDir, ie); - } - Thread.sleep(FS_ACCESS_RETRY_PERIOD); - } - - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException(); - } - - deleteLocalFiles(SUBDIR); - } - - /** - * A GroomServer registers with its status to BSPMaster when startup, which - * will update GroomServers cache. - * - * @param status to be updated in cache. - * @return true if registering successfully; false if fail. - */ - @Override - public boolean register(GroomServerStatus status) throws IOException { - if (null == status) { - LOG.error("No groom server status."); - throw new NullPointerException("No groom server status."); - } - Throwable e = null; - try { - GroomProtocol wc = (GroomProtocol) RPC.waitForProxy( - GroomProtocol.class, GroomProtocol.versionID, - resolveWorkerAddress(status.getRpcServer()), this.conf); - if (null == wc) { - LOG.warn("Fail to create Worker client at host " + status.getPeerName()); - return false; - } - // TODO: need to check if peer name has changed - groomServers.putIfAbsent(status, wc); - } catch (UnsupportedOperationException u) { - e = u; - } catch (ClassCastException c) { - e = c; - } catch (NullPointerException n) { - e = n; - } catch (IllegalArgumentException i) { - e = i; - } catch (Exception ex) { - e = ex; - } - - if (null != e) { - LOG.error("Fail to register GroomServer " + status.getGroomName(), e); - return false; - } - - return true; - } - - private static InetSocketAddress resolveWorkerAddress(String data) { - return new InetSocketAddress(data.split(":")[0], Integer.parseInt(data - .split(":")[1])); - } - - private void updateGroomServersKey(GroomServerStatus old, - GroomServerStatus newKey) { - synchronized (groomServers) { - GroomProtocol worker = groomServers.remove(old); - groomServers.put(newKey, worker); - } - } - - @Override - public boolean report(Directive directive) throws IOException { - instructor.put(directive); - return true; - } - - private JobInProgress whichJob(BSPJobID id) { - for (JobInProgress job : taskScheduler - .getJobs(SimpleTaskScheduler.PROCESSING_QUEUE)) { - if (job.getJobID().equals(id)) { - return job; - } - } - return null; - } - - // ///////////////////////////////////////////////////////////// - // BSPMaster methods - // ///////////////////////////////////////////////////////////// - - // Get the job directory in system directory - Path getSystemDirectoryForJob(BSPJobID id) { - return new Path(getSystemDir(), id.toString()); - } - - String[] getLocalDirs() throws IOException { - return conf.getStrings("bsp.local.dir"); - } - - void deleteLocalFiles() throws IOException { - String[] localDirs = getLocalDirs(); - for (int i = 0; i < localDirs.length; i++) { - FileSystem.getLocal(conf).delete(new Path(localDirs[i]), true); - } - } - - void deleteLocalFiles(String subdir) throws IOException { - try { - String[] localDirs = getLocalDirs(); - for (int i = 0; i < localDirs.length; i++) { - FileSystem.getLocal(conf).delete(new Path(localDirs[i], subdir), true); - } - } catch (NullPointerException e) { - LOG.info(e); - } - } - - /** - * Constructs a local file name. Files are distributed among configured local - * directories. - */ - Path getLocalPath(String pathString) throws IOException { - return conf.getLocalPath("bsp.local.dir", pathString); - } - - public static BSPMaster startMaster(HamaConfiguration conf) - throws IOException, InterruptedException { - return startMaster(conf, generateNewIdentifier()); - } - - public static BSPMaster startMaster(HamaConfiguration conf, String identifier) - throws IOException, InterruptedException { - - BSPMaster result = new BSPMaster(conf, identifier); - result.taskScheduler.setGroomServerManager(result); - result.taskScheduler.start(); - - return result; - } - - public static InetSocketAddress getAddress(Configuration conf) { - String hamaMasterStr = conf.get("bsp.master.address", "localhost"); - int defaultPort = conf.getInt("bsp.master.port", 40000); - - return NetUtils.createSocketAddr(hamaMasterStr, defaultPort); - } - - /** - * BSPMaster identifier - * - * @return String BSPMaster identification number - */ - private static String generateNewIdentifier() { - return new SimpleDateFormat("yyyyMMddHHmm").format(new Date()); - } - - public void offerService() throws InterruptedException, IOException { - - this.masterServer.start(); - - synchronized (this) { - state = State.RUNNING; - } - - instructor = new Instructor(); - instructor.bind(ReportGroomStatusDirective.class, - new ReportGroomStatusHandler()); - instructor.start(); - - LOG.info("Starting RUNNING"); - - this.masterServer.join(); - - LOG.info("Stopped RPC Master server."); - } - - // ////////////////////////////////////////////////// - // InterServerProtocol - // ////////////////////////////////////////////////// - @Override - public long getProtocolVersion(String protocol, long clientVersion) - throws IOException { - if (protocol.equals(MasterProtocol.class.getName())) { - return MasterProtocol.versionID; - } else if (protocol.equals(JobSubmissionProtocol.class.getName())) { - return JobSubmissionProtocol.versionID; - } else { - throw new IOException("Unknown protocol to BSPMaster: " + protocol); - } - } - - // ////////////////////////////////////////////////// - // JobSubmissionProtocol - // ////////////////////////////////////////////////// - /** - * This method returns new job id. The returned job id increases sequentially. - */ - @Override - public BSPJobID getNewJobId() throws IOException { - int id; - synchronized (nextJobId) { - id = nextJobId; - nextJobId = Integer.valueOf(id + 1); - } - return new BSPJobID(this.masterIdentifier, id); - } - - @Override - public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException { - if (jobs.containsKey(jobID)) { - // job already running, don't start twice - LOG.info("The job (" + jobID + ") was already submitted"); - return jobs.get(jobID).getStatus(); - } - - JobInProgress job = new JobInProgress(jobID, new Path(jobFile), this, - this.conf); - return addJob(jobID, job); - } - - // ////////////////////////////////////////////////// - // GroomServerManager functions - // ////////////////////////////////////////////////// - - @Override - public ClusterStatus getClusterStatus(boolean detailed) { - Map groomPeersMap = null; - - // give the caller a snapshot of the cluster status - int numGroomServers = groomServers.size(); - if (detailed) { - groomPeersMap = new HashMap(); - for (Map.Entry entry : groomServers - .entrySet()) { - GroomServerStatus s = entry.getKey(); - groomPeersMap.put(s.getGroomName(), s.getPeerName()); - } - } - - // TODO currently we only have one task slot per groom server - this.totalTaskCapacity = numGroomServers; - - if (detailed) { - return new ClusterStatus(groomPeersMap, totalTasks, totalTaskCapacity, - state); - } else { - return new ClusterStatus(numGroomServers, totalTasks, totalTaskCapacity, - state); - } - } - - @Override - public GroomProtocol findGroomServer(GroomServerStatus status) { - return groomServers.get(status); - } - - @Override - public Collection findGroomServers() { - return groomServers.values(); - } - - @Override - public Collection groomServerStatusKeySet() { - return groomServers.keySet(); - } - - @Override - public void addJobInProgressListener(JobInProgressListener listener) { - jobInProgressListeners.add(listener); - } - - @Override - public void removeJobInProgressListener(JobInProgressListener listener) { - jobInProgressListeners.remove(listener); - } - - @Override - public Map currentGroomServerPeers() { - Map tmp = new HashMap(); - for (GroomServerStatus status : groomServers.keySet()) { - tmp.put(status.getGroomName(), status.getPeerName()); - } - return tmp; - } - - public String getBSPMasterName() { - return host + ":" + port; - } - - public long getStartTime() { - return startTime; - } - - public String getBSPMasterIdentifier() { - return masterIdentifier; - } - - public int getHttpPort() { - return infoPort; - } - - /** - * Adds a job to the bsp master. Make sure that the checks are inplace before - * adding a job. This is the core job submission logic - * - * @param jobId The id for the job submitted which needs to be added - */ - private synchronized JobStatus addJob(BSPJobID jobId, JobInProgress job) { - totalSubmissions++; - synchronized (jobs) { - jobs.put(job.getProfile().getJobID(), job); - for (JobInProgressListener listener : jobInProgressListeners) { - try { - listener.jobAdded(job); - } catch (IOException ioe) { - LOG.error("Fail to alter Scheduler a job is added.", ioe); - } - } - } - return job.getStatus(); - } - - @Override - public JobStatus[] jobsToComplete() throws IOException { - return getJobStatus(jobs.values(), true); - } - - @Override - public JobStatus[] getAllJobs() throws IOException { - LOG.debug("returns all jobs: " + jobs.size()); - return getJobStatus(jobs.values(), false); - } - - private synchronized JobStatus[] getJobStatus(Collection jips, - boolean toComplete) { - if (jips == null) { - return new JobStatus[] {}; - } - List jobStatusList = new ArrayList(); - for (JobInProgress jip : jips) { - JobStatus status = jip.getStatus(); - - status.setStartTime(jip.getStartTime()); - // Sets the user name - status.setUsername(jip.getProfile().getUser()); - status.setName(jip.getJobName()); - - if (toComplete) { - if (status.getRunState() == JobStatus.RUNNING - || status.getRunState() == JobStatus.PREP) { - jobStatusList.add(status); - } - } else { - jobStatusList.add(status); - } - } - - return jobStatusList.toArray(new JobStatus[jobStatusList.size()]); - } - - @Override - public synchronized String getFilesystemName() throws IOException { - if (fs == null) { - throw new IllegalStateException("FileSystem object not available yet"); - } - return fs.getUri().toString(); - } - - /** - * Return system directory to which BSP store control files. - */ - @Override - public String getSystemDir() { - Path sysDir = new Path(conf.get("bsp.system.dir", "/tmp/hadoop/bsp/system")); - return fs.makeQualified(sysDir).toString(); - } - - @Override - public JobProfile getJobProfile(BSPJobID jobid) throws IOException { - synchronized (this) { - JobInProgress job = jobs.get(jobid); - if (job != null) { - return job.getProfile(); - } - } - return null; - } - - @Override - public JobStatus getJobStatus(BSPJobID jobid) throws IOException { - synchronized (this) { - JobInProgress job = jobs.get(jobid); - if (job != null) { - return job.getStatus(); - } - } - return null; - } - - @Override - public void killJob(BSPJobID jobid) throws IOException { - JobInProgress job = jobs.get(jobid); - - if (null == job) { - LOG.info("killJob(): JobId " + jobid.toString() + " is not a valid job"); - return; - } - - killJob(job); - } - - private synchronized void killJob(JobInProgress job) { - LOG.info("Killing job " + job.getJobID()); - job.kill(); - } - - @Override - public boolean killTask(TaskAttemptID taskId, boolean shouldFail) - throws IOException { - return false; - } - - public static BSPMaster constructMaster( - Class masterClass, final Configuration conf) { - try { - Constructor c = masterClass - .getConstructor(Configuration.class); - return c.newInstance(conf); - } catch (Exception e) { - throw new RuntimeException("Failed construction of " + "Master: " - + masterClass.toString() - + ((e.getCause() != null) ? e.getCause().getMessage() : ""), e); - } - } - - public void shutdown() { - this.masterServer.stop(); - } - - public BSPMaster.State currentState() { - return this.state; - } -} Index: src/java/org/apache/hama/bsp/TaskScheduler.java =================================================================== --- src/java/org/apache/hama/bsp/TaskScheduler.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskScheduler.java (working copy) @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; -import java.util.Collection; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; - -/** - * Used by a {@link BSPMaster} to schedule {@link Task}s on {@link GroomServer} - * s. - */ -abstract class TaskScheduler implements Configurable { - - protected Configuration conf; - protected GroomServerManager groomServerManager; - - public Configuration getConf() { - return conf; - } - - public void setConf(Configuration conf) { - this.conf = conf; - } - - public synchronized void setGroomServerManager( - GroomServerManager groomServerManager) { - this.groomServerManager = groomServerManager; - } - - /** - * Lifecycle method to allow the scheduler to start any work in separate - * threads. - * - * @throws IOException - */ - public void start() throws IOException { - // do nothing - } - - /** - * Lifecycle method to allow the scheduler to stop any work it is doing. - * - * @throws IOException - */ - public void terminate() throws IOException { - // do nothing - } - - // public abstract void addJob(JobInProgress job); - - /** - * Returns a collection of jobs in an order which is specific to the - * particular scheduler. - * - * @param Queue name. - * @return JobInProgress corresponded to the specified queue. - */ - public abstract Collection getJobs(String queue); -} Index: src/java/org/apache/hama/bsp/Messagable.java =================================================================== --- src/java/org/apache/hama/bsp/Messagable.java (revision 1152138) +++ src/java/org/apache/hama/bsp/Messagable.java (working copy) @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -/** - * A interface for BSP message class. - */ -public interface Messagable { - -} Index: src/java/org/apache/hama/bsp/TaskLog.java =================================================================== --- src/java/org/apache/hama/bsp/TaskLog.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskLog.java (working copy) @@ -1,343 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.File; -import java.io.FileFilter; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hama.HamaConfiguration; - -/** - * A simple logger to handle the task-specific user logs. - */ -public class TaskLog { - private static final Log LOG = LogFactory.getLog(TaskLog.class.getName()); - - private static final File LOG_DIR = new File( - System.getProperty("hama.log.dir"), "userlogs").getAbsoluteFile(); - - static { - if (!LOG_DIR.exists()) { - LOG_DIR.mkdirs(); - } - } - - public static File getTaskLogFile(TaskAttemptID taskid, LogName filter) { - return new File(new File(LOG_DIR, taskid.toString()), filter.toString()); - } - - /** - * The filter for userlogs. - */ - public static enum LogName { - /** Log on the stdout of the task. */ - STDOUT("stdout"), - - /** Log on the stderr of the task. */ - STDERR("stderr"), - - /** Log on the map-reduce system logs of the task. */ - SYSLOG("syslog"), - - /** The java profiler information. */ - PROFILE("profile.out"), - - /** Log the debug script's stdout */ - DEBUGOUT("debugout"); - - private String prefix; - - private LogName(String prefix) { - this.prefix = prefix; - } - - @Override - public String toString() { - return prefix; - } - } - - private static class TaskLogsPurgeFilter implements FileFilter { - long purgeTimeStamp; - - TaskLogsPurgeFilter(long purgeTimeStamp) { - this.purgeTimeStamp = purgeTimeStamp; - } - - public boolean accept(File file) { - LOG.debug("PurgeFilter - file: " + file + ", mtime: " - + file.lastModified() + ", purge: " + purgeTimeStamp); - return file.lastModified() < purgeTimeStamp; - } - } - - /** - * Purge old user logs. - * - * @throws IOException - */ - public static synchronized void cleanup(int logsRetainHours) - throws IOException { - // Purge logs of tasks on this tasktracker if their - // mtime has exceeded "mapred.task.log.retain" hours - long purgeTimeStamp = System.currentTimeMillis() - - (logsRetainHours * 60L * 60 * 1000); - File[] oldTaskLogs = LOG_DIR.listFiles(new TaskLogsPurgeFilter( - purgeTimeStamp)); - if (oldTaskLogs != null) { - for (int i = 0; i < oldTaskLogs.length; ++i) { - FileUtil.fullyDelete(oldTaskLogs[i]); - } - } - } - - static class Reader extends InputStream { - private long bytesRemaining; - private FileInputStream file; - - /** - * Read a log file from start to end positions. The offsets may be negative, - * in which case they are relative to the end of the file. For example, - * Reader(taskid, kind, 0, -1) is the entire file and Reader(taskid, kind, - * -4197, -1) is the last 4196 bytes. - * - * @param taskid the id of the task to read the log file for - * @param kind the kind of log to read - * @param start the offset to read from (negative is relative to tail) - * @param end the offset to read upto (negative is relative to tail) - * @throws IOException - */ - public Reader(TaskAttemptID taskid, LogName kind, long start, long end) - throws IOException { - // find the right log file - File filename = getTaskLogFile(taskid, kind); - // calculate the start and stop - long size = filename.length(); - if (start < 0) { - start += size + 1; - } - if (end < 0) { - end += size + 1; - } - start = Math.max(0, Math.min(start, size)); - end = Math.max(0, Math.min(end, size)); - bytesRemaining = end - start; - file = new FileInputStream(filename); - // skip upto start - long pos = 0; - while (pos < start) { - long result = file.skip(start - pos); - if (result < 0) { - bytesRemaining = 0; - break; - } - pos += result; - } - } - - @Override - public int read() throws IOException { - int result = -1; - if (bytesRemaining > 0) { - bytesRemaining -= 1; - result = file.read(); - } - return result; - } - - @Override - public int read(byte[] buffer, int offset, int length) throws IOException { - length = (int) Math.min(length, bytesRemaining); - int bytes = file.read(buffer, offset, length); - if (bytes > 0) { - bytesRemaining -= bytes; - } - return bytes; - } - - @Override - public int available() throws IOException { - return (int) Math.min(bytesRemaining, file.available()); - } - - @Override - public void close() throws IOException { - file.close(); - } - } - - private static final String bashCommand = "bash"; - private static final String tailCommand = "tail"; - - /** - * Get the desired maximum length of task's logs. - * - * @param conf the job to look in - * @return the number of bytes to cap the log files at - */ - public static long getTaskLogLength(HamaConfiguration conf) { - return conf.getLong("mapred.userlog.limit.kb", 100) * 1024; - } - - /** - * Wrap a command in a shell to capture stdout and stderr to files. If the - * tailLength is 0, the entire output will be saved. - * - * @param cmd The command and the arguments that should be run - * @param stdoutFilename The filename that stdout should be saved to - * @param stderrFilename The filename that stderr should be saved to - * @param tailLength The length of the tail to be saved. - * @return the modified command that should be run - */ - public static List captureOutAndError(List cmd, - File stdoutFilename, File stderrFilename, long tailLength) - throws IOException { - return captureOutAndError(null, cmd, stdoutFilename, stderrFilename, - tailLength); - } - - /** - * Wrap a command in a shell to capture stdout and stderr to files. Setup - * commands such as setting memory limit can be passed which will be executed - * before exec. If the tailLength is 0, the entire output will be saved. - * - * @param setup The setup commands for the execed process. - * @param cmd The command and the arguments that should be run - * @param stdoutFilename The filename that stdout should be saved to - * @param stderrFilename The filename that stderr should be saved to - * @param tailLength The length of the tail to be saved. - * @return the modified command that should be run - */ - public static List captureOutAndError(List setup, - List cmd, File stdoutFilename, File stderrFilename, - long tailLength) throws IOException { - String stdout = FileUtil.makeShellPath(stdoutFilename); - String stderr = FileUtil.makeShellPath(stderrFilename); - List result = new ArrayList(3); - result.add(bashCommand); - result.add("-c"); - StringBuffer mergedCmd = new StringBuffer(); - if (setup != null && setup.size() > 0) { - mergedCmd.append(addCommand(setup, false)); - mergedCmd.append(";"); - } - if (tailLength > 0) { - mergedCmd.append("("); - } else { - mergedCmd.append("exec "); - } - mergedCmd.append(addCommand(cmd, true)); - mergedCmd.append(" < /dev/null "); - if (tailLength > 0) { - mergedCmd.append(" | "); - mergedCmd.append(tailCommand); - mergedCmd.append(" -c "); - mergedCmd.append(tailLength); - mergedCmd.append(" >> "); - mergedCmd.append(stdout); - mergedCmd.append(" ; exit $PIPESTATUS ) 2>&1 | "); - mergedCmd.append(tailCommand); - mergedCmd.append(" -c "); - mergedCmd.append(tailLength); - mergedCmd.append(" >> "); - mergedCmd.append(stderr); - mergedCmd.append(" ; exit $PIPESTATUS"); - } else { - mergedCmd.append(" 1>> "); - mergedCmd.append(stdout); - mergedCmd.append(" 2>> "); - mergedCmd.append(stderr); - } - result.add(mergedCmd.toString()); - return result; - } - - /** - * Add quotes to each of the command strings and return as a single string - * - * @param cmd The command to be quoted - * @param isExecutable makes shell path if the first argument is executable - * @return returns The quoted string. - * @throws IOException - */ - public static String addCommand(List cmd, boolean isExecutable) - throws IOException { - StringBuffer command = new StringBuffer(); - for (String s : cmd) { - command.append('\''); - if (isExecutable) { - // the executable name needs to be expressed as a shell path for the - // shell to find it. - command.append(FileUtil.makeShellPath(new File(s))); - isExecutable = false; - } else { - command.append(s); - } - command.append('\''); - command.append(" "); - } - return command.toString(); - } - - /** - * Wrap a command in a shell to capture debug script's stdout and stderr to - * debugout. - * - * @param cmd The command and the arguments that should be run - * @param debugoutFilename The filename that stdout and stderr should be saved - * to. - * @return the modified command that should be run - * @throws IOException - */ - public static List captureDebugOut(List cmd, - File debugoutFilename) throws IOException { - String debugout = FileUtil.makeShellPath(debugoutFilename); - List result = new ArrayList(3); - result.add(bashCommand); - result.add("-c"); - StringBuffer mergedCmd = new StringBuffer(); - mergedCmd.append("exec "); - boolean isExecutable = true; - for (String s : cmd) { - if (isExecutable) { - // the executable name needs to be expressed as a shell path for the - // shell to find it. - mergedCmd.append(FileUtil.makeShellPath(new File(s))); - isExecutable = false; - } else { - mergedCmd.append(s); - } - mergedCmd.append(" "); - } - mergedCmd.append(" < /dev/null "); - mergedCmd.append(" >"); - mergedCmd.append(debugout); - mergedCmd.append(" 2>&1 "); - result.add(mergedCmd.toString()); - return result; - } - -} // TaskLog Index: src/java/org/apache/hama/bsp/BSPInterface.java =================================================================== --- src/java/org/apache/hama/bsp/BSPInterface.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPInterface.java (working copy) @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.zookeeper.KeeperException; - -/** - * Interface BSP defines the basic operations needed to implement the BSP - * algorithm. - */ -public interface BSPInterface extends Configurable { - - /** - * A user defined function for programming in the BSP style. - * - * Applications can use the {@link org.apache.hama.bsp.BSPPeer} to handle the - * communication and synchronization between processors. - * - * @param bspPeer - * @throws IOException - * @throws KeeperException - * @throws InterruptedException - */ - public void bsp(BSPPeer bspPeer) throws IOException, KeeperException, - InterruptedException; -} Index: src/java/org/apache/hama/bsp/TaskAttemptContext.java =================================================================== --- src/java/org/apache/hama/bsp/TaskAttemptContext.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskAttemptContext.java (working copy) @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.Progressable; - -/** - * The context for task attempts. - */ -public class TaskAttemptContext extends BSPJobContext implements Progressable { - private final TaskAttemptID taskId; - private String status = ""; - - public TaskAttemptContext(Configuration conf, TaskAttemptID taskId) { - super(conf, taskId.getJobID()); - this.taskId = taskId; - } - - /** - * Get the unique name for this task attempt. - */ - public TaskAttemptID getTaskAttemptID() { - return taskId; - } - - /** - * Set the current status of the task to the given string. - */ - public void setStatus(String msg) throws IOException { - status = msg; - } - - /** - * Get the last set status message. - * - * @return the current status message - */ - public String getStatus() { - return status; - } - - /** - * Report progress. The subtypes actually do work in this method. - */ - public void progress() { - } -} Index: src/java/org/apache/hama/bsp/TaskStatus.java =================================================================== --- src/java/org/apache/hama/bsp/TaskStatus.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskStatus.java (working copy) @@ -1,279 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; - -/** - * Describes the current status of a task. This is not intended to be a - * comprehensive piece of data. - */ -class TaskStatus implements Writable, Cloneable { - static final Log LOG = LogFactory.getLog(TaskStatus.class); - - // enumeration for reporting current phase of a task. - public static enum Phase { - STARTING, COMPUTE, BARRIER_SYNC, CLEANUP - } - - // what state is the task in? - public static enum State { - RUNNING, SUCCEEDED, FAILED, UNASSIGNED, KILLED, COMMIT_PENDING, FAILED_UNCLEAN, KILLED_UNCLEAN - } - - private BSPJobID jobId; - private TaskAttemptID taskId; - private float progress; - private volatile State runState; - private String stateString; - private String groomServer; - private long superstepCount; - - private long startTime; - private long finishTime; - - private volatile Phase phase = Phase.STARTING; - - /** - * - */ - public TaskStatus() { - jobId = new BSPJobID(); - taskId = new TaskAttemptID(); - this.superstepCount = 0; - } - - public TaskStatus(BSPJobID jobId, TaskAttemptID taskId, float progress, - State runState, String stateString, String groomServer, Phase phase) { - this.jobId = jobId; - this.taskId = taskId; - this.progress = progress; - this.runState = runState; - this.stateString = stateString; - this.groomServer = groomServer; - this.phase = phase; - this.superstepCount = 0; - } - - // ////////////////////////////////////////////////// - // Accessors and Modifiers - // ////////////////////////////////////////////////// - - public BSPJobID getJobId() { - return jobId; - } - - public TaskAttemptID getTaskId() { - return taskId; - } - - public float getProgress() { - return progress; - } - - public void setProgress(float progress) { - this.progress = progress; - } - - public State getRunState() { - return runState; - } - - public void setRunState(State state) { - this.runState = state; - } - - public String getStateString() { - return stateString; - } - - public void setStateString(String stateString) { - this.stateString = stateString; - } - - public String getGroomServer() { - return groomServer; - } - - public void setGroomServer(String groomServer) { - this.groomServer = groomServer; - } - - public long getFinishTime() { - return finishTime; - } - - void setFinishTime(long finishTime) { - this.finishTime = finishTime; - } - - /** - * Get start time of the task. - * - * @return 0 is start time is not set, else returns start time. - */ - public long getStartTime() { - return startTime; - } - - /** - * Set startTime of the task. - * - * @param startTime start time - */ - void setStartTime(long startTime) { - this.startTime = startTime; - } - - /** - * Get current phase of this task. - * - * @return . - */ - public Phase getPhase() { - return this.phase; - } - - /** - * Set current phase of this task. - * - * @param phase phase of this task - */ - void setPhase(Phase phase) { - this.phase = phase; - } - - /** - * Update the status of the task. - * - * This update is done by ping thread before sending the status. - * - * @param progress - * @param state - * @param counters - */ - synchronized void statusUpdate(float progress, String state) { - setProgress(progress); - setStateString(state); - } - - /** - * Update the status of the task. - * - * @param status updated status - */ - synchronized void statusUpdate(TaskStatus status) { - this.progress = status.getProgress(); - this.runState = status.getRunState(); - this.stateString = status.getStateString(); - - if (status.getStartTime() != 0) { - this.startTime = status.getStartTime(); - } - if (status.getFinishTime() != 0) { - this.finishTime = status.getFinishTime(); - } - - this.phase = status.getPhase(); - } - - /** - * Update specific fields of task status - * - * This update is done in BSPMaster when a cleanup attempt of task reports its - * status. Then update only specific fields, not all. - * - * @param runState - * @param progress - * @param state - * @param phase - * @param finishTime - */ - synchronized void statusUpdate(State runState, float progress, String state, - Phase phase, long finishTime) { - setRunState(runState); - setProgress(progress); - setStateString(state); - setPhase(phase); - if (finishTime != 0) { - this.finishTime = finishTime; - } - } - - /** - * @return The number of BSP super steps executed by the task. - */ - public long getSuperstepCount() { - return superstepCount; - } - - /** - * Increments the number of BSP super steps executed by the task. - */ - public void incrementSuperstepCount() { - superstepCount += 1; - } - - @Override - public Object clone() { - try { - return super.clone(); - } catch (CloneNotSupportedException cnse) { - // Shouldn't happen since we do implement Clonable - throw new InternalError(cnse.toString()); - } - } - - // //////////////////////////////////////////// - // Writable - // //////////////////////////////////////////// - - @Override - public void readFields(DataInput in) throws IOException { - this.jobId.readFields(in); - this.taskId.readFields(in); - this.progress = in.readFloat(); - this.runState = WritableUtils.readEnum(in, State.class); - this.stateString = Text.readString(in); - this.phase = WritableUtils.readEnum(in, Phase.class); - this.startTime = in.readLong(); - this.finishTime = in.readLong(); - this.superstepCount = in.readLong(); - } - - @Override - public void write(DataOutput out) throws IOException { - jobId.write(out); - taskId.write(out); - out.writeFloat(progress); - WritableUtils.writeEnum(out, runState); - Text.writeString(out, stateString); - WritableUtils.writeEnum(out, phase); - out.writeLong(startTime); - out.writeLong(finishTime); - out.writeLong(superstepCount); - } -} Index: src/java/org/apache/hama/bsp/BSPTask.java =================================================================== --- src/java/org/apache/hama/bsp/BSPTask.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPTask.java (working copy) @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hama.ipc.BSPPeerProtocol; -import org.apache.zookeeper.KeeperException; - -/** - * Base class for tasks. - */ -public class BSPTask extends Task { - - public static final Log LOG = LogFactory.getLog(BSPTask.class); - - private BSPJob conf; - - public BSPTask() { - } - - public BSPTask(BSPJobID jobId, String jobFile, TaskAttemptID taskid, int partition) { - this.jobId = jobId; - this.jobFile = jobFile; - this.taskId = taskid; - this.partition = partition; - } - - @Override - public BSPTaskRunner createRunner(GroomServer groom) { - return new BSPTaskRunner(this, groom, this.conf); - } - - @Override - public void run(BSPJob job, BSPPeer bspPeer, BSPPeerProtocol umbilical) - throws IOException { - - BSP bsp = (BSP) ReflectionUtils.newInstance(job.getConf().getClass( - "bsp.work.class", BSP.class), job.getConf()); - - try { - bsp.bsp(bspPeer); - } catch (IOException e) { - LOG.error("Exception during BSP execution!", e); - } catch (KeeperException e) { - LOG.error("Exception during BSP execution!", e); - } catch (InterruptedException e) { - LOG.error("Exception during BSP execution!", e); - } - - done(umbilical); - } - - public BSPJob getConf() { - return conf; - } - - public void setConf(BSPJob conf) { - this.conf = conf; - } - -} Index: src/java/org/apache/hama/bsp/GroomServer.java =================================================================== --- src/java/org/apache/hama/bsp/GroomServer.java (revision 1152138) +++ src/java/org/apache/hama/bsp/GroomServer.java (working copy) @@ -1,920 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.lang.reflect.Constructor; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.LinkedBlockingQueue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.filecache.DistributedCache; -import org.apache.hadoop.fs.FSError; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.net.DNS; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.util.DiskChecker; -import org.apache.hadoop.util.DiskChecker.DiskErrorException; -import org.apache.hadoop.util.RunJar; -import org.apache.hadoop.util.StringUtils; -import org.apache.hama.Constants; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.ipc.BSPPeerProtocol; -import org.apache.hama.ipc.MasterProtocol; -import org.apache.hama.ipc.GroomProtocol; -import org.apache.log4j.LogManager; - -/** - * A Groom Server (shortly referred to as groom) is a process that performs bsp - * tasks assigned by BSPMaster. Each groom contacts the BSPMaster, and it takes - * assigned tasks and reports its status by means of periodical piggybacks with - * BSPMaster. Each groom is designed to run with HDFS or other distributed - * storages. Basically, a groom server and a data node should be run on one - * physical node. - */ -public class GroomServer implements Runnable, GroomProtocol, BSPPeerProtocol { - public static final Log LOG = LogFactory.getLog(GroomServer.class); - static final String SUBDIR = "groomServer"; - - private volatile static int REPORT_INTERVAL = 1 * 1000; - - Configuration conf; - - // Constants - static enum State { - NORMAL, COMPUTE, SYNC, BARRIER, STALE, INTERRUPTED, DENIED - }; - - // Running States and its related things - volatile boolean initialized = false; - volatile boolean running = true; - volatile boolean shuttingDown = false; - boolean justInited = true; - GroomServerStatus status = null; - - // Attributes - String groomServerName; - String localHostname; - String groomHostName; - InetSocketAddress bspMasterAddr; - private Instructor instructor; - - // Filesystem - // private LocalDirAllocator localDirAllocator; - Path systemDirectory = null; - FileSystem systemFS = null; - - // Job - private int failures; - private int maxCurrentTasks = 1; - Map tasks = new HashMap(); - /** Map from taskId -> TaskInProgress. */ - Map runningTasks = null; - Map finishedTasks = null; - Map runningJobs = null; - - // new nexus between GroomServer and BSPMaster - // holds/ manage all tasks - // List tasksList = new - // CopyOnWriteArrayList(); - - private String rpcServer; - private Server workerServer; - MasterProtocol masterClient; - - InetSocketAddress taskReportAddress; - Server taskReportServer = null; - - private PeerNames allPeerNames = null; - - // private BlockingQueue tasksToCleanup = new - // LinkedBlockingQueue(); - - private class DispatchTasksHandler implements DirectiveHandler { - - public void handle(Directive directive) throws DirectiveException { - GroomServerAction[] actions = ((DispatchTasksDirective) directive) - .getActions(); - - allPeerNames = new PeerNames(((DispatchTasksDirective) directive) - .getGroomServerPeers().values()); - - if (LOG.isDebugEnabled()) { - LOG.debug("Got Response from BSPMaster with " - + ((actions != null) ? actions.length : 0) + " actions"); - } - - if (actions != null) { - for (GroomServerAction action : actions) { - if (action instanceof LaunchTaskAction) { - startNewTask((LaunchTaskAction) action); - } else { - - // TODO Use the cleanup thread - // tasksToCleanup.put(action); - - KillTaskAction killAction = (KillTaskAction) action; - if (tasks.containsKey(killAction.getTaskID())) { - TaskInProgress tip = tasks.get(killAction.getTaskID()); - tip.taskStatus.setRunState(TaskStatus.State.FAILED); - try { - tip.killAndCleanup(true); - } catch (IOException ioe) { - throw new DirectiveException("Error when killing a " - + "TaskInProgress.", ioe); - } - } - } - } - } - } - } - - private class Instructor extends Thread { - final BlockingQueue buffer = new LinkedBlockingQueue(); - final ConcurrentMap, DirectiveHandler> handlers = new ConcurrentHashMap, DirectiveHandler>(); - - public void bind(Class instruction, - DirectiveHandler handler) { - handlers.putIfAbsent(instruction, handler); - } - - public void put(Directive directive) { - try { - buffer.put(directive); - } catch (InterruptedException ie) { - LOG.error("Unable to put directive into queue.", ie); - Thread.currentThread().interrupt(); - } - } - - public void run() { - while (true) { - try { - Directive directive = buffer.take(); - if (directive instanceof DispatchTasksDirective) { - ((DirectiveHandler) handlers.get(DispatchTasksDirective.class)) - .handle(directive); - } else { - throw new RuntimeException("Directive is not supported." - + directive); - } - } catch (InterruptedException ie) { - LOG.error("Unable to retrieve directive from the queue.", ie); - Thread.currentThread().interrupt(); - } catch (Exception e) { - LOG.error("Fail to execute directive.", e); - } - } - } - } - - public GroomServer(Configuration conf) throws IOException { - LOG.info("groom start"); - this.conf = conf; - - bspMasterAddr = BSPMaster.getAddress(conf); - // FileSystem local = FileSystem.getLocal(conf); - // this.localDirAllocator = new LocalDirAllocator("bsp.local.dir"); - } - - public synchronized void initialize() throws IOException { - if (this.conf.get(Constants.PEER_HOST) != null) { - this.localHostname = conf.get(Constants.PEER_HOST); - } - - if (localHostname == null) { - this.localHostname = DNS.getDefaultHost( - conf.get("bsp.dns.interface", "default"), - conf.get("bsp.dns.nameserver", "default")); - } - // check local disk - checkLocalDirs(conf.getStrings("bsp.local.dir")); - deleteLocalFiles("groomserver"); - - // Clear out state tables - this.tasks.clear(); - this.runningJobs = new TreeMap(); - this.runningTasks = new LinkedHashMap(); - this.finishedTasks = new LinkedHashMap(); - this.conf.set(Constants.PEER_HOST, localHostname); - this.conf.set(Constants.GROOM_RPC_HOST, localHostname); - - int rpcPort = -1; - String rpcAddr = null; - if (false == this.initialized) { - rpcAddr = conf.get(Constants.GROOM_RPC_HOST, - Constants.DEFAULT_GROOM_RPC_HOST); - rpcPort = conf.getInt(Constants.GROOM_RPC_PORT, - Constants.DEFAULT_GROOM_RPC_PORT); - if (-1 == rpcPort || null == rpcAddr) - throw new IllegalArgumentException("Error rpc address " + rpcAddr - + " port" + rpcPort); - this.workerServer = RPC.getServer(this, rpcAddr, rpcPort, conf); - this.workerServer.start(); - this.rpcServer = rpcAddr + ":" + rpcPort; - - LOG.info("Worker rpc server --> " + rpcServer); - } - - @SuppressWarnings("deprecation") - String address = NetUtils.getServerAddress(conf, - "bsp.groom.report.bindAddress", "bsp.groom.report.port", - "bsp.groom.report.address"); - InetSocketAddress socAddr = NetUtils.createSocketAddr(address); - String bindAddress = socAddr.getHostName(); - int tmpPort = socAddr.getPort(); - - // RPC initialization - // TODO numHandlers should be a .. - this.taskReportServer = RPC.getServer(this, bindAddress, tmpPort, 10, - false, this.conf); - - this.taskReportServer.start(); - - // get the assigned address - this.taskReportAddress = taskReportServer.getListenerAddress(); - this.conf.set("bsp.groom.report.address", taskReportAddress.getHostName() - + ":" + taskReportAddress.getPort()); - LOG.info("GroomServer up at: " + this.taskReportAddress); - - this.groomHostName = rpcAddr; - this.groomServerName = "groomd_" + this.rpcServer.replace(':', '_'); - LOG.info("Starting groom: " + this.rpcServer); - - DistributedCache.purgeCache(this.conf); - - // establish the communication link to bsp master - this.masterClient = (MasterProtocol) RPC.waitForProxy(MasterProtocol.class, - MasterProtocol.versionID, bspMasterAddr, conf); - - // enroll in bsp master - if (-1 == rpcPort || null == rpcAddr) - throw new IllegalArgumentException("Error rpc address " + rpcAddr - + " port" + rpcPort); - if (!this.masterClient.register(new GroomServerStatus(groomServerName, - getBspPeerName(), cloneAndResetRunningTaskStatuses(), failures, - maxCurrentTasks, this.rpcServer))) { - LOG.error("There is a problem in establishing communication" - + " link with BSPMaster"); - throw new IOException("There is a problem in establishing" - + " communication link with BSPMaster."); - } - - this.instructor = new Instructor(); - this.instructor.bind(DispatchTasksDirective.class, - new DispatchTasksHandler()); - instructor.start(); - this.running = true; - this.initialized = true; - } - - /** Return the port at which the tasktracker bound to */ - public synchronized InetSocketAddress getTaskTrackerReportAddress() { - return taskReportAddress; - } - - @Override - public void dispatch(Directive directive) throws IOException { - if (!instructor.isAlive()) - throw new IOException(); - - instructor.put(directive); - } - - private static void checkLocalDirs(String[] localDirs) - throws DiskErrorException { - boolean writable = false; - - LOG.info(localDirs); - - if (localDirs != null) { - for (int i = 0; i < localDirs.length; i++) { - try { - LOG.info(localDirs[i]); - DiskChecker.checkDir(new File(localDirs[i])); - writable = true; - } catch (DiskErrorException e) { - LOG.warn("BSP Processor local " + e.getMessage()); - } - } - } - - if (!writable) - throw new DiskErrorException("all local directories are not writable"); - } - - public String[] getLocalDirs() { - return conf.getStrings("bsp.local.dir"); - } - - public void deleteLocalFiles() throws IOException { - String[] localDirs = getLocalDirs(); - for (int i = 0; i < localDirs.length; i++) { - FileSystem.getLocal(this.conf).delete(new Path(localDirs[i]), true); - } - } - - public void deleteLocalFiles(String subdir) throws IOException { - try { - String[] localDirs = getLocalDirs(); - for (int i = 0; i < localDirs.length; i++) { - FileSystem.getLocal(this.conf).delete(new Path(localDirs[i], subdir), - true); - } - } catch (NullPointerException e) { - LOG.info(e); - } - } - - public void cleanupStorage() throws IOException { - deleteLocalFiles(); - } - - private void startCleanupThreads() throws IOException { - - } - - public State offerService() throws Exception { - while (running && !shuttingDown) { - try { - - // Reports to a BSPMaster - for (Map.Entry e : runningTasks - .entrySet()) { - Thread.sleep(REPORT_INTERVAL); - TaskInProgress tip = e.getValue(); - TaskStatus taskStatus = tip.getStatus(); - - if (taskStatus.getRunState() == TaskStatus.State.RUNNING) { - taskStatus.setProgress(taskStatus.getSuperstepCount()); - - if (!tip.runner.isAlive()) { - if (taskStatus.getRunState() != TaskStatus.State.FAILED) { - taskStatus.setRunState(TaskStatus.State.SUCCEEDED); - } - taskStatus.setPhase(TaskStatus.Phase.CLEANUP); - } - } - - doReport(taskStatus); - } - - Thread.sleep(REPORT_INTERVAL); - } catch (InterruptedException ie) { - } - - try { - if (justInited) { - String dir = masterClient.getSystemDir(); - if (dir == null) { - LOG.error("Fail to get system directory."); - throw new IOException("Fail to get system directory."); - } - systemDirectory = new Path(dir); - systemFS = systemDirectory.getFileSystem(conf); - } - justInited = false; - } catch (DiskErrorException de) { - String msg = "Exiting groom server for disk error:\n" - + StringUtils.stringifyException(de); - LOG.error(msg); - - return State.STALE; - } catch (RemoteException re) { - return State.DENIED; - } catch (Exception except) { - String msg = "Caught exception: " - + StringUtils.stringifyException(except); - LOG.error(msg); - } - } - return State.NORMAL; - } - - private void startNewTask(LaunchTaskAction action) { - Task t = action.getTask(); - BSPJob jobConf = null; - try { - jobConf = new BSPJob(t.getJobID(), t.getJobFile()); - } catch (IOException e1) { - LOG.error(e1); - } - - TaskInProgress tip = new TaskInProgress(t, jobConf, this.groomServerName); - - synchronized (this) { - tasks.put(t.getTaskID(), tip); - runningTasks.put(t.getTaskID(), tip); - } - - try { - localizeJob(tip); - } catch (Throwable e) { - String msg = ("Error initializing " + tip.getTask().getTaskID() + ":\n" + StringUtils - .stringifyException(e)); - LOG.warn(msg); - } - } - - /** - * Update and report refresh status back to BSPMaster. - */ - public void doReport(TaskStatus taskStatus) { - GroomServerStatus groomStatus = new GroomServerStatus(groomServerName, - getBspPeerName(), updateTaskStatus(taskStatus), failures, - maxCurrentTasks, rpcServer); - try { - boolean ret = masterClient.report(new ReportGroomStatusDirective( - groomStatus)); - if (!ret) { - LOG.warn("Fail to renew BSPMaster's GroomServerStatus. " - + " groom name: " + groomStatus.getGroomName() + " peer name:" - + groomStatus.getPeerName() + " rpc server:" + rpcServer); - } - } catch (IOException ioe) { - LOG.error("Fail to communicate with BSPMaster for reporting.", ioe); - } - } - - public List updateTaskStatus(TaskStatus taskStatus) { - List tlist = new ArrayList(); - synchronized (runningTasks) { - - if (taskStatus.getRunState() == TaskStatus.State.SUCCEEDED - || taskStatus.getRunState() == TaskStatus.State.FAILED) { - synchronized (finishedTasks) { - TaskInProgress tip = runningTasks.remove(taskStatus.getTaskId()); - tlist.add((TaskStatus) taskStatus.clone()); - finishedTasks.put(taskStatus.getTaskId(), tip); - } - } else if (taskStatus.getRunState() == TaskStatus.State.RUNNING) { - tlist.add((TaskStatus) taskStatus.clone()); - } - - } - return tlist; - } - - private void localizeJob(TaskInProgress tip) throws IOException { - Task task = tip.getTask(); - conf.addResource(task.getJobFile()); - BSPJob defaultJobConf = new BSPJob((HamaConfiguration) conf); - Path localJobFile = defaultJobConf.getLocalPath(SUBDIR + "/" - + task.getTaskID() + "/" + "job.xml"); - - RunningJob rjob = addTaskToJob(task.getJobID(), localJobFile, tip); - BSPJob jobConf = null; - - synchronized (rjob) { - if (!rjob.localized) { - Path localJarFile = defaultJobConf.getLocalPath(SUBDIR + "/" - + task.getTaskID() + "/" + "job.jar"); - systemFS.copyToLocalFile(new Path(task.getJobFile()), localJobFile); - - HamaConfiguration conf = new HamaConfiguration(); - conf.addResource(localJobFile); - jobConf = new BSPJob(conf, task.getJobID().toString()); - - Path jarFile = new Path(jobConf.getJar()); - jobConf.setJar(localJarFile.toString()); - - if (jarFile != null) { - systemFS.copyToLocalFile(jarFile, localJarFile); - - // also unjar the job.jar files in workdir - File workDir = new File( - new File(localJobFile.toString()).getParent(), "work"); - if (!workDir.mkdirs()) { - if (!workDir.isDirectory()) { - throw new IOException("Mkdirs failed to create " - + workDir.toString()); - } - } - RunJar.unJar(new File(localJarFile.toString()), workDir); - } - rjob.localized = true; - } - } - - launchTaskForJob(tip, jobConf); - } - - private void launchTaskForJob(TaskInProgress tip, BSPJob jobConf) { - try { - tip.setJobConf(jobConf); - tip.launchTask(); - } catch (Throwable ie) { - tip.taskStatus.setRunState(TaskStatus.State.FAILED); - String error = StringUtils.stringifyException(ie); - LOG.info(error); - } - } - - private RunningJob addTaskToJob(BSPJobID jobId, Path localJobFile, - TaskInProgress tip) { - synchronized (runningJobs) { - RunningJob rJob = null; - if (!runningJobs.containsKey(jobId)) { - rJob = new RunningJob(jobId, localJobFile); - rJob.localized = false; - rJob.tasks = new HashSet(); - rJob.jobFile = localJobFile; - runningJobs.put(jobId, rJob); - } else { - rJob = runningJobs.get(jobId); - } - rJob.tasks.add(tip); - return rJob; - } - } - - /** - * The datastructure for initializing a job - */ - static class RunningJob { - private BSPJobID jobid; - private Path jobFile; - // keep this for later use - Set tasks; - boolean localized; - boolean keepJobFiles; - - RunningJob(BSPJobID jobid, Path jobFile) { - this.jobid = jobid; - localized = false; - tasks = new HashSet(); - this.jobFile = jobFile; - keepJobFiles = false; - } - - Path getJobFile() { - return jobFile; - } - - BSPJobID getJobId() { - return jobid; - } - } - - private synchronized List cloneAndResetRunningTaskStatuses() { - List result = new ArrayList(runningTasks.size()); - for (TaskInProgress tip : runningTasks.values()) { - TaskStatus status = tip.getStatus(); - result.add((TaskStatus) status.clone()); - } - return result; - } - - public void run() { - try { - initialize(); - startCleanupThreads(); - boolean denied = false; - while (running && !shuttingDown && !denied) { - - boolean staleState = false; - try { - while (running && !staleState && !shuttingDown && !denied) { - try { - State osState = offerService(); - if (osState == State.STALE) { - staleState = true; - } else if (osState == State.DENIED) { - denied = true; - } - } catch (Exception e) { - if (!shuttingDown) { - LOG.info("Lost connection to BSP Master [" + bspMasterAddr - + "]. Retrying...", e); - try { - Thread.sleep(5000); - } catch (InterruptedException ie) { - } - } - } - } - } finally { - // close(); - } - if (shuttingDown) { - return; - } - LOG.warn("Reinitializing local state"); - initialize(); - } - } catch (IOException ioe) { - LOG.error("Got fatal exception while reinitializing GroomServer: " - + StringUtils.stringifyException(ioe)); - return; - } - } - - public synchronized void shutdown() throws IOException { - shuttingDown = true; - close(); - } - - public synchronized void close() throws IOException { - this.running = false; - this.initialized = false; - cleanupStorage(); - this.workerServer.stop(); - RPC.stopProxy(masterClient); - - if (taskReportServer != null) { - taskReportServer.stop(); - taskReportServer = null; - } - } - - public static Thread startGroomServer(final GroomServer hrs) { - return startGroomServer(hrs, "regionserver" + hrs.groomServerName); - } - - public static Thread startGroomServer(final GroomServer hrs, final String name) { - Thread t = new Thread(hrs); - t.setName(name); - t.start(); - return t; - } - - // ///////////////////////////////////////////////////// - // TaskInProgress maintains all the info for a Task that - // lives at this GroomServer. It maintains the Task object, - // its TaskStatus, and the BSPTaskRunner. - // ///////////////////////////////////////////////////// - class TaskInProgress { - Task task; - BSPJob jobConf; - BSPJob localJobConf; - BSPTaskRunner runner; - volatile boolean done = false; - volatile boolean wasKilled = false; - private TaskStatus taskStatus; - - public TaskInProgress(Task task, BSPJob jobConf, String groomServer) { - this.task = task; - this.jobConf = jobConf; - this.localJobConf = null; - this.taskStatus = new TaskStatus(task.getJobID(), task.getTaskID(), 0, - TaskStatus.State.UNASSIGNED, "running", groomServer, - TaskStatus.Phase.STARTING); - } - - private void localizeTask(Task task) throws IOException { - Path localJobFile = this.jobConf.getLocalPath(SUBDIR + "/" - + task.getTaskID() + "/job.xml"); - Path localJarFile = this.jobConf.getLocalPath(SUBDIR + "/" - + task.getTaskID() + "/job.jar"); - - String jobFile = task.getJobFile(); - systemFS.copyToLocalFile(new Path(jobFile), localJobFile); - task.setJobFile(localJobFile.toString()); - - localJobConf = new BSPJob(task.getJobID(), localJobFile.toString()); - localJobConf.set("bsp.task.id", task.getTaskID().toString()); - String jarFile = localJobConf.getJar(); - - if (jarFile != null) { - systemFS.copyToLocalFile(new Path(jarFile), localJarFile); - localJobConf.setJar(localJarFile.toString()); - } - - LOG.debug("localizeTask : " + localJobConf.getJar()); - LOG.debug("localizeTask : " + localJobFile.toString()); - - task.setConf(localJobConf); - } - - public synchronized void setJobConf(BSPJob jobConf) { - this.jobConf = jobConf; - } - - public synchronized BSPJob getJobConf() { - return localJobConf; - } - - public void launchTask() throws IOException { - localizeTask(task); - taskStatus.setRunState(TaskStatus.State.RUNNING); - this.runner = task.createRunner(GroomServer.this); - this.runner.start(); - } - - /** - * This task has run on too long, and should be killed. - */ - public synchronized void killAndCleanup(boolean wasFailure) - throws IOException { - runner.kill(); - } - - /** - */ - public Task getTask() { - return task; - } - - /** - */ - public synchronized TaskStatus getStatus() { - return taskStatus; - } - - /** - */ - public TaskStatus.State getRunState() { - return taskStatus.getRunState(); - } - - public boolean wasKilled() { - return wasKilled; - } - - @Override - public boolean equals(Object obj) { - return (obj instanceof TaskInProgress) - && task.getTaskID().equals( - ((TaskInProgress) obj).getTask().getTaskID()); - } - - @Override - public int hashCode() { - return task.getTaskID().hashCode(); - } - } - - public boolean isRunning() { - return running; - } - - public static GroomServer constructGroomServer( - Class groomServerClass, final Configuration conf2) { - try { - Constructor c = groomServerClass - .getConstructor(Configuration.class); - return c.newInstance(conf2); - } catch (Exception e) { - throw new RuntimeException("Failed construction of " + "Master: " - + groomServerClass.toString(), e); - } - } - - @Override - public long getProtocolVersion(String protocol, long clientVersion) - throws IOException { - if (protocol.equals(GroomProtocol.class.getName())) { - return GroomProtocol.versionID; - } else if (protocol.equals(BSPPeerProtocol.class.getName())) { - return BSPPeerProtocol.versionID; - } else { - throw new IOException("Unknown protocol to GroomServer: " + protocol); - } - } - - /** - * @return bsp peer information in the form of "address:port". - */ - public String getBspPeerName() { - // TODO Later, peers list should be returned. - return this.groomHostName + ":" + Constants.DEFAULT_PEER_PORT; - } - - /** - * The main() for child processes. - */ - public static class Child { - - public static void main(String[] args) throws Throwable { - LOG.debug("Child starting"); - - HamaConfiguration defaultConf = new HamaConfiguration(); - // report address - String host = args[0]; - int port = Integer.parseInt(args[1]); - InetSocketAddress address = new InetSocketAddress(host, port); - TaskAttemptID taskid = TaskAttemptID.forName(args[2]); - - // ////////////////// - BSPPeerProtocol umbilical = (BSPPeerProtocol) RPC.getProxy( - BSPPeerProtocol.class, BSPPeerProtocol.versionID, address, - defaultConf); - - Task task = umbilical.getTask(taskid); - - defaultConf.addResource(new Path(task.getJobFile())); - BSPJob job = new BSPJob(task.getJobID(), task.getJobFile()); - - defaultConf.set(Constants.PEER_HOST, args[3]); - defaultConf.setInt(Constants.PEER_PORT, Constants.DEFAULT_PEER_PORT); - - BSPPeer bspPeer = new BSPPeer(defaultConf, taskid, umbilical); - bspPeer.setJobConf(job); - bspPeer.setAllPeerNames(umbilical.getAllPeerNames().getAllPeerNames()); - - TaskStatus taskStatus = new TaskStatus(task.getJobID(), task.getTaskID(), - 0, TaskStatus.State.RUNNING, "running", host, - TaskStatus.Phase.STARTING); - - bspPeer.setCurrentTaskStatus(taskStatus); - - try { - // use job-specified working directory - FileSystem.get(job.getConf()).setWorkingDirectory( - job.getWorkingDirectory()); - - task.run(job, bspPeer, umbilical); // run the task - - } catch (FSError e) { - LOG.fatal("FSError from child", e); - umbilical.fsError(taskid, e.getMessage()); - } catch (Throwable throwable) { - LOG.warn("Error running child", throwable); - // Report back any failures, for diagnostic purposes - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - throwable.printStackTrace(new PrintStream(baos)); - } finally { - bspPeer.close(); // close peer. - - RPC.stopProxy(umbilical); - // Shutting down log4j of the child-vm... - // This assumes that on return from Task.run() - // there is no more logging done. - LogManager.shutdown(); - } - } - } - - @Override - public Task getTask(TaskAttemptID taskid) throws IOException { - TaskInProgress tip = tasks.get(taskid); - if (tip != null) { - return tip.getTask(); - } else { - return null; - } - } - - public void incrementSuperstepCount(TaskAttemptID taskid) throws IOException { - TaskInProgress tip = tasks.get(taskid); - tip.getStatus().incrementSuperstepCount(); - } - - @Override - public boolean ping(TaskAttemptID taskid) throws IOException { - // TODO Auto-generated method stub - return false; - } - - @Override - public void done(TaskAttemptID taskid, boolean shouldBePromoted) - throws IOException { - // TODO Auto-generated method stub - } - - @Override - public void fsError(TaskAttemptID taskId, String message) throws IOException { - // TODO Auto-generated method stub - } - - @Override - public PeerNames getAllPeerNames() { - return allPeerNames; - } -} \ No newline at end of file Index: src/java/org/apache/hama/bsp/Directive.java =================================================================== --- src/java/org/apache/hama/bsp/Directive.java (revision 1152138) +++ src/java/org/apache/hama/bsp/Directive.java (working copy) @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.Writable; - -/** - * A generic directive from the {@link org.apache.hama.bsp.BSPMaster} to the - * {@link org.apache.hama.bsp.GroomServer} to take some 'action'. - */ -public class Directive implements Writable{ - - protected long timestamp; - protected Directive.Type type; - - public static enum Type { - Request(1), Response(2); - int t; - - Type(int t) { - this.t = t; - } - - public int value() { - return this.t; - } - }; - - public Directive(){} - - public Directive(Directive.Type type) { - this.timestamp = System.currentTimeMillis(); - this.type = type; - } - - public long getTimestamp() { - return this.timestamp; - } - - public Directive.Type getType() { - return this.type; - } - - /** - * Command for BSPMaster or GroomServer to execute. - public abstract void execute() throws Exception; - */ - - public void write(DataOutput out) throws IOException { - out.writeLong(this.timestamp); - out.writeInt(this.type.value()); - } - - public void readFields(DataInput in) throws IOException { - this.timestamp = in.readLong(); - int t = in.readInt(); - if (Directive.Type.Request.value() == t) { - this.type = Directive.Type.Request; - }else{ - this.type = Directive.Type.Response; - } - } -} Index: src/java/org/apache/hama/bsp/JobStatus.java =================================================================== --- src/java/org/apache/hama/bsp/JobStatus.java (revision 1152138) +++ src/java/org/apache/hama/bsp/JobStatus.java (working copy) @@ -1,279 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableFactories; -import org.apache.hadoop.io.WritableFactory; - -/** - * Describes the current status of a job. - */ -public class JobStatus implements Writable, Cloneable { - public static final Log LOG = LogFactory.getLog(JobStatus.class); - - static { - WritableFactories.setFactory(JobStatus.class, new WritableFactory() { - public Writable newInstance() { - return new JobStatus(); - } - }); - } - - public static enum State { - RUNNING(1), SUCCEEDED(2), FAILED(3), PREP(4), KILLED(5); - int s; - - State(int s) { - this.s = s; - } - - public int value() { - return this.s; - } - - @Override - public String toString() { - String name = null; - switch (this) { - case RUNNING: - name = "RUNNING"; - break; - case SUCCEEDED: - name = "SUCCEEDED"; - break; - case FAILED: - name = "FAILED"; - break; - case PREP: - name = "SETUP"; - break; - case KILLED: - name = "KILLED"; - break; - } - - return name; - } - - } - - public static final int RUNNING = 1; - public static final int SUCCEEDED = 2; - public static final int FAILED = 3; - public static final int PREP = 4; - public static final int KILLED = 5; - - private BSPJobID jobid; - private long progress; - private long cleanupProgress; - private long setupProgress; - private volatile State state;// runState in enum - private int runState; - private long startTime; - private String schedulingInfo = "NA"; - private String user; - private long superstepCount; - private String name; - - private long finishTime; - - public JobStatus() { - } - - public JobStatus(BSPJobID jobid, String user, long progress, int runState) { - this(jobid, user, progress, 0, runState); - } - - public JobStatus(BSPJobID jobid, String user, long progress, - long cleanupProgress, int runState) { - this(jobid, user, 0, progress, cleanupProgress, runState); - } - - public JobStatus(BSPJobID jobid, String user, long setupProgress, - long progress, long cleanupProgress, int runState) { - this(jobid, user, 0, progress, cleanupProgress, runState, 0); - } - - public JobStatus(BSPJobID jobid, String user, long setupProgress, - long progress, long cleanupProgress, int runState, long superstepCount) { - this.jobid = jobid; - this.setupProgress = setupProgress; - this.progress = progress; - this.cleanupProgress = cleanupProgress; - this.runState = runState; - this.state = State.values()[runState - 1]; - this.superstepCount = superstepCount; - this.user = user; - } - - public BSPJobID getJobID() { - return jobid; - } - - public synchronized long progress() { - return progress; - } - - synchronized void setprogress(long p) { - this.progress = p; - } - - public synchronized long cleanupProgress() { - return cleanupProgress; - } - - synchronized void setCleanupProgress(int p) { - this.cleanupProgress = p; - } - - public synchronized long setupProgress() { - return setupProgress; - } - - synchronized void setSetupProgress(long p) { - this.setupProgress = p; - } - - public JobStatus.State getState() { - return this.state; - } - - public void setState(JobStatus.State state) { - this.state = state; - } - - public synchronized int getRunState() { - return runState; - } - - public synchronized void setRunState(int state) { - this.runState = state; - } - - public synchronized long getSuperstepCount() { - return superstepCount; - } - - public synchronized void setSuperstepCount(long superstepCount) { - this.superstepCount = superstepCount; - } - - public synchronized void setStartTime(long startTime) { - this.startTime = startTime; - } - - public synchronized long getStartTime() { - return startTime; - } - - public synchronized void setFinishTime(long finishTime) { - this.finishTime = finishTime; - } - - /** - * Get the finish time of the job. - */ - public synchronized long getFinishTime() { - return finishTime; - } - - /** - * @param user The username of the job - */ - public synchronized void setUsername(String user) { - this.user = user; - } - - /** - * @return the username of the job - */ - public synchronized String getUsername() { - return user; - } - - @Override - public Object clone() { - try { - return super.clone(); - } catch (CloneNotSupportedException cnse) { - throw new InternalError(cnse.toString()); - } - } - - public synchronized String getSchedulingInfo() { - return schedulingInfo; - } - - public synchronized void setSchedulingInfo(String schedulingInfo) { - this.schedulingInfo = schedulingInfo; - } - - public synchronized boolean isJobComplete() { - return (runState == JobStatus.SUCCEEDED || runState == JobStatus.FAILED || runState == JobStatus.KILLED); - } - - public synchronized void write(DataOutput out) throws IOException { - jobid.write(out); - out.writeLong(setupProgress); - out.writeLong(progress); - out.writeLong(cleanupProgress); - out.writeInt(runState); - out.writeLong(startTime); - out.writeLong(finishTime); - Text.writeString(out, user); - Text.writeString(out, schedulingInfo); - out.writeLong(superstepCount); - } - - public synchronized void readFields(DataInput in) throws IOException { - this.jobid = new BSPJobID(); - jobid.readFields(in); - this.setupProgress = in.readLong(); - this.progress = in.readLong(); - this.cleanupProgress = in.readLong(); - this.runState = in.readInt(); - this.startTime = in.readLong(); - this.finishTime = in.readLong(); - this.user = Text.readString(in); - this.schedulingInfo = Text.readString(in); - this.superstepCount = in.readLong(); - } - - /** - * @param name the name to set - */ - public synchronized void setName(String name) { - this.name = name; - } - - /** - * @return the name - */ - public synchronized String getName() { - return name; - } - -} Index: src/java/org/apache/hama/bsp/TaskInProgress.java =================================================================== --- src/java/org/apache/hama/bsp/TaskInProgress.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskInProgress.java (working copy) @@ -1,288 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.JobStatus; - -/** - *TaskInProgress maintains all the info needed for a Task in the lifetime of - * its owning Job. - */ -class TaskInProgress { - public static final Log LOG = LogFactory.getLog(TaskInProgress.class); - - private Configuration conf; - - // Constants - static final int MAX_TASK_EXECS = 1; - int maxTaskAttempts = 4; - private boolean failed = false; - private static final int NUM_ATTEMPTS_PER_RESTART = 1000; - - // Job Meta - private String jobFile = null; - private int partition; - private BSPMaster bspMaster; - private TaskID id; - private JobInProgress job; - private int completes = 0; - - // Status - // private double progress = 0; - // private String state = ""; - private long startTime = 0; - - // The 'next' usable taskid of this tip - int nextTaskId = 0; - - // The taskid that took this TIP to SUCCESS - private TaskAttemptID successfulTaskId; - - // The first taskid of this tip - private TaskAttemptID firstTaskId; - - // Map from task Id -> GroomServer Id, contains tasks that are - // currently runnings - private TreeMap activeTasks = new TreeMap(); - // All attempt Ids of this TIP - // private TreeSet tasks = new TreeSet(); - /** - * Map from taskId -> TaskStatus - */ - private TreeMap taskStatuses = new TreeMap(); - - private BSPJobID jobId; - - /** - * Constructor for new nexus between BSPMaster and GroomServer. - * - * @param jobId is identification of JobInProgress. - * @param jobFile the path of job file - * @param partition which partition this TaskInProgress owns. - */ - public TaskInProgress(BSPJobID jobId, String jobFile, int partition) { - this.jobId = jobId; - this.jobFile = jobFile; - this.partition = partition; - - this.id = new TaskID(jobId, partition); - } - - public TaskInProgress(BSPJobID jobId, String jobFile, BSPMaster master, - Configuration conf, JobInProgress job, int partition) { - this.jobId = jobId; - this.jobFile = jobFile; - this.setBspMaster(master); - this.job = job; - this.setConf(conf); - this.partition = partition; - - this.id = new TaskID(jobId, partition); - } - - /** - * Return a Task that can be sent to a GroomServer for execution. - */ - public Task getTaskToRun(GroomServerStatus status) throws IOException { - Task t = null; - - TaskAttemptID taskid = null; - if (nextTaskId < (MAX_TASK_EXECS + maxTaskAttempts)) { - int attemptId = job.getNumRestarts() * NUM_ATTEMPTS_PER_RESTART - + nextTaskId; - taskid = new TaskAttemptID(id, attemptId); - ++nextTaskId; - } else { - LOG.warn("Exceeded limit of " + (MAX_TASK_EXECS + maxTaskAttempts) - + " attempts for the tip '" + getTIPId() + "'"); - return null; - } - - t = new BSPTask(jobId, jobFile, taskid, partition); - activeTasks.put(taskid, status.getGroomName()); - - return t; - } - - // ////////////////////////////////// - // Accessors - // ////////////////////////////////// - /** - * Return the start time - */ - public long getStartTime() { - return startTime; - } - - /** - * Return the parent job - */ - public JobInProgress getJob() { - return job; - } - - public TaskID getTIPId() { - return id; - } - - public TaskID getTaskId() { - return this.id; - } - - public TreeMap getTasks() { - return activeTasks; - } - - /** - * Is the Task associated with taskid is the first attempt of the tip? - * - * @param taskId - * @return Returns true if the Task is the first attempt of the tip - */ - public boolean isFirstAttempt(TaskAttemptID taskId) { - return firstTaskId == null ? false : firstTaskId.equals(taskId); - } - - /** - * Is this tip currently running any tasks? - * - * @return true if any tasks are running - */ - public boolean isRunning() { - return !activeTasks.isEmpty(); - } - - /** - * Is this tip complete? - * - * @return true if the tip is complete, else false - */ - public synchronized boolean isComplete() { - return (completes > 0); - } - - /** - * Is the given taskid the one that took this tip to completion? - * - * @param taskid taskid of attempt to check for completion - * @return true if taskid is complete, else false - */ - public boolean isComplete(TaskAttemptID taskid) { - return (completes > 0 && taskid.equals(getSuccessfulTaskid())); - } - - private TreeSet tasksReportedClosed = new TreeSet(); - - public boolean shouldCloseForClosedJob(TaskAttemptID taskid) { - TaskStatus ts = (TaskStatus) taskStatuses.get(taskid); - if ((ts != null) && (!tasksReportedClosed.contains(taskid)) - && (job.getStatus().getRunState() != JobStatus.RUNNING)) { - tasksReportedClosed.add(taskid); - return true; - } else { - return false; - } - } - - public void completed(TaskAttemptID taskid) { - LOG.info("Task '" + taskid.getTaskID().toString() + "' has completed."); - - TaskStatus status = (TaskStatus) taskStatuses.get(taskid); - status.setRunState(TaskStatus.State.SUCCEEDED); - activeTasks.remove(taskid); - - // Note the successful taskid - setSuccessfulTaskid(taskid); - - // - // Now that the TIP is complete, the other speculative - // subtasks will be closed when the owning groom server - // reports in and calls shouldClose() on this object. - // - - this.completes++; - } - - public void terminated(TaskAttemptID taskid) { - LOG.info("Task '" + taskid.getTaskID().toString() + "' has failed."); - - TaskStatus status = (TaskStatus) taskStatuses.get(taskid); - status.setRunState(TaskStatus.State.FAILED); - activeTasks.remove(taskid); - } - - private void setSuccessfulTaskid(TaskAttemptID taskid) { - this.successfulTaskId = taskid; - } - - private TaskAttemptID getSuccessfulTaskid() { - return successfulTaskId; - } - - public void updateStatus(TaskStatus status) { - taskStatuses.put(status.getTaskId(), status); - } - - public TaskStatus getTaskStatus(TaskAttemptID taskId) { - return this.taskStatuses.get(taskId); - } - - public void kill() { - this.failed = true; - } - - public boolean isFailed() { - return failed; - } - - /** - * @param conf the conf to set - */ - public void setConf(Configuration conf) { - this.conf = conf; - } - - /** - * @return the conf - */ - public Configuration getConf() { - return conf; - } - - /** - * @param bspMaster the bspMaster to set - */ - public void setBspMaster(BSPMaster bspMaster) { - this.bspMaster = bspMaster; - } - - /** - * @return the bspMaster - */ - public BSPMaster getBspMaster() { - return bspMaster; - } - -} Index: src/java/org/apache/hama/bsp/DoubleMessage.java =================================================================== --- src/java/org/apache/hama/bsp/DoubleMessage.java (revision 1152138) +++ src/java/org/apache/hama/bsp/DoubleMessage.java (working copy) @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * A message that consists of a string tag and a double data. - */ -public class DoubleMessage extends BSPMessage { - - private String tag; - private double data; - - public DoubleMessage() { - super(); - } - - public DoubleMessage(String tag, double data) { - super(); - this.data = data; - this.tag = tag; - } - - @Override - public String getTag() { - return tag; - } - - @Override - public Double getData() { - return data; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeUTF(tag); - out.writeDouble(data); - } - - @Override - public void readFields(DataInput in) throws IOException { - tag = in.readUTF(); - data = in.readDouble(); - } - -} Index: src/java/org/apache/hama/bsp/QueueManager.java =================================================================== --- src/java/org/apache/hama/bsp/QueueManager.java (revision 1152138) +++ src/java/org/apache/hama/bsp/QueueManager.java (working copy) @@ -1,99 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; - -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.hadoop.conf.Configuration; - -/** - * A BSPJob Queue Manager. - */ -public class QueueManager{ - - private ConcurrentMap> queues = - new ConcurrentHashMap>(); - - public QueueManager(Configuration conf){ } - - /** - * Initialize a job. - * @param job required initialzied. - */ - public void initJob(JobInProgress job){ - try{ - //job.updateStatus(); - job.initTasks(); - }catch(IOException ioe){ - ioe.printStackTrace(); - } - } - - /** - * Add a job to the specified queue. - * @param name of the queue. - * @param job to be added. - */ - public void addJob(String name, JobInProgress job){ - Queue queue = queues.get(name); - if(null != queue) queue.addJob(job); - } - - /** - * Remove a job from the head of a designated queue. - * @param name from which a job is removed. - * @param job to be removed from the queue. - */ - public void removeJob(String name, JobInProgress job){ - Queue queue = queues.get(name); - if(null != queue) queue.removeJob(job); - } - - /** - * Move a job from a queue to another. - * @param from a queue a job is to be removed. - * @param to a queue a job is to be added. - */ - public void moveJob(String from, String to, JobInProgress job){ - synchronized(queues){ - removeJob(from, job); - addJob(to, job); - } - } - - /** - * Create a FCFS queue with the name provided. - * @param name of the queue. - */ - public void createFCFSQueue(String name){ - queues.putIfAbsent(name, new FCFSQueue(name)); - } - - /** - * Find Queue according to the name specified. - * @param name of the queue. - * @return queue of JobInProgress - */ - public Queue findQueue(String name){ - return queues.get(name); - } - -} Index: src/java/org/apache/hama/bsp/ReportGroomStatusDirective.java =================================================================== --- src/java/org/apache/hama/bsp/ReportGroomStatusDirective.java (revision 1152138) +++ src/java/org/apache/hama/bsp/ReportGroomStatusDirective.java (working copy) @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Writable; - -/** - * Reports status of GroomServer. - */ -public class ReportGroomStatusDirective extends Directive implements Writable { - public static final Log LOG = LogFactory.getLog(ReportGroomStatusDirective.class); - - private GroomServerStatus status; - - public ReportGroomStatusDirective(){ super(); } - - public ReportGroomStatusDirective(GroomServerStatus status) { - super(Directive.Type.Response); - this.status = status; - } - - public GroomServerStatus getStatus() { - return this.status; - } - - public void write(DataOutput out) throws IOException { - super.write(out); - this.status.write(out); - } - - public void readFields(DataInput in) throws IOException { - super.readFields(in); - this.status = new GroomServerStatus(); - this.status.readFields(in); - } -} Index: src/java/org/apache/hama/bsp/TaskLogAppender.java =================================================================== --- src/java/org/apache/hama/bsp/TaskLogAppender.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskLogAppender.java (working copy) @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.util.LinkedList; -import java.util.Queue; - -import org.apache.log4j.FileAppender; -import org.apache.log4j.spi.LoggingEvent; - -/** - * A simple log4j-appender for the task child's BSP system logs. - */ -public class TaskLogAppender extends FileAppender { - private String taskId; // taskId should be managed as String rather than - // TaskID object - // so that log4j can configure it from the configuration(log4j.properties). - private int maxEvents; - private Queue tail = null; - - @Override - public void activateOptions() { - synchronized (this) { - if (maxEvents > 0) { - tail = new LinkedList(); - } - setFile(TaskLog.getTaskLogFile(TaskAttemptID.forName(taskId), - TaskLog.LogName.SYSLOG).toString()); - setAppend(true); - super.activateOptions(); - } - } - - @Override - public void append(LoggingEvent event) { - synchronized (this) { - if (tail == null) { - super.append(event); - } else { - if (tail.size() >= maxEvents) { - tail.remove(); - } - tail.add(event); - } - } - } - - @Override - public synchronized void close() { - if (tail != null) { - for (LoggingEvent event : tail) { - super.append(event); - } - } - super.close(); - } - - /** - * Getter/Setter methods for log4j. - */ - - public String getTaskId() { - return taskId; - } - - public void setTaskId(String taskId) { - this.taskId = taskId; - } - - private static final int EVENT_SIZE = 100; - - public long getTotalLogFileSize() { - return maxEvents * EVENT_SIZE; - } - - public void setTotalLogFileSize(long logSize) { - maxEvents = (int) logSize / EVENT_SIZE; - } - -} Index: src/java/org/apache/hama/bsp/BSPJobContext.java =================================================================== --- src/java/org/apache/hama/bsp/BSPJobContext.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPJobContext.java (working copy) @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; -import java.io.OutputStream; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hama.HamaConfiguration; - -/** - * A read-only view of the bsp job that is provided to the tasks while they are - * running. - */ -public class BSPJobContext { - // Put all of the attribute names in here so that BSPJob and JobContext are - // consistent. - protected static final String WORK_CLASS_ATTR = "bsp.work.class"; - protected static final String INPUT_FORMAT_CLASS_ATTR = "bsp.inputformat.class"; - protected static final String OUTPUT_FORMAT_CLASS_ATTR = "bsp.outputformat.class"; - protected static final String WORKING_DIR = "bsp.working.dir"; - - protected final Configuration conf; - private final BSPJobID jobId; - - public BSPJobContext(Configuration conf, BSPJobID jobId) { - this.conf = conf; - this.jobId = jobId; - } - - public BSPJobContext(Path config, BSPJobID jobId) throws IOException { - this.conf = new HamaConfiguration(); - this.jobId = jobId; - this.conf.addResource(config); - } - - public BSPJobID getJobID() { - return jobId; - } - - public Path getWorkingDirectory() throws IOException { - String name = conf.get(WORKING_DIR); - - if (name != null) { - return new Path(name); - } else { - try { - Path dir = FileSystem.get(conf).getWorkingDirectory(); - conf.set(WORKING_DIR, dir.toString()); - return dir; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - public String getJobName() { - return conf.get("bsp.job.name", ""); - } - - public String getJar() { - return conf.get("bsp.jar"); - } - - /** - * Constructs a local file name. Files are distributed among configured - * local directories. - */ - public Path getLocalPath(String pathString) throws IOException { - return conf.getLocalPath("bsp.local.dir", pathString); - } - - public String getUser() { - return conf.get("user.name"); - } - - public void writeXml(OutputStream out) throws IOException { - conf.writeXml(out); - } - - public Configuration getConf() { - return this.conf; - } - - public String get(String name) { - return conf.get(name); - } - - public int getInt(String name, int defaultValue) { - return conf.getInt(name, defaultValue); - } -} Index: src/java/org/apache/hama/bsp/BooleanMessage.java =================================================================== --- src/java/org/apache/hama/bsp/BooleanMessage.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BooleanMessage.java (working copy) @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * A message that consists of a string tag and a boolean value. - */ -public class BooleanMessage extends BSPMessage { - - String tag; - boolean data; - - public BooleanMessage() { - super(); - } - - public BooleanMessage(String tag, boolean data) { - super(); - this.tag = tag; - this.data = data; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeUTF(tag); - out.writeBoolean(data); - } - - @Override - public void readFields(DataInput in) throws IOException { - tag = in.readUTF(); - data = in.readBoolean(); - } - - @Override - public String getTag() { - return tag; - } - - @Override - public Boolean getData() { - return data; - } -} \ No newline at end of file Index: src/java/org/apache/hama/bsp/ClusterStatus.java =================================================================== --- src/java/org/apache/hama/bsp/ClusterStatus.java (revision 1152138) +++ src/java/org/apache/hama/bsp/ClusterStatus.java (working copy) @@ -1,176 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; - -/** - * Status information on the current state of the BSP cluster. - * - *

ClusterStatus provides clients with information such as: - *

    - *
  1. - * Size of the cluster. - *
  2. - *
  3. - * Name of the grooms. - *
  4. - *
  5. - * Task capacity of the cluster. - *
  6. - *
  7. - * The number of currently running bsp tasks. - *
  8. - *
  9. - * State of the BSPMaster. - *
  10. - *

- * - *

Clients can query for the latest ClusterStatus, via - * {@link BSPJobClient#getClusterStatus(boolean)}.

- * - * @see BSPMaster - */ -public class ClusterStatus implements Writable { - - private int numActiveGrooms; - private Map activeGrooms = new HashMap(); - private int tasks; - private int maxTasks; - private BSPMaster.State state; - - /** - * - */ - public ClusterStatus() {} - - public ClusterStatus(int grooms, int tasks, int maxTasks, BSPMaster.State state) { - this.numActiveGrooms = grooms; - this.tasks = tasks; - this.maxTasks = maxTasks; - this.state = state; - } - - public ClusterStatus(Map activeGrooms, int tasks, int maxTasks, - BSPMaster.State state) { - this(activeGrooms.size(), tasks, maxTasks, state); - this.activeGrooms = activeGrooms; - } - - /** - * Get the number of groom servers in the cluster. - * - * @return the number of groom servers in the cluster. - */ - public int getGroomServers() { - return numActiveGrooms; - } - - /** - * Get the names of groom servers, and their peers, in the cluster. - * - * @return the active groom servers in the cluster. - */ - public Map getActiveGroomNames() { - return activeGrooms; - } - - /** - * Get the number of currently running tasks in the cluster. - * - * @return the number of currently running tasks in the cluster. - */ - public int getTasks() { - return tasks; - } - - /** - * Get the maximum capacity for running tasks in the cluster. - * - * @return the maximum capacity for running tasks in the cluster. - */ - public int getMaxTasks() { - return maxTasks; - } - - /** - * Get the current state of the BSPMaster, - * as {@link BSPMaster.State} - * - * @return the current state of the BSPMaster. - */ - public BSPMaster.State getBSPMasterState() { - return state; - } - - ////////////////////////////////////////////// - // Writable - ////////////////////////////////////////////// - @Override - public void write(DataOutput out) throws IOException { - if (activeGrooms.isEmpty()) { - out.writeInt(numActiveGrooms); - out.writeBoolean(false); - } else { - out.writeInt(activeGrooms.size()); - out.writeBoolean(true); - - String[] groomNames = activeGrooms.keySet().toArray(new String[0]); - List peerNames = new ArrayList(); - - for (String groomName : groomNames) { - peerNames.add(activeGrooms.get(groomName)); - } - - WritableUtils.writeCompressedStringArray(out, groomNames); - WritableUtils.writeCompressedStringArray(out, peerNames.toArray(new String[0])); - } - out.writeInt(tasks); - out.writeInt(maxTasks); - WritableUtils.writeEnum(out, state); - } - - @Override - public void readFields(DataInput in) throws IOException { - numActiveGrooms = in.readInt(); - boolean groomListFollows = in.readBoolean(); - - if (groomListFollows) { - String[] groomNames = WritableUtils.readCompressedStringArray(in); - String[] peerNames = WritableUtils.readCompressedStringArray(in); - activeGrooms = new HashMap(groomNames.length); - - for (int i = 0; i < groomNames.length; i++) { - activeGrooms.put(groomNames[i], peerNames[i]); - } - } - - tasks = in.readInt(); - maxTasks = in.readInt(); - state = WritableUtils.readEnum(in, BSPMaster.State.class); - } -} Index: src/java/org/apache/hama/bsp/FCFSQueue.java =================================================================== --- src/java/org/apache/hama/bsp/FCFSQueue.java (revision 1152138) +++ src/java/org/apache/hama/bsp/FCFSQueue.java (working copy) @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.util.Collection; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -class FCFSQueue implements Queue { - - public static final Log LOG = LogFactory.getLog(FCFSQueue.class); - private final String name; - private BlockingQueue queue = new LinkedBlockingQueue(); - - public FCFSQueue(String name) { - this.name = name; - } - - @Override - public String getName() { - return this.name; - } - - @Override - public void addJob(JobInProgress job) { - try { - queue.put(job); - } catch (InterruptedException ie) { - LOG.error("Fail to add a job to the " + this.name + " queue.", ie); - } - } - - @Override - public void removeJob(JobInProgress job) { - queue.remove(job); - } - - @Override - public JobInProgress removeJob() { - try { - return queue.take(); - } catch (InterruptedException ie) { - LOG.error("Fail to remove a job from the " + this.name + " queue.", ie); - } - return null; - } - - @Override - public Collection jobs() { - return queue; - } - -} Index: src/java/org/apache/hama/bsp/JobInProgress.java =================================================================== --- src/java/org/apache/hama/bsp/JobInProgress.java (revision 1152138) +++ src/java/org/apache/hama/bsp/JobInProgress.java (working copy) @@ -1,360 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; - -/** - * JobInProgress maintains all the info for keeping a Job on the straight and - * narrow. It keeps its JobProfile and its latest JobStatus, plus a set of - * tables for doing bookkeeping of its Tasks.ss - */ -class JobInProgress { - - /** - * Used when the a kill is issued to a job which is initializing. - */ - static class KillInterruptedException extends InterruptedException { - private static final long serialVersionUID = 1L; - - public KillInterruptedException(String msg) { - super(msg); - } - } - - static final Log LOG = LogFactory.getLog(JobInProgress.class); - boolean tasksInited = false; - boolean jobInited = false; - - Configuration conf; - JobProfile profile; - JobStatus status; - Path jobFile = null; - Path localJobFile = null; - Path localJarFile = null; - private LocalFileSystem localFs; - // Indicates how many times the job got restarted - private int restartCount; - - long startTime; - long launchTime; - long finishTime; - - private String jobName; - - // private LocalFileSystem localFs; - private BSPJobID jobId; - final BSPMaster master; - TaskInProgress tasks[] = new TaskInProgress[0]; - private long superstepCounter; - - int numBSPTasks = 0; - int clusterSize; - - public JobInProgress(BSPJobID jobId, Path jobFile, BSPMaster master, - Configuration conf) throws IOException { - this.conf = conf; - this.jobId = jobId; - this.localFs = FileSystem.getLocal(conf); - this.jobFile = jobFile; - this.master = master; - this.status = new JobStatus(jobId, null, 0L, 0L, - JobStatus.State.PREP.value()); - this.startTime = System.currentTimeMillis(); - this.superstepCounter = 0; - this.restartCount = 0; - - this.localJobFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId - + ".xml"); - this.localJarFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId - + ".jar"); - - Path jobDir = master.getSystemDirectoryForJob(jobId); - FileSystem fs = jobDir.getFileSystem(conf); - fs.copyToLocalFile(jobFile, localJobFile); - BSPJob job = new BSPJob(jobId, localJobFile.toString()); - this.numBSPTasks = job.getNumBspTask(); - - this.profile = new JobProfile(job.getUser(), jobId, jobFile.toString(), - job.getJobName()); - - this.setJobName(job.getJobName()); - - status.setUsername(job.getUser()); - status.setStartTime(startTime); - - String jarFile = job.getJar(); - if (jarFile != null) { - fs.copyToLocalFile(new Path(jarFile), localJarFile); - } - - } - - public JobProfile getProfile() { - return profile; - } - - public JobStatus getStatus() { - return status; - } - - public synchronized long getLaunchTime() { - return launchTime; - } - - public long getStartTime() { - return startTime; - } - - public long getFinishTime() { - return finishTime; - } - - /** - * @return the number of desired tasks. - */ - public int desiredBSPTasks() { - return numBSPTasks; - } - - /** - * @return The JobID of this JobInProgress. - */ - public BSPJobID getJobID() { - return jobId; - } - - public synchronized TaskInProgress findTaskInProgress(TaskID id) { - if (areTasksInited()) { - for (TaskInProgress tip : tasks) { - if (tip.getTaskId().equals(id)) { - return tip; - } - } - } - return null; - } - - public synchronized boolean areTasksInited() { - return this.tasksInited; - } - - public String toString() { - return "jobName:" + profile.getJobName() + "\n" + "submit user:" - + profile.getUser() + "\n" + "JobId:" + jobId + "\n" + "JobFile:" - + jobFile + "\n"; - } - - // /////////////////////////////////////////////////// - // Create/manage tasks - // /////////////////////////////////////////////////// - - public synchronized void initTasks() throws IOException { - if (tasksInited) { - return; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("numBSPTasks: " + numBSPTasks); - } - - // adjust number of map tasks to actual number of splits - this.tasks = new TaskInProgress[numBSPTasks]; - for (int i = 0; i < numBSPTasks; i++) { - tasks[i] = new TaskInProgress(getJobID(), this.jobFile.toString(), - this.master, this.conf, this, i); - } - - // Update job status - this.status = new JobStatus(this.status.getJobID(), this.profile.getUser(), - 0L, 0L, JobStatus.RUNNING); - - tasksInited = true; - LOG.debug("Job is initialized."); - } - - public synchronized Task obtainNewTask(GroomServerStatus status, - int clusterSize) { - this.clusterSize = clusterSize; - - if (this.status.getRunState() != JobStatus.RUNNING) { - LOG.info("Cannot create task split for " + profile.getJobID()); - return null; - } - - Task result = null; - try { - for (int i = 0; i < tasks.length; i++) { - if (!tasks[i].isRunning() && !tasks[i].isComplete()) { - result = tasks[i].getTaskToRun(status); - break; - } - } - - } catch (IOException e) { - LOG.error("Exception while obtaining new task!", e); - } - return result; - } - - public synchronized void completedTask(TaskInProgress tip, TaskStatus status) { - TaskAttemptID taskid = status.getTaskId(); - updateTaskStatus(tip, status); - LOG.info("Taskid '" + taskid + "' has finished successfully."); - tip.completed(taskid); - - // - // If all tasks are complete, then the job is done! - // - - boolean allDone = true; - for (TaskInProgress taskInProgress : tasks) { - if (!taskInProgress.isComplete()) { - allDone = false; - break; - } - } - - if (allDone) { - this.status = new JobStatus(this.status.getJobID(), - this.profile.getUser(), superstepCounter, superstepCounter, - superstepCounter, JobStatus.SUCCEEDED, superstepCounter); - this.finishTime = System.currentTimeMillis(); - this.status.setFinishTime(this.finishTime); - - LOG.debug("Job successfully done."); - - garbageCollect(); - } - } - - public void failedTask(TaskInProgress tip, TaskStatus status) { - TaskAttemptID taskid = status.getTaskId(); - updateTaskStatus(tip, status); - LOG.info("Taskid '" + taskid + "' has failed."); - tip.terminated(taskid); - - // - // If all tasks are complete, then the job is done! - // - - boolean allDone = true; - for (TaskInProgress taskInProgress : tasks) { - if (!taskInProgress.isFailed()) { - allDone = false; - break; - } - } - - if (allDone) { - this.status = new JobStatus(this.status.getJobID(), - this.profile.getUser(), superstepCounter, superstepCounter, - superstepCounter, JobStatus.FAILED, superstepCounter); - this.finishTime = System.currentTimeMillis(); - this.status.setFinishTime(this.finishTime); - - LOG.debug("Job failed."); - - garbageCollect(); - } - } - - public synchronized void updateTaskStatus(TaskInProgress tip, - TaskStatus taskStatus) { - tip.updateStatus(taskStatus); // update tip - - if (superstepCounter < taskStatus.getSuperstepCount()) { - superstepCounter = taskStatus.getSuperstepCount(); - // TODO Later, we have to update JobInProgress status here - - } - } - - public synchronized void kill() { - if (status.getRunState() != JobStatus.KILLED) { - this.status = new JobStatus(status.getJobID(), this.profile.getUser(), - 0L, 0L, 0L, JobStatus.KILLED); - this.finishTime = System.currentTimeMillis(); - this.status.setFinishTime(this.finishTime); - // - // kill all TIPs. - // - for (int i = 0; i < tasks.length; i++) { - tasks[i].kill(); - } - - garbageCollect(); - } - - } - - /** - * The job is dead. We're now GC'ing it, getting rid of the job from all - * tables. Be sure to remove all of this job's tasks from the various tables. - */ - synchronized void garbageCollect() { - try { - // Definitely remove the local-disk copy of the job file - if (localJobFile != null) { - localFs.delete(localJobFile, true); - localJobFile = null; - } - if (localJarFile != null) { - localFs.delete(localJarFile, true); - localJarFile = null; - } - - // JobClient always creates a new directory with job files - // so we remove that directory to cleanup - FileSystem fs = FileSystem.get(conf); - fs.delete(new Path(profile.getJobFile()).getParent(), true); - - } catch (IOException e) { - LOG.info("Error cleaning up " + profile.getJobID() + ": " + e); - } - } - - /** - * Get the number of times the job has restarted - */ - int getNumRestarts() { - return restartCount; - } - - /** - * @param jobName the jobName to set - */ - public void setJobName(String jobName) { - this.jobName = jobName; - } - - /** - * @return the jobName - */ - public String getJobName() { - return jobName; - } - -} Index: src/java/org/apache/hama/bsp/BSPMessage.java =================================================================== --- src/java/org/apache/hama/bsp/BSPMessage.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPMessage.java (working copy) @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import org.apache.hadoop.io.Writable; - -/** - * BSPMessage consists of the tag and the arbitrary amount of data to be - * communicated. - */ -public abstract class BSPMessage implements Messagable, Writable { - - /** - * BSP messages are typically identified with tags. This allows to get the tag - * of data. - * - * @return tag of data of BSP message - */ - public abstract Object getTag(); - - /** - * @return data of BSP message - */ - public abstract Object getData(); - -} Index: src/java/org/apache/hama/bsp/ID.java =================================================================== --- src/java/org/apache/hama/bsp/ID.java (revision 1152138) +++ src/java/org/apache/hama/bsp/ID.java (working copy) @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.WritableComparable; - -/** - * A general identifier, which internally stores the id as an integer. This is - * the super class of {@link BSPJobID}, {@link TaskID} and {@link TaskAttemptID} - * . - */ -public abstract class ID implements WritableComparable { - protected static final char SEPARATOR = '_'; - protected int id; - - public ID(int id) { - this.id = id; - } - - protected ID() { - } - - public int getId() { - return id; - } - - @Override - public String toString() { - return String.valueOf(id); - } - - @Override - public int hashCode() { - return Integer.valueOf(id).hashCode(); - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null) - return false; - if (o.getClass() == this.getClass()) { - ID that = (ID) o; - return this.id == that.id; - } else - return false; - } - - public int compareTo(ID that) { - return this.id - that.id; - } - - public void readFields(DataInput in) throws IOException { - this.id = in.readInt(); - } - - public void write(DataOutput out) throws IOException { - out.writeInt(id); - } -} Index: src/java/org/apache/hama/bsp/IntegerDoubleMessage.java =================================================================== --- src/java/org/apache/hama/bsp/IntegerDoubleMessage.java (revision 1152138) +++ src/java/org/apache/hama/bsp/IntegerDoubleMessage.java (working copy) @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * A message that consists of a int tag and a double data. - */ -public class IntegerDoubleMessage extends BSPMessage { - - int tag; - double data; - - public IntegerDoubleMessage() { - super(); - } - - public IntegerDoubleMessage(int tag, double data) { - super(); - this.tag = tag; - this.data = data; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(tag); - out.writeDouble(data); - } - - @Override - public void readFields(DataInput in) throws IOException { - tag = in.readInt(); - data = in.readDouble(); - } - - @Override - public Integer getTag() { - return tag; - } - - @Override - public Double getData() { - return data; - } - -} Index: src/java/org/apache/hama/bsp/BSPPeer.java =================================================================== --- src/java/org/apache/hama/bsp/BSPPeer.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPPeer.java (working copy) @@ -1,423 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ipc.RPC.Server; -import org.apache.hama.Constants; -import org.apache.hama.ipc.BSPPeerProtocol; -import org.apache.hama.util.Bytes; -import org.apache.hama.zookeeper.QuorumPeer; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; - -/** - * This class represents a BSP peer. - */ -public class BSPPeer implements Watcher, BSPPeerInterface { - - public static final Log LOG = LogFactory.getLog(BSPPeer.class); - - private Configuration conf; - private BSPJob jobConf; - - private Server server = null; - private ZooKeeper zk = null; - private volatile Integer mutex = 0; - - private final String bspRoot; - private final String quorumServers; - - private final Map peers = new ConcurrentHashMap(); - private final Map> outgoingQueues = new ConcurrentHashMap>(); - private ConcurrentLinkedQueue localQueue = new ConcurrentLinkedQueue(); - private ConcurrentLinkedQueue localQueueForNextIteration = new ConcurrentLinkedQueue(); - private final Map peerSocketCache = new ConcurrentHashMap(); - - private SortedSet allPeerNames = new TreeSet(); - private InetSocketAddress peerAddress; - private TaskStatus currentTaskStatus; - - private TaskAttemptID taskid; - private BSPPeerProtocol umbilical; - - /** - * Protected default constructor for LocalBSPRunner. - */ - protected BSPPeer() { - bspRoot = null; - quorumServers = null; - } - - /** - * Constructor - * - * @param umbilical - * @param taskid - */ - public BSPPeer(Configuration conf, TaskAttemptID taskid, - BSPPeerProtocol umbilical) throws IOException { - this.conf = conf; - this.taskid = taskid; - this.umbilical = umbilical; - - String bindAddress = conf.get(Constants.PEER_HOST, - Constants.DEFAULT_PEER_HOST); - int bindPort = conf - .getInt(Constants.PEER_PORT, Constants.DEFAULT_PEER_PORT); - bspRoot = conf.get(Constants.ZOOKEEPER_ROOT, - Constants.DEFAULT_ZOOKEEPER_ROOT); - quorumServers = QuorumPeer.getZKQuorumServersString(conf); - LOG.debug("Quorum " + quorumServers); - - // TODO: may require to dynamic reflect the underlying - // network e.g. ip address, port. - peerAddress = new InetSocketAddress(bindAddress, bindPort); - reinitialize(); - } - - public void reinitialize() { - try { - LOG.debug("reinitialize(): " + getPeerName()); - server = RPC.getServer(this, peerAddress.getHostName(), - peerAddress.getPort(), conf); - server.start(); - LOG.info(" BSPPeer address:" + peerAddress.getHostName() + " port:" - + peerAddress.getPort()); - } catch (IOException e) { - LOG.error("Exception during reinitialization!", e); - } - - try { - zk = new ZooKeeper(quorumServers, conf.getInt( - Constants.ZOOKEEPER_SESSION_TIMEOUT, 1200000), this); - } catch (IOException e) { - LOG.error("Exception during reinitialization!", e); - } - - Stat s = null; - if (zk != null) { - try { - s = zk.exists(Constants.DEFAULT_ZOOKEEPER_ROOT, false); - } catch (Exception e) { - LOG.error(s, e); - } - - if (s == null) { - try { - zk.create(Constants.DEFAULT_ZOOKEEPER_ROOT, new byte[0], - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } catch (KeeperException e) { - LOG.error(e); - } catch (InterruptedException e) { - LOG.error(e); - } - } - } - - } - - @Override - public BSPMessage getCurrentMessage() throws IOException { - return localQueue.poll(); - } - - /* - * (non-Javadoc) - * @see org.apache.hama.bsp.BSPPeerInterface#send(java.net.InetSocketAddress, - * org.apache.hadoop.io.Writable, org.apache.hadoop.io.Writable) - */ - @Override - public void send(String peerName, BSPMessage msg) throws IOException { - if (peerName.equals(getPeerName())) { - LOG.debug("Local send bytes (" + msg.getData().toString() + ")"); - localQueueForNextIteration.add(msg); - } else { - LOG.debug("Send bytes (" + msg.getData().toString() + ") to " + peerName); - InetSocketAddress targetPeerAddress = null; - // Get socket for target peer. - if (peerSocketCache.containsKey(peerName)) { - targetPeerAddress = peerSocketCache.get(peerName); - } else { - targetPeerAddress = getAddress(peerName); - peerSocketCache.put(peerName, targetPeerAddress); - } - ConcurrentLinkedQueue queue = outgoingQueues - .get(targetPeerAddress); - if (queue == null) { - queue = new ConcurrentLinkedQueue(); - } - queue.add(msg); - outgoingQueues.put(targetPeerAddress, queue); - } - } - - /* - * (non-Javadoc) - * @see org.apache.hama.bsp.BSPPeerInterface#sync() - */ - @Override - public void sync() throws IOException, KeeperException, InterruptedException { - enterBarrier(); - long startTime = System.currentTimeMillis(); - Iterator>> it = this.outgoingQueues - .entrySet().iterator(); - - while (it.hasNext()) { - Entry> entry = it - .next(); - - BSPPeerInterface peer = peers.get(entry.getKey()); - if (peer == null) { - peer = getBSPPeerConnection(entry.getKey()); - } - Iterable messages = entry.getValue(); - BSPMessageBundle bundle = new BSPMessageBundle(); - for (BSPMessage message : messages) { - bundle.addMessage(message); - } - peer.put(bundle); - } - - if ((System.currentTimeMillis() - startTime) < 200) { - Thread.sleep(200); - } - - leaveBarrier(); - currentTaskStatus.incrementSuperstepCount(); - umbilical.incrementSuperstepCount(taskid); - - startTime = System.currentTimeMillis(); - // Clear outgoing queues. - clearOutgoingQueues(); - - // Add non-processed messages from this iteration for the next's queue. - while (!localQueue.isEmpty()) { - BSPMessage message = localQueue.poll(); - localQueueForNextIteration.add(message); - } - // Switch local queues. - localQueue = localQueueForNextIteration; - localQueueForNextIteration = new ConcurrentLinkedQueue(); - - // TODO: This is a quite temporary solution of HAMA-387. - // If zk.getChildren() response is slower than 200 milliseconds, - // BSP system will be hanged. - - // We have to consider another way to avoid this problem. - if ((System.currentTimeMillis() - startTime) < 200) { - Thread.sleep(200); // at least wait - } - } - - protected boolean enterBarrier() throws KeeperException, InterruptedException { - LOG.debug("[" + getPeerName() + "] enter the enterbarrier: " - + this.getSuperstepCount()); - zk.create(bspRoot + "/" + getPeerName(), - Bytes.toBytes(this.getSuperstepCount()), Ids.OPEN_ACL_UNSAFE, - CreateMode.EPHEMERAL); - - while (true) { - synchronized (mutex) { - List list = zk.getChildren(bspRoot, true); - - if (list.size() < jobConf.getNumBspTask()) { - mutex.wait(); - } else { - return true; - } - } - } - } - - protected boolean leaveBarrier() throws KeeperException, InterruptedException { - zk.delete(bspRoot + "/" + getPeerName(), 0); - while (true) { - synchronized (mutex) { - List list = zk.getChildren(bspRoot, true); - - if (list.size() > 0) { - mutex.wait(); - } else { - return true; - } - } - } - } - - @Override - public void process(WatchedEvent event) { - synchronized (mutex) { - mutex.notify(); - } - } - - public void clear() { - this.localQueue.clear(); - this.outgoingQueues.clear(); - } - - @Override - public void close() throws IOException { - server.stop(); - } - - @Override - public void put(BSPMessage msg) throws IOException { - this.localQueueForNextIteration.add(msg); - } - - @Override - public void put(BSPMessageBundle messages) throws IOException { - for (BSPMessage message : messages.getMessages()) { - this.localQueueForNextIteration.add(message); - } - } - - @Override - public long getProtocolVersion(String arg0, long arg1) throws IOException { - return BSPPeerInterface.versionID; - } - - protected BSPPeerInterface getBSPPeerConnection(InetSocketAddress addr) { - BSPPeerInterface peer; - synchronized (this.peers) { - peer = peers.get(addr); - - if (peer == null) { - try { - peer = (BSPPeerInterface) RPC.getProxy(BSPPeerInterface.class, - BSPPeerInterface.versionID, addr, this.conf); - } catch (IOException e) { - - } - this.peers.put(addr, peer); - } - } - - return peer; - } - - /** - * @return the string as host:port of this Peer - */ - public String getPeerName() { - return peerAddress.getHostName() + ":" + peerAddress.getPort(); - } - - private InetSocketAddress getAddress(String peerName) { - String[] peerAddrParts = peerName.split(":"); - return new InetSocketAddress(peerAddrParts[0], - Integer.parseInt(peerAddrParts[1])); - } - - @Override - public String[] getAllPeerNames() { - return allPeerNames.toArray(new String[0]); - } - - /** - * To be invoked by the Groom Server with a list of peers received from an - * heartbeat response (BSPMaster). - * - * @param allPeers - */ - void setAllPeerNames(Collection allPeerNames) { - this.allPeerNames = new TreeSet(allPeerNames); - } - - /** - * @return the number of messages - */ - public int getNumCurrentMessages() { - return localQueue.size(); - } - - /** - * Sets the current status - * - * @param currentTaskStatus - */ - public void setCurrentTaskStatus(TaskStatus currentTaskStatus) { - this.currentTaskStatus = currentTaskStatus; - } - - /** - * @return the count of current super-step - */ - public long getSuperstepCount() { - return currentTaskStatus.getSuperstepCount(); - } - - /** - * Sets the job configuration - * - * @param jobConf - */ - public void setJobConf(BSPJob jobConf) { - this.jobConf = jobConf; - } - - /** - * @return the size of local queue - */ - public int getLocalQueueSize() { - return localQueue.size(); - } - - /** - * @return the size of outgoing queue - */ - public int getOutgoingQueueSize() { - return outgoingQueues.size(); - } - - /** - * Clears local queue - */ - public void clearLocalQueue() { - this.localQueue.clear(); - } - - /** - * Clears outgoing queues - */ - public void clearOutgoingQueues() { - this.outgoingQueues.clear(); - } -} Index: src/java/org/apache/hama/bsp/BSPTaskRunner.java =================================================================== --- src/java/org/apache/hama/bsp/BSPTaskRunner.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPTaskRunner.java (working copy) @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Base class that runs a task in a separate process. - */ -public class BSPTaskRunner extends TaskRunner { - - public static final Log LOG = LogFactory.getLog(BSPTaskRunner.class); - - public BSPTaskRunner(BSPTask bspTask, GroomServer groom, BSPJob conf) { - super(bspTask, groom, conf); - } - -} Index: src/java/org/apache/hama/bsp/IntegerMessage.java =================================================================== --- src/java/org/apache/hama/bsp/IntegerMessage.java (revision 1152138) +++ src/java/org/apache/hama/bsp/IntegerMessage.java (working copy) @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * A message that consists of a string tag and a int data. - */ -public class IntegerMessage extends BSPMessage { - - String tag; - int data; - - public IntegerMessage() { - super(); - } - - public IntegerMessage(String tag, int data) { - super(); - this.tag = tag; - this.data = data; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeUTF(tag); - out.writeInt(data); - } - - @Override - public void readFields(DataInput in) throws IOException { - tag = in.readUTF(); - data = in.readInt(); - } - - @Override - public String getTag() { - return tag; - } - - @Override - public Integer getData() { - return data; - } - -} \ No newline at end of file Index: src/java/org/apache/hama/bsp/Task.java =================================================================== --- src/java/org/apache/hama/bsp/Task.java (revision 1152138) +++ src/java/org/apache/hama/bsp/Task.java (working copy) @@ -1,140 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.LocalDirAllocator; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hama.ipc.BSPPeerProtocol; - -/** - * Base class for tasks. - */ -public abstract class Task implements Writable { - public static final Log LOG = LogFactory.getLog(Task.class); - // ////////////////////////////////////////// - // Fields - // ////////////////////////////////////////// - - protected BSPJobID jobId; - protected String jobFile; - protected TaskAttemptID taskId; - protected int partition; - - protected LocalDirAllocator lDirAlloc; - - public Task() { - jobId = new BSPJobID(); - taskId = new TaskAttemptID(); - } - - public Task(BSPJobID jobId, String jobFile, TaskAttemptID taskId, - int partition) { - this.jobId = jobId; - this.jobFile = jobFile; - this.taskId = taskId; - this.partition = partition; - } - - // ////////////////////////////////////////// - // Accessors - // ////////////////////////////////////////// - public void setJobFile(String jobFile) { - this.jobFile = jobFile; - } - - public String getJobFile() { - return jobFile; - } - - public TaskAttemptID getTaskAttemptId() { - return this.taskId; - } - - public TaskAttemptID getTaskID() { - return taskId; - } - - /** - * Get the job name for this task. - * - * @return the job name - */ - public BSPJobID getJobID() { - return jobId; - } - - /** - * Get the index of this task within the job. - * - * @return the integer part of the task id - */ - public int getPartition() { - return partition; - } - - @Override - public String toString() { - return taskId.toString(); - } - - // ////////////////////////////////////////// - // Writable - // ////////////////////////////////////////// - @Override - public void write(DataOutput out) throws IOException { - jobId.write(out); - Text.writeString(out, jobFile); - taskId.write(out); - out.writeInt(partition); - } - - @Override - public void readFields(DataInput in) throws IOException { - jobId.readFields(in); - jobFile = Text.readString(in); - taskId.readFields(in); - partition = in.readInt(); - } - - /** - * Run this task as a part of the named job. This method is executed in the - * child process. - * - * @param bspPeer for communications - * @param umbilical for communications with GroomServer - */ - public abstract void run(BSPJob job, BSPPeer bspPeer, BSPPeerProtocol umbilical) - throws IOException; - - public abstract BSPTaskRunner createRunner(GroomServer groom); - - public void done(BSPPeerProtocol umbilical) throws IOException { - umbilical.done(getTaskID(), true); - } - - public abstract BSPJob getConf(); - public abstract void setConf(BSPJob localJobConf); - -} Index: src/java/org/apache/hama/bsp/DispatchTasksDirective.java =================================================================== --- src/java/org/apache/hama/bsp/DispatchTasksDirective.java (revision 1152138) +++ src/java/org/apache/hama/bsp/DispatchTasksDirective.java (working copy) @@ -1,111 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; - -/** - * Handles the tasks dispatching between the BSPMaster and the GroomServers. - */ -public final class DispatchTasksDirective extends Directive implements Writable { - - public static final Log LOG = LogFactory.getLog(DispatchTasksDirective.class); - - private Map groomServerPeers; - private GroomServerAction[] actions; - - public DispatchTasksDirective() { - super(); - } - - public DispatchTasksDirective(Map groomServerPeers, - GroomServerAction[] actions) { - super(Directive.Type.Request); - this.groomServerPeers = groomServerPeers; - this.actions = actions; - } - - public Map getGroomServerPeers() { - return this.groomServerPeers; - } - - public GroomServerAction[] getActions() { - return this.actions; - } - - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - if (this.actions == null) { - WritableUtils.writeVInt(out, 0); - } else { - WritableUtils.writeVInt(out, actions.length); - for (GroomServerAction action : this.actions) { - WritableUtils.writeEnum(out, action.getActionType()); - action.write(out); - } - } - String[] groomServerNames = groomServerPeers.keySet() - .toArray(new String[0]); - WritableUtils.writeCompressedStringArray(out, groomServerNames); - - List groomServerAddresses = new ArrayList( - groomServerNames.length); - for (String groomName : groomServerNames) { - groomServerAddresses.add(groomServerPeers.get(groomName)); - } - WritableUtils.writeCompressedStringArray(out, groomServerAddresses - .toArray(new String[0])); - - } - - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - int length = WritableUtils.readVInt(in); - if (length > 0) { - this.actions = new GroomServerAction[length]; - for (int i = 0; i < length; ++i) { - GroomServerAction.ActionType actionType = WritableUtils.readEnum(in, - GroomServerAction.ActionType.class); - actions[i] = GroomServerAction.createAction(actionType); - actions[i].readFields(in); - } - } else { - this.actions = null; - } - String[] groomServerNames = WritableUtils.readCompressedStringArray(in); - String[] groomServerAddresses = WritableUtils.readCompressedStringArray(in); - groomServerPeers = new HashMap(groomServerNames.length); - - for (int i = 0; i < groomServerNames.length; i++) { - groomServerPeers.put(groomServerNames[i], groomServerAddresses[i]); - } - } -} Index: src/java/org/apache/hama/bsp/GroomServerManager.java =================================================================== --- src/java/org/apache/hama/bsp/GroomServerManager.java (revision 1152138) +++ src/java/org/apache/hama/bsp/GroomServerManager.java (working copy) @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.util.Collection; -import java.util.Map; - -import org.apache.hama.ipc.GroomProtocol; - -/** - * Manages information about the {@link GroomServer}s in the cluster - * environment. This interface is not intended to be implemented by users. - */ -interface GroomServerManager { - - /** - * Get the current status of the cluster - * @param detailed if true then report groom names as well - * @return summary of the state of the cluster - */ - ClusterStatus getClusterStatus(boolean detailed); - - /** - * Find WorkerProtocol with corresponded groom server status - * - * @param groomId The identification value of GroomServer - * @return GroomServerStatus - */ - GroomProtocol findGroomServer(GroomServerStatus status); - - /** - * Find the collection of groom servers. - * - * @return Collection of groom servers list. - */ - Collection findGroomServers(); - - /** - * Collection of GroomServerStatus as the key set. - * - * @return Collection of GroomServerStatus. - */ - Collection groomServerStatusKeySet(); - - /** - * Registers a JobInProgressListener to GroomServerManager. Therefore, - * adding a JobInProgress will trigger the jobAdded function. - * @param the JobInProgressListener listener to be added. - */ - void addJobInProgressListener(JobInProgressListener listener); - - /** - * Unregisters a JobInProgressListener to GroomServerManager. Therefore, - * the remove of a JobInProgress will trigger the jobRemoved action. - * @param the JobInProgressListener to be removed. - */ - void removeJobInProgressListener(JobInProgressListener listener); - - /** - * Current GroomServer Peers. - * @return GroomName and PeerName(host:port) in pair. - */ - Map currentGroomServerPeers(); -} Index: src/java/org/apache/hama/bsp/GroomServerAction.java =================================================================== --- src/java/org/apache/hama/bsp/GroomServerAction.java (revision 1152138) +++ src/java/org/apache/hama/bsp/GroomServerAction.java (working copy) @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; - -/** - * A generic directive from the {@link org.apache.hama.bsp.BSPMaster} - * to the {@link org.apache.hama.bsp.GroomServer} to take some 'action'. - */ -abstract class GroomServerAction implements Writable { - - /** - * Ennumeration of various 'actions' that the {@link BSPMaster} - * directs the {@link GroomServer} to perform periodically. - * - */ - public static enum ActionType { - /** Launch a new task. */ - LAUNCH_TASK, - - /** Kill a task. */ - KILL_TASK, - - /** Kill any tasks of this job and cleanup. */ - KILL_JOB, - - /** Reinitialize the groomserver. */ - REINIT_GROOM, - - /** Ask a task to save its output. */ - COMMIT_TASK - }; - - /** - * A factory-method to create objects of given {@link ActionType}. - * @param actionType the {@link ActionType} of object to create. - * @return an object of {@link ActionType}. - */ - public static GroomServerAction createAction(ActionType actionType) { - GroomServerAction action = null; - - switch (actionType) { - case LAUNCH_TASK: - { - action = new LaunchTaskAction(); - } - break; - case KILL_TASK: - { - action = new KillTaskAction(); - } - break; - case KILL_JOB: - { - action = new KillJobAction(); - } - break; - case REINIT_GROOM: - { - action = new ReinitGroomAction(); - } - break; - case COMMIT_TASK: - { - action = new CommitTaskAction(); - } - break; - } - - return action; - } - - private ActionType actionType; - - protected GroomServerAction(ActionType actionType) { - this.actionType = actionType; - } - - /** - * Return the {@link ActionType}. - * @return the {@link ActionType}. - */ - ActionType getActionType() { - return actionType; - } - - public void write(DataOutput out) throws IOException { - WritableUtils.writeEnum(out, actionType); - } - - public void readFields(DataInput in) throws IOException { - actionType = WritableUtils.readEnum(in, ActionType.class); - } -} Index: src/java/org/apache/hama/bsp/ByteMessage.java =================================================================== --- src/java/org/apache/hama/bsp/ByteMessage.java (revision 1152138) +++ src/java/org/apache/hama/bsp/ByteMessage.java (working copy) @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * A message that consists of a byte tag and a byte data. - */ -public class ByteMessage extends BSPMessage { - - private byte[] tag; - private byte[] data; - - public ByteMessage() { - super(); - } - - public ByteMessage(byte[] tag, byte[] data) { - super(); - this.tag = tag; - this.data = data; - } - - @Override - public byte[] getTag() { - return this.tag; - } - - @Override - public byte[] getData() { - return this.data; - } - - @Override - public void readFields(DataInput in) throws IOException { - this.tag = new byte[in.readInt()]; - in.readFully(tag, 0, this.tag.length); - this.data = new byte[in.readInt()]; - in.readFully(data, 0, this.data.length); - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(tag.length); - out.write(tag); - out.writeInt(data.length); - out.write(data); - } - -} Index: src/java/org/apache/hama/bsp/CommitTaskAction.java =================================================================== --- src/java/org/apache/hama/bsp/CommitTaskAction.java (revision 1152138) +++ src/java/org/apache/hama/bsp/CommitTaskAction.java (working copy) @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} - * to the {@link org.apache.hama.bsp.GroomServer} to commit the output - * of the task. - */ -class CommitTaskAction extends GroomServerAction { - private TaskAttemptID taskId; - - public CommitTaskAction() { - super(ActionType.COMMIT_TASK); - taskId = new TaskAttemptID(); - } - - public CommitTaskAction(TaskAttemptID taskId) { - super(ActionType.COMMIT_TASK); - this.taskId = taskId; - } - - public TaskAttemptID getTaskID() { - return taskId; - } - - public void write(DataOutput out) throws IOException { - taskId.write(out); - } - - public void readFields(DataInput in) throws IOException { - taskId.readFields(in); - } -} Index: src/java/org/apache/hama/bsp/RunningJob.java =================================================================== --- src/java/org/apache/hama/bsp/RunningJob.java (revision 1152138) +++ src/java/org/apache/hama/bsp/RunningJob.java (working copy) @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; - -/** - * RunningJob is the user-interface to query for details on a - * running BSP job. - * - *

- * Clients can get hold of RunningJob via the {@link BSPJobClient} - * and then query the running-job for details such as name, configuration, - * progress etc. - *

- * - * @see BSPJobClient - */ -public interface RunningJob { - /** - * Get the job identifier. - * - * @return the job identifier. - */ - public BSPJobID getID(); - - /** - * Get the name of the job. - * - * @return the name of the job. - */ - public String getJobName(); - - /** - * Get the path of the submitted job configuration. - * - * @return the path of the submitted job configuration. - */ - public String getJobFile(); - - /** - * Get the progress of the job's tasks, as a float between 0.0 and 1.0. - * When all bsp tasks have completed, the function returns 1.0. - * - * @return the progress of the job's tasks. - * @throws IOException - */ - public long progress() throws IOException; - - /** - * Check if the job is finished or not. This is a non-blocking call. - * - * @return true if the job is complete, else false. - * @throws IOException - */ - public boolean isComplete() throws IOException; - - /** - * Check if the job completed successfully. - * - * @return true if the job succeeded, else false. - * @throws IOException - */ - public boolean isSuccessful() throws IOException; - - /** - * Blocks until the job is complete. - * - * @throws IOException - */ - public void waitForCompletion() throws IOException; - - /** - * Returns the current state of the Job. {@link JobStatus} - * - * @throws IOException - */ - public int getJobState() throws IOException; - - /** - * Kill the running job. Blocks until all job tasks have been killed as well. - * If the job is no longer running, it simply returns. - * - * @throws IOException - */ - public void killJob() throws IOException; - - /** - * Kill indicated task attempt. - * - * @param taskId the id of the task to be terminated. - * @param shouldFail if true the task is failed and added to failed tasks - * list, otherwise it is just killed, w/o affecting job failure - * status. - * @throws IOException - */ - public void killTask(TaskAttemptID taskId, boolean shouldFail) - throws IOException; - - public long getSuperstepCount() throws IOException; -} Index: src/java/org/apache/hama/bsp/Schedulable.java =================================================================== --- src/java/org/apache/hama/bsp/Schedulable.java (revision 1152138) +++ src/java/org/apache/hama/bsp/Schedulable.java (working copy) @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; - -/** - * This is the class that schedules commands to GroomServer(s) - */ -public interface Schedulable{ - - /** - * Schedule job to designated GroomServer(s) immediately. - * @param job to be scheduled. - * @param statuses of GroomServer(s). - * @throws IOException - */ - void schedule(JobInProgress job, GroomServerStatus... statuses) - throws IOException; -} Index: src/java/org/apache/hama/bsp/SimpleTaskScheduler.java =================================================================== --- src/java/org/apache/hama/bsp/SimpleTaskScheduler.java (revision 1152138) +++ src/java/org/apache/hama/bsp/SimpleTaskScheduler.java (working copy) @@ -1,183 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import static java.util.concurrent.TimeUnit.SECONDS; - -import java.io.IOException; -import java.util.Collection; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hama.ipc.GroomProtocol; - -/** - * A simple task scheduler. - */ -class SimpleTaskScheduler extends TaskScheduler { - - private static final Log LOG = LogFactory.getLog(SimpleTaskScheduler.class); - - public static final String WAIT_QUEUE = "waitQueue"; - public static final String PROCESSING_QUEUE = "processingQueue"; - public static final String FINISHED_QUEUE = "finishedQueue"; - - private QueueManager queueManager; - private volatile boolean initialized; - private JobListener jobListener; - private JobProcessor jobProcessor; - - private class JobListener extends JobInProgressListener { - @Override - public void jobAdded(JobInProgress job) throws IOException { - queueManager.initJob(job); // init task - queueManager.addJob(WAIT_QUEUE, job); - } - - @Override - public void jobRemoved(JobInProgress job) throws IOException { - // queueManager.removeJob(WAIT_QUEUE, job); - queueManager.moveJob(PROCESSING_QUEUE, FINISHED_QUEUE, job); - } - } - - private class JobProcessor extends Thread implements Schedulable { - JobProcessor() { - super("JobProcess"); - } - - /** - * Main logic scheduling task to GroomServer(s). Also, it will move - * JobInProgress from Wait Queue to Processing Queue. - */ - public void run() { - if (false == initialized) { - throw new IllegalStateException("SimpleTaskScheduler initialization" - + " is not yet finished!"); - } - while (initialized) { - Queue queue = queueManager.findQueue(WAIT_QUEUE); - if (null == queue) { - LOG.error(WAIT_QUEUE + " does not exist."); - throw new NullPointerException(WAIT_QUEUE + " does not exist."); - } - // move a job from the wait queue to the processing queue - JobInProgress j = queue.removeJob(); - queueManager.addJob(PROCESSING_QUEUE, j); - // schedule - Collection glist = groomServerManager - .groomServerStatusKeySet(); - schedule(j, (GroomServerStatus[]) glist - .toArray(new GroomServerStatus[glist.size()])); - } - } - - /** - * Schedule job to designated GroomServer(s) immediately. - * - * @param Targeted GroomServer(s). - * @param Job to be scheduled. - */ - @Override - public void schedule(JobInProgress job, GroomServerStatus... statuses) { - ClusterStatus clusterStatus = groomServerManager.getClusterStatus(false); - final int numGroomServers = clusterStatus.getGroomServers(); - final ScheduledExecutorService sched = Executors - .newScheduledThreadPool(statuses.length + 5); - for (GroomServerStatus status : statuses) { - sched - .schedule(new TaskWorker(status, numGroomServers, job), 0, SECONDS); - }// for - } - } - - private class TaskWorker implements Runnable { - private final GroomServerStatus stus; - private final int groomNum; - private final JobInProgress jip; - - TaskWorker(final GroomServerStatus stus, final int num, - final JobInProgress jip) { - this.stus = stus; - this.groomNum = num; - this.jip = jip; - if (null == this.stus) - throw new NullPointerException("Target groom server is not " - + "specified."); - if (-1 == this.groomNum) - throw new IllegalArgumentException("Groom number is not specified."); - if (null == this.jip) - throw new NullPointerException("No job is specified."); - } - - public void run() { - // obtain tasks - Task t = jip.obtainNewTask(this.stus, groomNum); - - // assembly into actions - // List tasks = new ArrayList(); - if (jip.getStatus().getRunState() == JobStatus.RUNNING) { - GroomProtocol worker = groomServerManager.findGroomServer(this.stus); - try { - // dispatch() to the groom server - Directive d1 = new DispatchTasksDirective(groomServerManager - .currentGroomServerPeers(), new GroomServerAction[] { - new LaunchTaskAction(t)}); - worker.dispatch(d1); - } catch (IOException ioe) { - LOG.error("Fail to dispatch tasks to GroomServer " - + this.stus.getGroomName(), ioe); - } - } else { - LOG.warn("Currently master only shcedules job in running state. " - + "This may be refined in the future. JobId:" + jip.getJobID()); - } - } - } - - public SimpleTaskScheduler() { - this.jobListener = new JobListener(); - this.jobProcessor = new JobProcessor(); - } - - @Override - public void start() { - this.queueManager = new QueueManager(getConf()); // TODO: need factory? - this.queueManager.createFCFSQueue(WAIT_QUEUE); - this.queueManager.createFCFSQueue(PROCESSING_QUEUE); - this.queueManager.createFCFSQueue(FINISHED_QUEUE); - groomServerManager.addJobInProgressListener(this.jobListener); - this.initialized = true; - this.jobProcessor.start(); - } - - @Override - public void terminate() { - this.initialized = false; - if (null != this.jobListener) - groomServerManager.removeJobInProgressListener(this.jobListener); - } - - @Override - public Collection getJobs(String queue) { - return (queueManager.findQueue(queue)).jobs(); - // return jobQueue; - } -} Index: src/java/org/apache/hama/bsp/DirectiveHandler.java =================================================================== --- src/java/org/apache/hama/bsp/DirectiveHandler.java (revision 1152138) +++ src/java/org/apache/hama/bsp/DirectiveHandler.java (working copy) @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -/** - * A DirectiveHandler interface. - */ -public interface DirectiveHandler{ - - /** - * Handle directives on demand. - * @param directive to be handled. - */ - void handle(Directive directive) throws DirectiveException; -} Index: src/java/org/apache/hama/bsp/KillTaskAction.java =================================================================== --- src/java/org/apache/hama/bsp/KillTaskAction.java (revision 1152138) +++ src/java/org/apache/hama/bsp/KillTaskAction.java (working copy) @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} - * to the {@link org.apache.hama.bsp.GroomServer} to kill a task. - */ -class KillTaskAction extends GroomServerAction { - TaskAttemptID taskId; - - public KillTaskAction() { - super(ActionType.KILL_TASK); - taskId = new TaskAttemptID(); - } - - public KillTaskAction(TaskAttemptID killTaskId) { - super(ActionType.KILL_TASK); - this.taskId = killTaskId; - } - - public TaskAttemptID getTaskID() { - return taskId; - } - - @Override - public void write(DataOutput out) throws IOException { - taskId.write(out); - } - - @Override - public void readFields(DataInput in) throws IOException { - taskId.readFields(in); - } -} Index: src/java/org/apache/hama/bsp/GroomServerStatus.java =================================================================== --- src/java/org/apache/hama/bsp/GroomServerStatus.java (revision 1152138) +++ src/java/org/apache/hama/bsp/GroomServerStatus.java (working copy) @@ -1,227 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableFactories; -import org.apache.hadoop.io.WritableFactory; - -/** - * A GroomServerStatus is a BSP primitive. Keeps info on a BSPMaster. The - * BSPMaster maintains a set of the most recent GroomServerStatus objects for - * each unique GroomServer it knows about. - */ -public class GroomServerStatus implements Writable { - public static final Log LOG = LogFactory.getLog(GroomServerStatus.class); - - static { - WritableFactories.setFactory(GroomServerStatus.class, - new WritableFactory() { - public Writable newInstance() { - return new GroomServerStatus(); - } - }); - } - - String groomName; - String peerName; - String rpcServer; - int failures; - List taskReports; - - volatile long lastSeen; - private int maxTasks; - - public GroomServerStatus() { - // taskReports = new ArrayList(); - taskReports = new CopyOnWriteArrayList(); - } - - public GroomServerStatus(String groomName, String peerName, - List taskReports, int failures, int maxTasks) { - this(groomName, peerName, taskReports, failures, maxTasks, ""); - } - - public GroomServerStatus(String groomName, String peerName, - List taskReports, int failures, int maxTasks, String rpc) { - this.groomName = groomName; - this.peerName = peerName; - this.taskReports = new ArrayList(taskReports); - this.failures = failures; - this.maxTasks = maxTasks; - this.rpcServer = rpc; - } - - public String getGroomName() { - return groomName; - } - - /** - * The host (and port) from where the groom server can be reached. - * - * @return The groom server address in the form of "hostname:port" - */ - public String getPeerName() { - return peerName; - } - - public String getRpcServer() { - return rpcServer; - } - - /** - * Get the current tasks at the GroomServer. Tasks are tracked by a - * {@link TaskStatus} object. - * - * @return a list of {@link TaskStatus} representing the current tasks at the - * GroomServer. - */ - public List getTaskReports() { - return taskReports; - } - - public int getFailures() { - return failures; - } - - public long getLastSeen() { - return lastSeen; - } - - public void setLastSeen(long lastSeen) { - this.lastSeen = lastSeen; - } - - public int getMaxTasks() { - return maxTasks; - } - - /** - * Return the current MapTask count - */ - public int countTasks() { - int taskCount = 0; - for (Iterator it = taskReports.iterator(); it.hasNext();) { - TaskStatus ts = it.next(); - TaskStatus.State state = ts.getRunState(); - if (state == TaskStatus.State.RUNNING - || state == TaskStatus.State.UNASSIGNED) { - taskCount++; - } - } - - return taskCount; - } - - /** - * For BSPMaster to distinguish between different GroomServers, because - * BSPMaster stores using GroomServerStatus as key. - */ - @Override - public int hashCode() { - int result = 17; - result = 37 * result + groomName.hashCode(); - result = 37 * result + peerName.hashCode(); - result = 37 * result + rpcServer.hashCode(); - /* - * result = 37*result + (int)failures; result = 37*result + - * taskReports.hashCode(); result = 37*result + - * (int)(lastSeen^(lastSeen>>>32)); result = 37*result + (int)maxTasks; - */ - return result; - } - - @Override - public boolean equals(Object o) { - if (o == this) - return true; - if (null == o) - return false; - if (getClass() != o.getClass()) - return false; - - GroomServerStatus s = (GroomServerStatus) o; - if (!s.groomName.equals(groomName)) - return false; - if (!s.peerName.equals(peerName)) - return false; - if (!s.rpcServer.equals(rpcServer)) - return false; - /* - * if(s.failures != failures) return false; if(null == s.taskReports){ - * if(null != s.taskReports) return false; }else - * if(!s.taskReports.equals(taskReports)){ return false; } if(s.lastSeen != - * lastSeen) return false; if(s.maxTasks != maxTasks) return false; - */ - return true; - } - - /* - * (non-Javadoc) - * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput) - */ - @Override - public void readFields(DataInput in) throws IOException { - this.groomName = Text.readString(in); - this.peerName = Text.readString(in); - this.rpcServer = Text.readString(in); - this.failures = in.readInt(); - this.maxTasks = in.readInt(); - taskReports.clear(); - int numTasks = in.readInt(); - - TaskStatus status; - for (int i = 0; i < numTasks; i++) { - status = new TaskStatus(); - status.readFields(in); - taskReports.add(status); - } - } - - /* - * (non-Javadoc) - * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput) - */ - @Override - public void write(DataOutput out) throws IOException { - Text.writeString(out, groomName); - Text.writeString(out, peerName); - Text.writeString(out, rpcServer); - out.writeInt(failures); - out.writeInt(maxTasks); - out.writeInt(taskReports.size()); - for (TaskStatus taskStatus : taskReports) { - taskStatus.write(out); - } - } - - public Iterator taskReports() { - return taskReports.iterator(); - } -} Index: src/java/org/apache/hama/bsp/BSPJobClient.java =================================================================== --- src/java/org/apache/hama/bsp/BSPJobClient.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPJobClient.java (working copy) @@ -1,679 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Map; -import java.util.Random; - -import javax.security.auth.login.LoginException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.UnixUserGroupInformation; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.ipc.JobSubmissionProtocol; - -/** - * BSPJobClient is the primary interface for the user-job to interact with the - * BSPMaster. - * - * BSPJobClient provides facilities to submit jobs, track their progress, access - * component-tasks' reports/logs, get the BSP cluster status information etc. - */ -public class BSPJobClient extends Configured implements Tool { - private static final Log LOG = LogFactory.getLog(BSPJobClient.class); - - public static enum TaskStatusFilter { - NONE, KILLED, FAILED, SUCCEEDED, ALL - } - - private static final long MAX_JOBPROFILE_AGE = 1000 * 2; - - class NetworkedJob implements RunningJob { - JobProfile profile; - JobStatus status; - long statustime; - - public NetworkedJob(JobStatus job) throws IOException { - this.status = job; - this.profile = jobSubmitClient.getJobProfile(job.getJobID()); - this.statustime = System.currentTimeMillis(); - } - - /** - * Some methods rely on having a recent job profile object. Refresh it, if - * necessary - */ - synchronized void ensureFreshStatus() throws IOException { - if (System.currentTimeMillis() - statustime > MAX_JOBPROFILE_AGE) { - updateStatus(); - } - } - - /** - * Some methods need to update status immediately. So, refresh immediately - * - * @throws IOException - */ - synchronized void updateStatus() throws IOException { - this.status = jobSubmitClient.getJobStatus(profile.getJobID()); - this.statustime = System.currentTimeMillis(); - } - - /* - * (non-Javadoc) - * @see org.apache.hama.bsp.RunningJob#getID() - */ - @Override - public BSPJobID getID() { - return profile.getJobID(); - } - - /* - * (non-Javadoc) - * @see org.apache.hama.bsp.RunningJob#getJobName() - */ - @Override - public String getJobName() { - return profile.getJobName(); - } - - /* - * (non-Javadoc) - * @see org.apache.hama.bsp.RunningJob#getJobFile() - */ - @Override - public String getJobFile() { - return profile.getJobFile(); - } - - @Override - public long progress() throws IOException { - ensureFreshStatus(); - return status.progress(); - } - - /** - * Returns immediately whether the whole job is done yet or not. - */ - public synchronized boolean isComplete() throws IOException { - updateStatus(); - return (status.getRunState() == JobStatus.SUCCEEDED - || status.getRunState() == JobStatus.FAILED || status.getRunState() == JobStatus.KILLED); - } - - /** - * True if job completed successfully. - */ - public synchronized boolean isSuccessful() throws IOException { - updateStatus(); - return status.getRunState() == JobStatus.SUCCEEDED; - } - - public synchronized long getSuperstepCount() throws IOException { - ensureFreshStatus(); - return status.getSuperstepCount(); - } - - /** - * Blocks until the job is finished - */ - public void waitForCompletion() throws IOException { - while (!isComplete()) { - try { - Thread.sleep(5000); - } catch (InterruptedException ie) { - } - } - } - - /** - * Tells the service to get the state of the current job. - */ - public synchronized int getJobState() throws IOException { - updateStatus(); - return status.getRunState(); - } - - /** - * Tells the service to terminate the current job. - */ - public synchronized void killJob() throws IOException { - jobSubmitClient.killJob(getID()); - } - - @Override - public void killTask(TaskAttemptID taskId, boolean shouldFail) - throws IOException { - jobSubmitClient.killTask(taskId, shouldFail); - } - } - - private JobSubmissionProtocol jobSubmitClient = null; - private Path sysDir = null; - private FileSystem fs = null; - - // job files are world-wide readable and owner writable - final private static FsPermission JOB_FILE_PERMISSION = FsPermission - .createImmutable((short) 0644); // rw-r--r-- - - // job submission directory is world readable/writable/executable - final static FsPermission JOB_DIR_PERMISSION = FsPermission - .createImmutable((short) 0777); // rwx-rwx-rwx - - public BSPJobClient(Configuration conf) throws IOException { - setConf(conf); - init(conf); - } - - public BSPJobClient() { - } - - public void init(Configuration conf) throws IOException { - String masterAdress = conf.get("bsp.master.address"); - if (masterAdress != null && !masterAdress.equals("local")) { - this.jobSubmitClient = (JobSubmissionProtocol) RPC.getProxy( - JobSubmissionProtocol.class, JobSubmissionProtocol.versionID, - BSPMaster.getAddress(conf), conf, - NetUtils.getSocketFactory(conf, JobSubmissionProtocol.class)); - } else { - LOG.debug("Using local BSP runner."); - this.jobSubmitClient = new LocalBSPRunner(conf); - } - } - - /** - * Close the JobClient. - */ - public synchronized void close() throws IOException { - RPC.stopProxy(jobSubmitClient); - } - - /** - * Get a filesystem handle. We need this to prepare jobs for submission to the - * BSP system. - * - * @return the filesystem handle. - */ - public synchronized FileSystem getFs() throws IOException { - if (this.fs == null) { - Path sysDir = getSystemDir(); - this.fs = sysDir.getFileSystem(getConf()); - } - return fs; - } - - /** - * Gets the jobs that are submitted. - * - * @return array of {@link JobStatus} for the submitted jobs. - * @throws IOException - */ - public JobStatus[] getAllJobs() throws IOException { - return jobSubmitClient.getAllJobs(); - } - - /** - * Gets the jobs that are not completed and not failed. - * - * @return array of {@link JobStatus} for the running/to-be-run jobs. - * @throws IOException - */ - public JobStatus[] jobsToComplete() throws IOException { - return jobSubmitClient.jobsToComplete(); - } - - private UnixUserGroupInformation getUGI(Configuration conf) - throws IOException { - UnixUserGroupInformation ugi = null; - try { - ugi = UnixUserGroupInformation.login(conf, true); - } catch (LoginException e) { - throw (IOException) (new IOException( - "Failed to get the current user's information.").initCause(e)); - } - return ugi; - } - - /** - * Submit a job to the BSP system. This returns a handle to the - * {@link RunningJob} which can be used to track the running-job. - * - * @param job the job configuration. - * @return a handle to the {@link RunningJob} which can be used to track the - * running-job. - * @throws FileNotFoundException - * @throws IOException - */ - public RunningJob submitJob(BSPJob job) throws FileNotFoundException, - IOException { - return submitJobInternal(job); - } - - static Random r = new Random(); - - public RunningJob submitJobInternal(BSPJob job) throws IOException { - BSPJobID jobId = jobSubmitClient.getNewJobId(); - - Path submitJobDir = new Path(getSystemDir(), "submit_" - + Integer.toString(Math.abs(r.nextInt()), 36)); - Path submitJarFile = new Path(submitJobDir, "job.jar"); - Path submitJobFile = new Path(submitJobDir, "job.xml"); - LOG.debug("BSPJobClient.submitJobDir: " + submitJobDir); - - /* - * set this user's id in job configuration, so later job files can be - * accessed using this user's id - */ - UnixUserGroupInformation ugi = getUGI(job.getConf()); - - ClusterStatus clusterStatus = getClusterStatus(true); - - // check the number of BSP tasks - int tasks = job.getNumBspTask(); - int maxTasks = clusterStatus.getMaxTasks(); - - if (tasks <= 0 || tasks > maxTasks) { - LOG.warn("The number of tasks you've entered was invalid. Using default value of " - + maxTasks + "!"); - job.setNumBspTask(maxTasks); - } - - // Create a number of filenames in the BSPMaster's fs namespace - FileSystem fs = getFs(); - fs.delete(submitJobDir, true); - submitJobDir = fs.makeQualified(submitJobDir); - submitJobDir = new Path(submitJobDir.toUri().getPath()); - FsPermission bspSysPerms = new FsPermission(JOB_DIR_PERMISSION); - FileSystem.mkdirs(fs, submitJobDir, bspSysPerms); - fs.mkdirs(submitJobDir); - short replication = (short) job.getInt("bsp.submit.replication", 10); - - String originalJarPath = job.getJar(); - - if (originalJarPath != null) { // copy jar to BSPMaster's fs - // use jar name if job is not named. - if ("".equals(job.getJobName())) { - job.setJobName(new Path(originalJarPath).getName()); - } - job.setJar(submitJarFile.toString()); - fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile); - - fs.setReplication(submitJarFile, replication); - fs.setPermission(submitJarFile, new FsPermission(JOB_FILE_PERMISSION)); - } else { - LOG.warn("No job jar file set. User classes may not be found. " - + "See BSPJob#setJar(String) or check Your jar file."); - } - - // Set the user's name and working directory - job.setUser(ugi.getUserName()); - if (ugi.getGroupNames().length > 0) { - job.set("group.name", ugi.getGroupNames()[0]); - } - if (job.getWorkingDirectory() == null) { - job.setWorkingDirectory(fs.getWorkingDirectory()); - } - - // Write job file to BSPMaster's fs - FSDataOutputStream out = FileSystem.create(fs, submitJobFile, - new FsPermission(JOB_FILE_PERMISSION)); - - try { - job.writeXml(out); - } finally { - out.close(); - } - - // - // Now, actually submit the job (using the submit name) - // - JobStatus status = jobSubmitClient.submitJob(jobId, - submitJobFile.toString()); - if (status != null) { - return new NetworkedJob(status); - } else { - throw new IOException("Could not launch job"); - } - } - - /** - * Monitor a job and print status in real-time as progress is made and tasks - * fail. - * - * @param job - * @param info - * @return true, if job is successful - * @throws IOException - * @throws InterruptedException - */ - public boolean monitorAndPrintJob(BSPJob job, RunningJob info) - throws IOException, InterruptedException { - - String lastReport = null; - LOG.info("Running job: " + info.getID()); - - while (!job.isComplete()) { - Thread.sleep(3000); - long step = job.progress(); - String report = "Current supersteps number: " + step; - - if (!report.equals(lastReport)) { - LOG.info(report); - lastReport = report; - } - } - - LOG.info("The total number of supersteps: " + info.getSuperstepCount()); - return job.isSuccessful(); - } - - /** - * Grab the bspmaster system directory path where job-specific files are to be - * placed. - * - * @return the system directory where job-specific files are to be placed. - */ - public Path getSystemDir() { - if (sysDir == null) { - sysDir = new Path(jobSubmitClient.getSystemDir()); - } - return sysDir; - } - - public static void runJob(BSPJob job) throws FileNotFoundException, - IOException { - BSPJobClient jc = new BSPJobClient(job.getConf()); - - if (job.getNumBspTask() == 0 - || job.getNumBspTask() > jc.getClusterStatus(false).getMaxTasks()) { - job.setNumBspTask(jc.getClusterStatus(false).getMaxTasks()); - } - - RunningJob running = jc.submitJobInternal(job); - BSPJobID jobId = running.getID(); - LOG.info("Running job: " + jobId.toString()); - - while (true) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - - if (running.isComplete()) { - break; - } - - running = jc.getJob(jobId); - } - - LOG.info("Job complete: " + jobId); - LOG.info("The total number of supersteps: " + running.getSuperstepCount()); - - // TODO if error found, kill job - // running.killJob(); - jc.close(); - } - - /** - * Get an RunningJob object to track an ongoing job. Returns null if the id - * does not correspond to any known job. - * - * @throws IOException - */ - private RunningJob getJob(BSPJobID jobId) throws IOException { - JobStatus status = jobSubmitClient.getJobStatus(jobId); - if (status != null) { - return new NetworkedJob(status); - } else { - return null; - } - } - - /** - * Get status information about the BSP cluster - * - * @param detailed if true then get a detailed status including the - * groomserver names - * - * @return the status information about the BSP cluster as an object of - * {@link ClusterStatus}. - * - * @throws IOException - */ - public ClusterStatus getClusterStatus(boolean detailed) throws IOException { - return jobSubmitClient.getClusterStatus(detailed); - } - - @Override - public int run(String[] args) throws Exception { - int exitCode = -1; - if (args.length < 1) { - displayUsage(""); - return exitCode; - } - - // process arguments - String cmd = args[0]; - boolean listJobs = false; - boolean listAllJobs = false; - boolean listActiveGrooms = false; - boolean killJob = false; - boolean submitJob = false; - boolean getStatus = false; - String submitJobFile = null; - String jobid = null; - - HamaConfiguration conf = new HamaConfiguration(getConf()); - init(conf); - - if ("-list".equals(cmd)) { - if (args.length != 1 && !(args.length == 2 && "all".equals(args[1]))) { - displayUsage(cmd); - return exitCode; - } - if (args.length == 2 && "all".equals(args[1])) { - listAllJobs = true; - } else { - listJobs = true; - } - } else if ("-list-active-grooms".equals(cmd)) { - if (args.length != 1) { - displayUsage(cmd); - return exitCode; - } - listActiveGrooms = true; - } else if ("-submit".equals(cmd)) { - if (args.length == 1) { - displayUsage(cmd); - return exitCode; - } - - submitJob = true; - submitJobFile = args[1]; - } else if ("-kill".equals(cmd)) { - if (args.length == 1) { - displayUsage(cmd); - return exitCode; - } - killJob = true; - jobid = args[1]; - - } else if ("-status".equals(cmd)) { - if (args.length != 2) { - displayUsage(cmd); - return exitCode; - } - jobid = args[1]; - getStatus = true; - - // TODO Later, below functions should be implemented - // with the Fault Tolerant mechanism. - } else if ("-list-attempt-ids".equals(cmd)) { - System.out.println("This function is not implemented yet."); - return exitCode; - } else if ("-kill-task".equals(cmd)) { - System.out.println("This function is not implemented yet."); - return exitCode; - } else if ("-fail-task".equals(cmd)) { - System.out.println("This function is not implemented yet."); - return exitCode; - } - - BSPJobClient jc = new BSPJobClient(new HamaConfiguration()); - if (listJobs) { - listJobs(); - exitCode = 0; - } else if (listAllJobs) { - listAllJobs(); - exitCode = 0; - } else if (listActiveGrooms) { - listActiveGrooms(); - exitCode = 0; - } else if (submitJob) { - HamaConfiguration tConf = new HamaConfiguration(new Path(submitJobFile)); - RunningJob job = jc.submitJob(new BSPJob(tConf)); - System.out.println("Created job " + job.getID().toString()); - } else if (killJob) { - RunningJob job = jc.getJob(BSPJobID.forName(jobid)); - if (job == null) { - System.out.println("Could not find job " + jobid); - } else { - job.killJob(); - System.out.println("Killed job " + jobid); - } - exitCode = 0; - } else if (getStatus) { - RunningJob job = jc.getJob(BSPJobID.forName(jobid)); - if (job == null) { - System.out.println("Could not find job " + jobid); - } else { - JobStatus jobStatus = jobSubmitClient.getJobStatus(job.getID()); - System.out.println("Job name: " + job.getJobName()); - System.out.printf("States are:\n\tRunning : 1\tSucceded : 2" - + "\tFailed : 3\tPrep : 4\n"); - System.out.printf("%s\t%d\t%d\t%s\n", jobStatus.getJobID(), - jobStatus.getRunState(), jobStatus.getStartTime(), - jobStatus.getUsername()); - - exitCode = 0; - } - } - - return 0; - } - - /** - * Display usage of the command-line tool and terminate execution - */ - private void displayUsage(String cmd) { - String prefix = "Usage: hama job "; - String taskStates = "running, completed"; - if ("-submit".equals(cmd)) { - System.err.println(prefix + "[" + cmd + " ]"); - } else if ("-status".equals(cmd) || "-kill".equals(cmd)) { - System.err.println(prefix + "[" + cmd + " ]"); - } else if ("-list".equals(cmd)) { - System.err.println(prefix + "[" + cmd + " [all]]"); - } else if ("-kill-task".equals(cmd) || "-fail-task".equals(cmd)) { - System.err.println(prefix + "[" + cmd + " ]"); - } else if ("-list-active-grooms".equals(cmd)) { - System.err.println(prefix + "[" + cmd + "]"); - } else if ("-list-attempt-ids".equals(cmd)) { - System.err.println(prefix + "[" + cmd + " ]. " - + "Valid values for are " + taskStates); - } else { - System.err.printf(prefix + " \n"); - System.err.printf("\t[-submit ]\n"); - System.err.printf("\t[-status ]\n"); - System.err.printf("\t[-kill ]\n"); - System.err.printf("\t[-list [all]]\n"); - System.err.printf("\t[-list-active-grooms]\n"); - System.err.println("\t[-list-attempt-ids " + "]\n"); - System.err.printf("\t[-kill-task ]\n"); - System.err.printf("\t[-fail-task ]\n\n"); - } - } - - /** - * Dump a list of currently running jobs - * - * @throws IOException - */ - private void listJobs() throws IOException { - JobStatus[] jobs = jobsToComplete(); - if (jobs == null) - jobs = new JobStatus[0]; - - System.out.printf("%d jobs currently running\n", jobs.length); - displayJobList(jobs); - } - - /** - * Dump a list of all jobs submitted. - * - * @throws IOException - */ - private void listAllJobs() throws IOException { - JobStatus[] jobs = getAllJobs(); - if (jobs == null) - jobs = new JobStatus[0]; - System.out.printf("%d jobs submitted\n", jobs.length); - System.out.printf("States are:\n\tRunning : 1\tSucceded : 2" - + "\tFailed : 3\tPrep : 4\n"); - displayJobList(jobs); - } - - void displayJobList(JobStatus[] jobs) { - System.out.printf("JobId\tState\tStartTime\tUserName\n"); - for (JobStatus job : jobs) { - System.out.printf("%s\t%d\t%d\t%s\n", job.getJobID(), job.getRunState(), - job.getStartTime(), job.getUsername()); - } - } - - /** - * Display the list of active groom servers - */ - private void listActiveGrooms() throws IOException { - ClusterStatus c = jobSubmitClient.getClusterStatus(true); - Map grooms = c.getActiveGroomNames(); - for (String groomName : grooms.keySet()) { - System.out.println(groomName); - } - } - - /** - */ - public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new BSPJobClient(), args); - System.exit(res); - } -} Index: src/java/org/apache/hama/bsp/TaskAttemptID.java =================================================================== --- src/java/org/apache/hama/bsp/TaskAttemptID.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskAttemptID.java (working copy) @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * TaskAttemptID is a unique identifier for a task attempt. - */ -public class TaskAttemptID extends ID { - protected static final String ATTEMPT = "attempt"; - private TaskID taskId; - - public TaskAttemptID(TaskID taskId, int id) { - super(id); - if (taskId == null) { - throw new IllegalArgumentException("taskId cannot be null"); - } - this.taskId = taskId; - } - - public TaskAttemptID(String jtIdentifier, int jobId, int taskId, int id) { - this(new TaskID(jtIdentifier, jobId, taskId), id); - } - - public TaskAttemptID() { - taskId = new TaskID(); - } - - public BSPJobID getJobID() { - return taskId.getJobID(); - } - - public TaskID getTaskID() { - return taskId; - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) - return false; - - TaskAttemptID that = (TaskAttemptID) o; - return this.taskId.equals(that.taskId); - } - - protected StringBuilder appendTo(StringBuilder builder) { - return taskId.appendTo(builder).append(SEPARATOR).append(id); - } - - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - taskId.readFields(in); - } - - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - taskId.write(out); - } - - @Override - public int hashCode() { - return taskId.hashCode() * 5 + id; - } - - @Override - public int compareTo(ID o) { - TaskAttemptID that = (TaskAttemptID) o; - int tipComp = this.taskId.compareTo(that.taskId); - if (tipComp == 0) { - return this.id - that.id; - } else - return tipComp; - } - - @Override - public String toString() { - return appendTo(new StringBuilder(ATTEMPT)).toString(); - } - - public static TaskAttemptID forName(String str) - throws IllegalArgumentException { - if (str == null) - return null; - try { - String[] parts = str.split(Character.toString(SEPARATOR)); - if (parts.length == 5) { - if (parts[0].equals(ATTEMPT)) { - return new TaskAttemptID(parts[1], Integer.parseInt(parts[2]), - Integer.parseInt(parts[3]), Integer.parseInt(parts[4])); - } - } - } catch (Exception ex) { - // fall below - } - throw new IllegalArgumentException("TaskAttemptId string : " + str - + " is not properly formed"); - } -} Index: src/java/org/apache/hama/bsp/package.html =================================================================== --- src/java/org/apache/hama/bsp/package.html (revision 1152138) +++ src/java/org/apache/hama/bsp/package.html (working copy) @@ -1,23 +0,0 @@ - - - - - -BSP computing framework. - - Index: src/java/org/apache/hama/bsp/BSPJob.java =================================================================== --- src/java/org/apache/hama/bsp/BSPJob.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPJob.java (working copy) @@ -1,215 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; -import java.net.URL; -import java.net.URLDecoder; -import java.util.Enumeration; - -import org.apache.hadoop.fs.Path; -import org.apache.hama.HamaConfiguration; - -/** - * A BSP job configuration. - * - * BSPJob is the primary interface for a user to describe a BSP job to the Hama - * BSP framework for execution. - */ -public class BSPJob extends BSPJobContext { - public static enum JobState { - DEFINE, RUNNING - }; - - private JobState state = JobState.DEFINE; - private BSPJobClient jobClient; - private RunningJob info; - - public BSPJob() throws IOException { - this(new HamaConfiguration()); - } - - public BSPJob(HamaConfiguration conf) throws IOException { - super(conf, null); - jobClient = new BSPJobClient(conf); - } - - public BSPJob(HamaConfiguration conf, String jobName) throws IOException { - this(conf); - setJobName(jobName); - } - - public BSPJob(BSPJobID jobID, String jobFile) throws IOException { - super(new Path(jobFile), jobID); - } - - public BSPJob(HamaConfiguration conf, Class exampleClass) - throws IOException { - this(conf); - setJarByClass(exampleClass); - } - - public BSPJob(HamaConfiguration conf, int numPeer) { - super(conf, null); - this.setNumBspTask(numPeer); - } - - private void ensureState(JobState state) throws IllegalStateException { - if (state != this.state) { - throw new IllegalStateException("Job in state " + this.state - + " instead of " + state); - } - } - - // ///////////////////////////////////// - // Setter for Job Submission - // ///////////////////////////////////// - public void setWorkingDirectory(Path dir) throws IOException { - ensureState(JobState.DEFINE); - dir = new Path(getWorkingDirectory(), dir); - conf.set(WORKING_DIR, dir.toString()); - } - - /** - * Set the BSP algorithm class for the job. - * - * @param cls - * @throws IllegalStateException - */ - public void setBspClass(Class cls) - throws IllegalStateException { - ensureState(JobState.DEFINE); - conf.setClass(WORK_CLASS_ATTR, cls, BSP.class); - } - - @SuppressWarnings("unchecked") - public Class getBspClass() { - return (Class) conf.getClass(WORK_CLASS_ATTR, BSP.class); - } - - public void setJar(String jar) { - conf.set("bsp.jar", jar); - } - - public void setJarByClass(Class cls) { - String jar = findContainingJar(cls); - if (jar != null) { - conf.set("bsp.jar", jar); - } - } - - private static String findContainingJar(Class my_class) { - ClassLoader loader = my_class.getClassLoader(); - String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; - try { - for (Enumeration itr = loader.getResources(class_file); itr - .hasMoreElements();) { - - URL url = itr.nextElement(); - if ("jar".equals(url.getProtocol())) { - String toReturn = url.getPath(); - if (toReturn.startsWith("file:")) { - toReturn = toReturn.substring("file:".length()); - } - toReturn = URLDecoder.decode(toReturn, "UTF-8"); - return toReturn.replaceAll("!.*$", ""); - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return null; - } - - public void setJobName(String name) throws IllegalStateException { - ensureState(JobState.DEFINE); - conf.set("bsp.job.name", name); - } - - public void setInputPath(HamaConfiguration conf, Path iNPUTPATH) { - - } - - public void setUser(String user) { - conf.set("user.name", user); - } - - // ///////////////////////////////////// - // Methods for Job Control - // ///////////////////////////////////// - public long progress() throws IOException { - ensureState(JobState.RUNNING); - return info.progress(); - } - - public boolean isComplete() throws IOException { - ensureState(JobState.RUNNING); - return info.isComplete(); - } - - public boolean isSuccessful() throws IOException { - ensureState(JobState.RUNNING); - return info.isSuccessful(); - } - - public void killJob() throws IOException { - ensureState(JobState.RUNNING); - info.killJob(); - } - - public void killTask(TaskAttemptID taskId) throws IOException { - ensureState(JobState.RUNNING); - info.killTask(taskId, false); - } - - public void failTask(TaskAttemptID taskId) throws IOException { - ensureState(JobState.RUNNING); - info.killTask(taskId, true); - } - - public void submit() throws IOException, InterruptedException { - ensureState(JobState.DEFINE); - info = jobClient.submitJobInternal(this); - state = JobState.RUNNING; - } - - public boolean waitForCompletion(boolean verbose) throws IOException, - InterruptedException, ClassNotFoundException { - if (state == JobState.DEFINE) { - submit(); - } - if (verbose) { - jobClient.monitorAndPrintJob(this, info); - } else { - info.waitForCompletion(); - } - return isSuccessful(); - } - - public void set(String name, String value) { - conf.set(name, value); - } - - public void setNumBspTask(int tasks) { - conf.setInt("bsp.peers.num", tasks); - } - - public int getNumBspTask() { - return conf.getInt("bsp.peers.num", 0); - } -} Index: src/java/org/apache/hama/bsp/PeerNames.java =================================================================== --- src/java/org/apache/hama/bsp/PeerNames.java (revision 1152138) +++ src/java/org/apache/hama/bsp/PeerNames.java (working copy) @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; - -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; - -/** - * - */ -public class PeerNames implements Writable { - Collection allPeers; - - public PeerNames() { - this.allPeers = new ArrayList(); - } - - public PeerNames(Collection allPeers) { - this.allPeers = allPeers; - } - - public Collection getAllPeerNames() { - return allPeers; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(allPeers.size()); - for (String peerName : allPeers) { - Text.writeString(out, peerName); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - int peersNum = in.readInt(); - for (int i = 0; i < peersNum; i++) { - allPeers.add(Text.readString(in)); - } - } - -} Index: src/java/org/apache/hama/bsp/TaskID.java =================================================================== --- src/java/org/apache/hama/bsp/TaskID.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskID.java (working copy) @@ -1,122 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.text.NumberFormat; - -/** - * TaskID represents the immutable and unique identifier for a BSP Task. - */ -public class TaskID extends ID { - protected static final String TASK = "task"; - protected static final NumberFormat idFormat = NumberFormat.getInstance(); - static { - idFormat.setGroupingUsed(false); - idFormat.setMinimumIntegerDigits(6); - } - - private BSPJobID jobId; - - public TaskID(BSPJobID jobId, int id) { - super(id); - if (jobId == null) { - throw new IllegalArgumentException("jobId cannot be null"); - } - this.jobId = jobId; - } - - public TaskID(String jtIdentifier, int jobId, int id) { - this(new BSPJobID(jtIdentifier, jobId), id); - } - - public TaskID() { - jobId = new BSPJobID(); - } - - /** Returns the {@link BSPJobID} object that this tip belongs to */ - public BSPJobID getJobID() { - return jobId; - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) - return false; - - TaskID that = (TaskID) o; - return this.jobId.equals(that.jobId); - } - - @Override - public int compareTo(ID o) { - TaskID that = (TaskID) o; - int jobComp = this.jobId.compareTo(that.jobId); - if (jobComp == 0) { - return this.id - that.id; - } else { - return jobComp; - } - } - - @Override - public String toString() { - return appendTo(new StringBuilder(TASK)).toString(); - } - - protected StringBuilder appendTo(StringBuilder builder) { - return jobId.appendTo(builder).append(SEPARATOR) - .append(idFormat.format(id)); - } - - @Override - public int hashCode() { - return jobId.hashCode() * 524287 + id; - } - - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - jobId.readFields(in); - } - - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - jobId.write(out); - } - - public static TaskID forName(String str) throws IllegalArgumentException { - if (str == null) - return null; - try { - String[] parts = str.split("_"); - if (parts.length == 5) { - if (parts[0].equals(TASK)) { - return new TaskID(parts[1], Integer.parseInt(parts[2]), Integer - .parseInt(parts[4])); - } - } - } catch (Exception ex) { - } - throw new IllegalArgumentException("TaskId string : " + str - + " is not properly formed"); - } -} Index: src/java/org/apache/hama/bsp/BSPPeerInterface.java =================================================================== --- src/java/org/apache/hama/bsp/BSPPeerInterface.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPPeerInterface.java (working copy) @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.Closeable; -import java.io.IOException; - -import org.apache.hama.Constants; -import org.apache.hama.ipc.HamaRPCProtocolVersion; -import org.apache.zookeeper.KeeperException; - -/** - * BSP communication interface. - */ -public interface BSPPeerInterface extends HamaRPCProtocolVersion, Closeable, - Constants { - - /** - * Send a data with a tag to another BSPSlave corresponding to hostname. - * Messages sent by this method are not guaranteed to be received in a sent - * order. - * - * @param peerName - * @param msg - * @throws IOException - */ - public void send(String peerName, BSPMessage msg) throws IOException; - - /** - * Puts a message to local queue. - * - * @param msg - * @throws IOException - */ - public void put(BSPMessage msg) throws IOException; - - /** - * Puts a bundle of messages to local queue. - * - * @param messages - * @throws IOException - */ - public void put(BSPMessageBundle messages) throws IOException; - - /** - * @return A message from the peer's received messages queue (a FIFO). - * @throws IOException - */ - public BSPMessage getCurrentMessage() throws IOException; - - /** - * @return The number of messages in the peer's received messages queue. - */ - public int getNumCurrentMessages(); - - /** - * Barrier Synchronization. - * - * Sends all the messages in the outgoing message queues to the corresponding - * remote peers. - * - * @throws InterruptedException - * @throws KeeperException - */ - public void sync() throws IOException, KeeperException, InterruptedException; - - /** - * @return the count of current super-step - */ - public long getSuperstepCount(); - - /** - * @return The name of this peer in the format "hostname:port". - */ - public String getPeerName(); - - /** - * @return The names of all the peers executing tasks from the same job - * (including this peer). - */ - public String[] getAllPeerNames(); - - /** - * Clears all queues entries. - */ - public void clear(); -} Index: src/java/org/apache/hama/bsp/JobChangeEvent.java =================================================================== --- src/java/org/apache/hama/bsp/JobChangeEvent.java (revision 1152138) +++ src/java/org/apache/hama/bsp/JobChangeEvent.java (working copy) @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -/** - * {@link JobChangeEvent} is used to capture state changes in a job. A job can - * change its state w.r.t priority, progress, run-state etc. - */ -abstract class JobChangeEvent { - private JobInProgress jip; - - JobChangeEvent(JobInProgress jip) { - this.jip = jip; - } - - /** - * Get the job object for which the change is reported - */ - JobInProgress getJobInProgress() { - return jip; - } -} Index: src/java/org/apache/hama/bsp/BSPMessageBundle.java =================================================================== --- src/java/org/apache/hama/bsp/BSPMessageBundle.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPMessageBundle.java (working copy) @@ -1,124 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map.Entry; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.util.ReflectionUtils; - -/** - * BSPMessageBundle stores a group of BSPMessages so that they can be sent in - * batch rather than individually. - * - */ -public class BSPMessageBundle implements Writable { - - public static final Log LOG = LogFactory.getLog(BSPMessageBundle.class); - - private HashMap> messages = new HashMap>(); - private HashMap> classCache = new HashMap>(); - - public BSPMessageBundle() { - } - - /** - * Add message to this bundle. - * - * @param message BSPMessage to add. - */ - public void addMessage(BSPMessage message) { - String className = message.getClass().getName(); - if (!messages.containsKey(className)) { - // use linked list because we're just iterating over them - LinkedList list = new LinkedList(); - list.add(message); - messages.put(className, list); - } else { - messages.get(className).add(message); - } - } - - public List getMessages() { - // here we use an arraylist, because we know the size and outside may need - // random access - List mergeList = new ArrayList(messages.size()); - for (LinkedList c : messages.values()) { - mergeList.addAll(c); - } - return mergeList; - } - - public void write(DataOutput out) throws IOException { - // writes the k/v mapping size - out.writeInt(messages.size()); - if (messages.size() > 0) { - for (Entry> entry : messages.entrySet()) { - out.writeUTF(entry.getKey()); - LinkedList messageList = entry.getValue(); - out.writeInt(messageList.size()); - for (BSPMessage msg : messageList) { - msg.write(out); - } - } - } - } - - @SuppressWarnings("unchecked") - public void readFields(DataInput in) throws IOException { - if (messages == null) { - messages = new HashMap>(); - } - int numMessages = in.readInt(); - if (numMessages > 0) { - for (int entries = 0; entries < numMessages; entries++) { - String className = in.readUTF(); - int size = in.readInt(); - LinkedList msgList = new LinkedList(); - messages.put(className, msgList); - - Class clazz = null; - if ((clazz = classCache.get(className)) == null) { - try { - clazz = (Class) Class.forName(className); - classCache.put(className, clazz); - } catch (ClassNotFoundException e) { - LOG.error("Class was not found.",e); - } - } - - for (int i = 0; i < size; i++) { - BSPMessage msg = ReflectionUtils.newInstance(clazz, null); - msg.readFields(in); - msgList.add(msg); - } - - } - } - } - -} Index: src/java/org/apache/hama/bsp/BSP.java =================================================================== --- src/java/org/apache/hama/bsp/BSP.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSP.java (working copy) @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -/** - * This class provides an abstract implementation of the BSP interface. - */ -public abstract class BSP implements BSPInterface { -} Index: src/java/org/apache/hama/bsp/KillJobAction.java =================================================================== --- src/java/org/apache/hama/bsp/KillJobAction.java (revision 1152138) +++ src/java/org/apache/hama/bsp/KillJobAction.java (working copy) @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.Text; - -/** - * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the - * {@link org.apache.hama.bsp.GroomServer} to kill the task of a job and cleanup - * resources. - */ -class KillJobAction extends GroomServerAction { - String jobId; - - public KillJobAction() { - super(ActionType.KILL_JOB); - jobId = new String(); - } - - public KillJobAction(String killJobId) { - super(ActionType.KILL_JOB); - this.jobId = killJobId; - } - - public String getJobID() { - return jobId; - } - - @Override - public void write(DataOutput out) throws IOException { - Text.writeString(out, jobId); - } - - @Override - public void readFields(DataInput in) throws IOException { - jobId = Text.readString(in); - } - -} Index: src/java/org/apache/hama/bsp/Queue.java =================================================================== --- src/java/org/apache/hama/bsp/Queue.java (revision 1152138) +++ src/java/org/apache/hama/bsp/Queue.java (working copy) @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.util.Collection; - -/** - * Job Queue interface. - * - * @param - */ -public interface Queue{ - - /** - * The queue name. - * @return the name of current queue. - */ - String getName(); - - /** - * Add a job to a queue. - * @param job to be added to the queue. - */ - void addJob(T job); - - /** - * Remove a job from the queue. - * @param job to be removed from the queue. - */ - void removeJob(T job); - - /** - * Get a job - * @return job that is removed from the queue. - */ - T removeJob(); - - /** - * Return all data stored in this queue. - * @return Collection of jobs. - */ - public Collection jobs(); - -} Index: src/java/org/apache/hama/bsp/LocalBSPRunner.java =================================================================== --- src/java/org/apache/hama/bsp/LocalBSPRunner.java (revision 1152138) +++ src/java/org/apache/hama/bsp/LocalBSPRunner.java (working copy) @@ -1,356 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hama.bsp.BSPMaster.State; -import org.apache.hama.ipc.JobSubmissionProtocol; -import org.apache.zookeeper.KeeperException; - -/** - * A multithreaded local BSP runner that can be used for debugging BSP's. It - * uses the working directory "/user/hama/bsp/" and starts runners based on the - * number of the machines core. - * - */ -public class LocalBSPRunner implements JobSubmissionProtocol { - public static final Log LOG = LogFactory.getLog(LocalBSPRunner.class); - - private static final String IDENTIFIER = "localrunner"; - private static String WORKING_DIR = "/user/hama/bsp/"; - protected static volatile ThreadPoolExecutor threadPool; - protected static int threadPoolSize; - protected static final LinkedList> futureList = new LinkedList>(); - protected static CyclicBarrier barrier; - - static { - threadPoolSize = Runtime.getRuntime().availableProcessors(); - barrier = new CyclicBarrier(threadPoolSize); - threadPool = (ThreadPoolExecutor) Executors - .newFixedThreadPool(threadPoolSize); - } - - protected HashMap localGrooms = new HashMap(); - protected String jobFile; - protected String jobName; - - protected JobStatus currentJobStatus; - - protected Configuration conf; - protected FileSystem fs; - - public LocalBSPRunner(Configuration conf) throws IOException { - super(); - this.conf = conf; - this.fs = FileSystem.get(conf); - String path = conf.get("bsp.local.dir"); - if (path != null && !path.isEmpty()) - WORKING_DIR = path; - - threadPoolSize = conf.getInt("bsp.local.tasks.maximum", 20); - threadPool = (ThreadPoolExecutor) Executors - .newFixedThreadPool(threadPoolSize); - barrier = new CyclicBarrier(threadPoolSize); - - for (int i = 0; i < threadPoolSize; i++) { - String name = IDENTIFIER + " " + i; - localGrooms.put(name, new LocalGroom(name)); - } - - } - - @Override - public long getProtocolVersion(String protocol, long clientVersion) - throws IOException { - return 3; - } - - @Override - public BSPJobID getNewJobId() throws IOException { - return new BSPJobID(IDENTIFIER, 1); - } - - @Override - public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException { - this.jobFile = jobFile; - BSPJob job = new BSPJob(jobID, jobFile); - job.setNumBspTask(threadPoolSize); - this.jobName = job.getJobName(); - currentJobStatus = new JobStatus(jobID, System.getProperty("user.name"), 0, - JobStatus.RUNNING); - for (int i = 0; i < threadPoolSize; i++) { - String name = IDENTIFIER + " " + i; - LocalGroom localGroom = new LocalGroom(name); - localGrooms.put(name, localGroom); - futureList.add(threadPool.submit(new BSPRunner(conf, job, ReflectionUtils - .newInstance(job.getBspClass(), conf), localGroom))); - } - new Thread(new ThreadObserver(currentJobStatus)).start(); - return currentJobStatus; - } - - @Override - public ClusterStatus getClusterStatus(boolean detailed) throws IOException { - Map map = new HashMap(); - for (Entry entry : localGrooms.entrySet()) { - map.put(entry.getKey(), entry.getValue().getPeerName()); - } - return new ClusterStatus(map, threadPoolSize, threadPoolSize, State.RUNNING); - } - - @Override - public JobProfile getJobProfile(BSPJobID jobid) throws IOException { - return new JobProfile(System.getProperty("user.name"), jobid, jobFile, - jobName); - } - - @Override - public JobStatus getJobStatus(BSPJobID jobid) throws IOException { - if (currentJobStatus == null) { - currentJobStatus = new JobStatus(jobid, System.getProperty("user.name"), - 0L, JobStatus.RUNNING); - } - return currentJobStatus; - } - - @Override - public String getFilesystemName() throws IOException { - return fs.getUri().toString(); - } - - @Override - public JobStatus[] jobsToComplete() throws IOException { - return null; - } - - @Override - public JobStatus[] getAllJobs() throws IOException { - return null; - } - - @Override - public String getSystemDir() { - return WORKING_DIR; - } - - @Override - public void killJob(BSPJobID jobid) throws IOException { - return; - } - - @Override - public boolean killTask(TaskAttemptID taskId, boolean shouldFail) - throws IOException { - return false; - } - - // this class will spawn a new thread and executes the BSP - class BSPRunner implements Callable { - - Configuration conf; - BSPJob job; - BSP bsp; - LocalGroom groom; - - public BSPRunner(Configuration conf, BSPJob job, BSP bsp, LocalGroom groom) { - super(); - this.conf = conf; - this.job = job; - this.bsp = bsp; - this.groom = groom; - } - - public void run() { - bsp.setConf(conf); - try { - bsp.bsp(groom); - } catch (Exception e) { - LOG.error("Exception during BSP execution!", e); - } - } - - @Override - public BSP call() throws Exception { - run(); - return bsp; - } - } - - // this thread observes the status of the runners. - class ThreadObserver implements Runnable { - - JobStatus status; - - public ThreadObserver(JobStatus currentJobStatus) { - this.status = currentJobStatus; - } - - @Override - public void run() { - boolean success = true; - for (Future future : futureList) { - try { - future.get(); - } catch (InterruptedException e) { - LOG.error("Exception during BSP execution!", e); - success = false; - } catch (ExecutionException e) { - LOG.error("Exception during BSP execution!", e); - success = false; - } - } - if (success) { - currentJobStatus.setState(JobStatus.State.SUCCEEDED); - currentJobStatus.setRunState(JobStatus.SUCCEEDED); - } else { - currentJobStatus.setState(JobStatus.State.FAILED); - currentJobStatus.setRunState(JobStatus.FAILED); - } - threadPool.shutdownNow(); - } - - } - - class LocalGroom extends BSPPeer { - private long superStepCount = 0; - private final ConcurrentLinkedQueue localMessageQueue = new ConcurrentLinkedQueue(); - // outgoing queue - private final Map> outgoingQueues = new ConcurrentHashMap>(); - private final String peerName; - - public LocalGroom(String peerName) throws IOException { - this.peerName = peerName; - } - - @Override - public void send(String peerName, BSPMessage msg) throws IOException { - if (this.peerName.equals(peerName)) { - put(msg); - } else { - // put this into a outgoing queue - if (outgoingQueues.get(peerName) == null) { - outgoingQueues.put(peerName, new ConcurrentLinkedQueue()); - } - outgoingQueues.get(peerName).add(msg); - } - } - - @Override - public void put(BSPMessage msg) throws IOException { - localMessageQueue.add(msg); - } - - @Override - public BSPMessage getCurrentMessage() throws IOException { - return localMessageQueue.poll(); - } - - @Override - public int getNumCurrentMessages() { - return localMessageQueue.size(); - } - - @Override - public void sync() throws IOException, KeeperException, - InterruptedException { - // wait until all threads reach this barrier - barrierSync(); - // send the messages - for (Entry> entry : outgoingQueues - .entrySet()) { - String peerName = entry.getKey(); - for (BSPMessage msg : entry.getValue()) - localGrooms.get(peerName).put(msg); - } - // clear the local outgoing queue - outgoingQueues.clear(); - // sync again to avoid data inconsistency - barrierSync(); - incrementSuperSteps(); - } - - private void barrierSync() throws InterruptedException { - try { - barrier.await(); - } catch (BrokenBarrierException e) { - throw new InterruptedException("Barrier has been broken!" + e); - } - } - - private void incrementSuperSteps() { - currentJobStatus.setprogress(superStepCount++); - currentJobStatus.setSuperstepCount(currentJobStatus.progress()); - } - - @Override - public long getSuperstepCount() { - return superStepCount; - } - - @Override - public String getPeerName() { - return peerName; - } - - @Override - public String[] getAllPeerNames() { - return localGrooms.keySet().toArray( - new String[localGrooms.keySet().size()]); - } - - @Override - public void clear() { - localMessageQueue.clear(); - } - - @Override - public long getProtocolVersion(String protocol, long clientVersion) - throws IOException { - return 3; - } - - @Override - public void close() throws IOException { - - } - - @Override - public void put(BSPMessageBundle messages) throws IOException { - } - - } -} Index: src/java/org/apache/hama/bsp/JobProfile.java =================================================================== --- src/java/org/apache/hama/bsp/JobProfile.java (revision 1152138) +++ src/java/org/apache/hama/bsp/JobProfile.java (working copy) @@ -1,111 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableFactories; -import org.apache.hadoop.io.WritableFactory; - -/** - * A JobProfile tracks job's status. - */ -public class JobProfile implements Writable { - - static { // register actor - WritableFactories.setFactory(JobProfile.class, new WritableFactory() { - public Writable newInstance() { - return new JobProfile(); - } - }); - } - - String user; - final BSPJobID jobid; - String jobFile; - String name; - - /** - * Construct an empty {@link JobProfile}. - */ - public JobProfile() { - jobid = new BSPJobID(); - } - - /** - * Construct a {@link JobProfile} the userid, jobid, job config-file, - * job-details url and job name. - * - * @param user userid of the person who submitted the job. - * @param jobid id of the job. - * @param jobFile job configuration file. - * @param name user-specified job name. - */ - public JobProfile(String user, BSPJobID jobid, String jobFile, String name) { - this.user = user; - this.jobid = jobid; - this.jobFile = jobFile; - this.name = name; - } - - /** - * Get the user id. - */ - public String getUser() { - return user; - } - - /** - * Get the job id. - */ - public BSPJobID getJobID() { - return jobid; - } - - /** - * Get the configuration file for the job. - */ - public String getJobFile() { - return jobFile; - } - - /** - * Get the user-specified job name. - */ - public String getJobName() { - return name; - } - - public void write(DataOutput out) throws IOException { - jobid.write(out); - Text.writeString(out, jobFile); - Text.writeString(out, user); - Text.writeString(out, name); - } - - public void readFields(DataInput in) throws IOException { - jobid.readFields(in); - this.jobFile = Text.readString(in); - this.user = Text.readString(in); - this.name = Text.readString(in); - } -} Index: src/java/org/apache/hama/bsp/LaunchTaskAction.java =================================================================== --- src/java/org/apache/hama/bsp/LaunchTaskAction.java (revision 1152138) +++ src/java/org/apache/hama/bsp/LaunchTaskAction.java (working copy) @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the - * {@link org.apache.hama.bsp.GroomServer} to launch a new task. - */ -class LaunchTaskAction extends GroomServerAction { - private Task task; - - public LaunchTaskAction() { - super(ActionType.LAUNCH_TASK); - } - - public LaunchTaskAction(Task task) { - super(ActionType.LAUNCH_TASK); - this.task = task; - } - - public Task getTask() { - return task; - } - - public void write(DataOutput out) throws IOException { - task.write(out); - } - - public void readFields(DataInput in) throws IOException { - task = new BSPTask(); - task.readFields(in); - } - -} Index: src/java/org/apache/hama/bsp/BSPJobID.java =================================================================== --- src/java/org/apache/hama/bsp/BSPJobID.java (revision 1152138) +++ src/java/org/apache/hama/bsp/BSPJobID.java (working copy) @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; - -import java.io.DataOutput; -import java.io.IOException; -import java.text.NumberFormat; - -import org.apache.hadoop.io.Text; - -/** - * BSPJobID represents the immutable and unique identifier for the job. - */ -public class BSPJobID extends ID implements Comparable { - protected static final String JOB = "job"; - private final Text jtIdentifier; - - protected static final NumberFormat idFormat = NumberFormat.getInstance(); - static { - idFormat.setGroupingUsed(false); - idFormat.setMinimumIntegerDigits(4); - } - - public BSPJobID(String jtIdentifier, int id) { - super(id); - this.jtIdentifier = new Text(jtIdentifier); - } - - public BSPJobID() { - jtIdentifier = new Text(); - } - - public String getJtIdentifier() { - return jtIdentifier.toString(); - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) - return false; - - BSPJobID that = (BSPJobID) o; - return this.jtIdentifier.equals(that.jtIdentifier); - } - - @Override - public int compareTo(ID o) { - BSPJobID that = (BSPJobID) o; - int jtComp = this.jtIdentifier.compareTo(that.jtIdentifier); - if (jtComp == 0) { - return this.id - that.id; - } else - return jtComp; - } - - public StringBuilder appendTo(StringBuilder builder) { - builder.append(SEPARATOR); - builder.append(jtIdentifier); - builder.append(SEPARATOR); - builder.append(idFormat.format(id)); - return builder; - } - - @Override - public int hashCode() { - return jtIdentifier.hashCode() + id; - } - - @Override - public String toString() { - return appendTo(new StringBuilder(JOB)).toString(); - } - - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - this.jtIdentifier.readFields(in); - } - - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - jtIdentifier.write(out); - } - - public static BSPJobID forName(String str) throws IllegalArgumentException { - if (str == null) - return null; - try { - String[] parts = str.split("_"); - if (parts.length == 3) { - if (parts[0].equals(JOB)) { - return new BSPJobID(parts[1], Integer.parseInt(parts[2])); - } - } - } catch (Exception ex) { - } - throw new IllegalArgumentException("JobId string : " + str - + " is not properly formed"); - } -} Index: src/java/org/apache/hama/bsp/JobInProgressListener.java =================================================================== --- src/java/org/apache/hama/bsp/JobInProgressListener.java (revision 1152138) +++ src/java/org/apache/hama/bsp/JobInProgressListener.java (working copy) @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.IOException; - -/** - * A listener for changes in a {@link JobInProgress job}'s lifecycle in the - * {@link BSPMaster}. - */ -abstract class JobInProgressListener { - - /** - * Invoked when a new job has been added to the {@link BSPMaster}. - * @param job The job to be added. - * @throws IOException - */ - public abstract void jobAdded(JobInProgress job) throws IOException; - - /** - * Invoked when a job has been removed from the {@link BSPMaster}. - * @param job The job to be removed . - * @throws IOException - */ - public abstract void jobRemoved(JobInProgress job) throws IOException; - -} Index: src/java/org/apache/hama/bsp/ReinitGroomAction.java =================================================================== --- src/java/org/apache/hama/bsp/ReinitGroomAction.java (revision 1152138) +++ src/java/org/apache/hama/bsp/ReinitGroomAction.java (working copy) @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -/** - * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the - * {@link org.apache.hama.bsp.GroomServer} to reinitialize itself. - */ -class ReinitGroomAction extends GroomServerAction { - - public ReinitGroomAction() { - super(ActionType.REINIT_GROOM); - } - - public void write(DataOutput out) throws IOException { - } - - public void readFields(DataInput in) throws IOException { - } - -} Index: src/java/org/apache/hama/bsp/TaskRunner.java =================================================================== --- src/java/org/apache/hama/bsp/TaskRunner.java (revision 1152138) +++ src/java/org/apache/hama/bsp/TaskRunner.java (working copy) @@ -1,185 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.InetSocketAddress; -import java.util.Vector; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.RunJar; - -/** - * Base class that runs a task in a separate process. - */ -public class TaskRunner extends Thread { - - public static final Log LOG = LogFactory.getLog(TaskRunner.class); - - boolean killed = false; - private Process process; - private Task task; - private BSPJob conf; - private GroomServer groomServer; - - public TaskRunner(BSPTask bspTask, GroomServer groom, BSPJob conf) { - this.task = bspTask; - this.conf = conf; - this.groomServer = groom; - } - - public Task getTask() { - return task; - } - - /** - * Called to assemble this task's input. This method is run in the parent - * process before the child is spawned. It should not execute user code, only - * system code. - */ - public boolean prepare() throws IOException { - return true; - } - - public void run() { - try { - String sep = System.getProperty("path.separator"); - File workDir = new File(new File(task.getJobFile()).getParent(), "work"); - boolean isCreated = workDir.mkdirs(); - if(!isCreated) { - LOG.debug("TaskRunner.workDir : " + workDir); - } - - StringBuffer classPath = new StringBuffer(); - // start with same classpath as parent process - classPath.append(System.getProperty("java.class.path")); - classPath.append(sep); - - String jar = conf.getJar(); - if (jar != null) { // if jar exists, it into workDir - RunJar.unJar(new File(jar), workDir); - File[] libs = new File(workDir, "lib").listFiles(); - if (libs != null) { - for (int i = 0; i < libs.length; i++) { - classPath.append(sep); // add libs from jar to classpath - classPath.append(libs[i]); - } - } - classPath.append(sep); - classPath.append(new File(workDir, "classes")); - classPath.append(sep); - classPath.append(workDir); - } - - // Build exec child jmv args. - Vector vargs = new Vector(); - File jvm = // use same jvm as parent - new File(new File(System.getProperty("java.home"), "bin"), "java"); - vargs.add(jvm.toString()); - - // bsp.child.java.opts - String javaOpts = conf.getConf().get("bsp.child.java.opts", "-Xmx200m"); - javaOpts = javaOpts.replace("@taskid@", task.getTaskID().toString()); - - String[] javaOptsSplit = javaOpts.split(" "); - for (int i = 0; i < javaOptsSplit.length; i++) { - vargs.add(javaOptsSplit[i]); - } - - // Add classpath. - vargs.add("-classpath"); - vargs.add(classPath.toString()); - // Add main class and its arguments - vargs.add(GroomServer.Child.class.getName()); // main of Child - - InetSocketAddress addr = groomServer.getTaskTrackerReportAddress(); - vargs.add(addr.getHostName()); - vargs.add(Integer.toString(addr.getPort())); - vargs.add(task.getTaskID().toString()); - vargs.add(groomServer.groomHostName); - - // Run java - runChild((String[]) vargs.toArray(new String[0]), workDir); - } catch (IOException e) { - LOG.error(e); - } - } - - /** - * Run the child process - */ - private void runChild(String[] args, File dir) throws IOException { - this.process = Runtime.getRuntime().exec(args, null, dir); - try { - new Thread() { - public void run() { - logStream(process.getErrorStream()); // copy log output - } - }.start(); - - logStream(process.getInputStream()); // normally empty - - int exit_code = process.waitFor(); - if (!killed && exit_code != 0) { - throw new IOException("Task process exit with nonzero status of " - + exit_code + "."); - } - - } catch (InterruptedException e) { - throw new IOException(e.toString()); - } finally { - kill(); - } - } - - /** - * Kill the child process - */ - public void kill() { - if (process != null) { - process.destroy(); - } - killed = true; - } - - /** - */ - private void logStream(InputStream output) { - try { - BufferedReader in = new BufferedReader(new InputStreamReader(output)); - String line; - while ((line = in.readLine()) != null) { - LOG.info(task.getTaskID() + " " + line); - } - } catch (IOException e) { - LOG.warn(task.getTaskID() + " Error reading child output", e); - } finally { - try { - output.close(); - } catch (IOException e) { - LOG.warn(task.getTaskID() + " Error closing child output", e); - } - } - } - -} Index: src/java/org/apache/hama/bsp/DirectiveException.java =================================================================== --- src/java/org/apache/hama/bsp/DirectiveException.java (revision 1152138) +++ src/java/org/apache/hama/bsp/DirectiveException.java (working copy) @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.bsp; - -/** - * A custom exception class for Directive. - */ -public class DirectiveException extends RuntimeException{ - private static final long serialVersionUID = -8052582046894492822L; - - public DirectiveException(){ - super(); - } - - public DirectiveException(String message){ - super(message); - } - - public DirectiveException(String message, Throwable t){ - super(message, t); - } -} Index: src/java/org/apache/hama/BSPMasterRunner.java =================================================================== --- src/java/org/apache/hama/BSPMasterRunner.java (revision 1152138) +++ src/java/org/apache/hama/BSPMasterRunner.java (working copy) @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hama.bsp.BSPMaster; - -/** - * This class starts and runs the BSPMaster. - */ -public class BSPMasterRunner extends Configured implements Tool { - - public static final Log LOG = LogFactory.getLog(BSPMasterRunner.class); - - @Override - public int run(String[] args) throws Exception { - StringUtils.startupShutdownMessage(BSPMaster.class, args, LOG); - - if (args.length != 0) { - System.out.println("usage: BSPMasterRunner"); - System.exit(-1); - } - - try { - HamaConfiguration conf = new HamaConfiguration(getConf()); - BSPMaster master = BSPMaster.startMaster(conf); - master.offerService(); - } catch (Throwable e) { - LOG.fatal(StringUtils.stringifyException(e)); - return -1; - } - return 0; - } - - public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new BSPMasterRunner(), args); - System.exit(exitCode); - } - -} Index: src/java/org/apache/hama/ipc/BSPPeerProtocol.java =================================================================== --- src/java/org/apache/hama/ipc/BSPPeerProtocol.java (revision 1152138) +++ src/java/org/apache/hama/ipc/BSPPeerProtocol.java (working copy) @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ipc; - -import java.io.Closeable; -import java.io.IOException; - -import org.apache.hama.Constants; -import org.apache.hama.bsp.PeerNames; -import org.apache.hama.bsp.Task; -import org.apache.hama.bsp.TaskAttemptID; - -/** - * Protocol that task child process uses to contact its parent process. - */ -public interface BSPPeerProtocol extends HamaRPCProtocolVersion, Closeable, - Constants { - - /** Called when a child task process starts, to get its task. */ - Task getTask(TaskAttemptID taskid) throws IOException; - - /** - * Periodically called by child to check if parent is still alive. - * - * @return True if the task is known - */ - boolean ping(TaskAttemptID taskid) throws IOException; - - /** - * Report that the task is successfully completed. Failure is assumed if the - * task process exits without calling this. - * - * @param taskid task's id - * @param shouldBePromoted whether to promote the task's output or not - */ - void done(TaskAttemptID taskid, boolean shouldBePromoted) throws IOException; - - /** Report that the task encounted a local filesystem error. */ - void fsError(TaskAttemptID taskId, String message) throws IOException; - - void incrementSuperstepCount(TaskAttemptID taskid) throws IOException; - - /** - * @return the all BSPPeer names. - */ - PeerNames getAllPeerNames(); - -} Index: src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java =================================================================== --- src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java (revision 1152138) +++ src/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java (working copy) @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ipc; - -import org.apache.hadoop.ipc.VersionedProtocol; - -/** - * There is one version id for all the RPC interfaces. If any interface is - * changed, the versionID must be changed here. - */ -public interface HamaRPCProtocolVersion extends VersionedProtocol { - public static final long versionID = 1L; -} Index: src/java/org/apache/hama/ipc/GroomProtocol.java =================================================================== --- src/java/org/apache/hama/ipc/GroomProtocol.java (revision 1152138) +++ src/java/org/apache/hama/ipc/GroomProtocol.java (working copy) @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ipc; - -import java.io.IOException; - -import org.apache.hama.bsp.Directive; - -/** - * A protocol for BSPMaster talks to GroomServer. - */ -public interface GroomProtocol extends HamaRPCProtocolVersion { - - /** - * Instruct GroomServer performaning tasks. - * - * @param directive instructs a GroomServer performing necessary - * execution. - * @throws IOException - */ - void dispatch(Directive directive) throws IOException; - -} Index: src/java/org/apache/hama/ipc/JobSubmissionProtocol.java =================================================================== --- src/java/org/apache/hama/ipc/JobSubmissionProtocol.java (revision 1152138) +++ src/java/org/apache/hama/ipc/JobSubmissionProtocol.java (working copy) @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ipc; - -import java.io.IOException; - -import org.apache.hama.bsp.ClusterStatus; -import org.apache.hama.bsp.BSPJobID; -import org.apache.hama.bsp.JobProfile; -import org.apache.hama.bsp.JobStatus; -import org.apache.hama.bsp.TaskAttemptID; - -/** - * Protocol that a groom server and the central BSP Master use to communicate. - * This interface will contains several methods: submitJob, killJob, and - * killTask. - */ -public interface JobSubmissionProtocol extends HamaRPCProtocolVersion { - - /** - * Allocate a new id for the job. - * - * @return job id - * @throws IOException - */ - public BSPJobID getNewJobId() throws IOException; - - /** - * Submit a Job for execution. Returns the latest profile for that job. The - * job files should be submitted in system-dir/jobName. - * - * @param jobID - * @param jobFile - * @return jobStatus - * @throws IOException - */ - // public JobStatus submitJob(BSPJobID jobName) throws IOException; - - public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException; - - /** - * Get the current status of the cluster - * - * @param detailed if true then report groom names as well - * @return summary of the state of the cluster - */ - public ClusterStatus getClusterStatus(boolean detailed) throws IOException; - - /** - * Grab a handle to a job that is already known to the BSPMaster. - * - * @return Profile of the job, or null if not found. - */ - public JobProfile getJobProfile(BSPJobID jobid) throws IOException; - - /** - * Grab a handle to a job that is already known to the BSPMaster. - * - * @return Status of the job, or null if not found. - */ - public JobStatus getJobStatus(BSPJobID jobid) throws IOException; - - /** - * A BSP system always operates on a single filesystem. This function returns - * the fs name. ('local' if the localfs; 'addr:port' if dfs). The client can - * then copy files into the right locations prior to submitting the job. - */ - public String getFilesystemName() throws IOException; - - /** - * Get the jobs that are not completed and not failed - * - * @return array of JobStatus for the running/to-be-run jobs. - */ - public JobStatus[] jobsToComplete() throws IOException; - - /** - * Get all the jobs submitted. - * - * @return array of JobStatus for the submitted jobs - */ - public JobStatus[] getAllJobs() throws IOException; - - /** - * Grab the bspmaster system directory path where job-specific files are to be - * placed. - * - * @return the system directory where job-specific files are to be placed. - */ - public String getSystemDir(); - - /** - * Kill the indicated job - */ - public void killJob(BSPJobID jobid) throws IOException; - - /** - * Kill indicated task attempt. - * - * @param taskId the id of the task to kill. - * @param shouldFail if true the task is failed and added to failed tasks - * list, otherwise it is just killed, w/o affecting job failure - * status. - */ - public boolean killTask(TaskAttemptID taskId, boolean shouldFail) - throws IOException; - -} Index: src/java/org/apache/hama/ipc/MasterProtocol.java =================================================================== --- src/java/org/apache/hama/ipc/MasterProtocol.java (revision 1152138) +++ src/java/org/apache/hama/ipc/MasterProtocol.java (working copy) @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ipc; - -import java.io.IOException; - -import org.apache.hama.bsp.GroomServerStatus; -import org.apache.hama.bsp.Directive; - -/** - * A new protocol for GroomServers communicate with BSPMaster. This - * protocol paired with WorkerProtocl, let GroomServers enrol with - * BSPMaster, so that BSPMaster can dispatch tasks to GroomServers. - */ -public interface MasterProtocol extends HamaRPCProtocolVersion { - - /** - * A GroomServer register with its status to BSPMaster, which will update - * GroomServers cache. - * - * @param status to be updated in cache. - * @return true if successfully register with BSPMaster; false if fail. - */ - boolean register(GroomServerStatus status) throws IOException; - - /** - * A GroomServer (periodically) reports task statuses back to the BSPMaster. - * @param directive - */ - boolean report(Directive directive) throws IOException; - - public String getSystemDir(); - -} Index: src/java/org/apache/hama/ipc/package.html =================================================================== --- src/java/org/apache/hama/ipc/package.html (revision 1152138) +++ src/java/org/apache/hama/ipc/package.html (working copy) @@ -1,23 +0,0 @@ - - - - - -Tools to help define network clients and servers. - - Index: src/java/org/apache/hama/Constants.java =================================================================== --- src/java/org/apache/hama/Constants.java (revision 1152138) +++ src/java/org/apache/hama/Constants.java (working copy) @@ -1,95 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -/** - * Some constants used in the Hama. - */ -public interface Constants { - - public static final String GROOM_RPC_HOST = "bsp.groom.rpc.hostname"; - - public static final String DEFAULT_GROOM_RPC_HOST = "0.0.0.0"; - - public static final String GROOM_RPC_PORT = "bsp.groom.rpc.port"; - - /** Default port region rpc server listens on. */ - public static final int DEFAULT_GROOM_RPC_PORT = 50000; - - - /////////////////////////////////////// - // Constants for BSP Package - /////////////////////////////////////// - /** default host address */ - public static final String PEER_HOST = "bsp.peer.hostname"; - /** default host address */ - public static final String DEFAULT_PEER_HOST = "0.0.0.0"; - - public static final String PEER_PORT = "bsp.peer.port"; - /** Default port region server listens on. */ - public static final int DEFAULT_PEER_PORT = 61000; - - public static final String PEER_ID = "bsp.peer.id"; - - /** Parameter name for what groom server implementation to use. */ - public static final String GROOM_SERVER_IMPL= "hama.groomserver.impl"; - - /** When we encode strings, we always specify UTF8 encoding */ - static final String UTF8_ENCODING = "UTF-8"; - - /////////////////////////////////////// - // Constants for ZooKeeper - /////////////////////////////////////// - /** zookeeper root */ - public static final String ZOOKEEPER_ROOT = "bsp.zookeeper.root"; - /** zookeeper default root */ - public static final String DEFAULT_ZOOKEEPER_ROOT = "/bsp"; - - /** zookeeper server address */ - public static final String ZOOKEEPER_SERVER_ADDRS = "zookeeper.server"; - /** zookeeper default server address */ - static final String DEFAULT_ZOOKEEPER_SERVER_ADDR = "localhost:21810"; - /** Parameter name for number of times to retry writes to ZooKeeper. */ - public static final String ZOOKEEPER_RETRIES = "zookeeper.retries"; - /** Default number of times to retry writes to ZooKeeper. */ - public static final int DEFAULT_ZOOKEEPER_RETRIES = 5; - /** Parameter name for ZooKeeper pause between retries. In milliseconds. */ - public static final String ZOOKEEPER_PAUSE = "zookeeper.pause"; - /** Default ZooKeeper pause value. In milliseconds. */ - public static final int DEFAULT_ZOOKEEPER_PAUSE = 2 * 1000; - - static final String ZOOKEEPER_CONFIG_NAME = "zoo.cfg"; - static final String ZOOKEEPER_CLIENT_PORT = "hama.zookeeper.property.clientPort"; - static final String ZOOKEEPER_SESSION_TIMEOUT = "hama.zookeeper.session.timeout"; - static final int DEFAULT_ZOOKEEPER_CLIENT_PORT = 21810; - static final String ZOOKEEPER_QUORUM = "hama.zookeeper.quorum"; - /** Cluster is in distributed mode or not */ - static final String CLUSTER_DISTRIBUTED = "hama.cluster.distributed"; - /** Cluster is fully-distributed */ - static final String CLUSTER_IS_DISTRIBUTED = "true"; - - - // Other constants - - /** - * An empty instance. - */ - static final byte [] EMPTY_BYTE_ARRAY = new byte [0]; -} Index: src/java/org/apache/hama/ZooKeeperRunner.java =================================================================== --- src/java/org/apache/hama/ZooKeeperRunner.java (revision 1152138) +++ src/java/org/apache/hama/ZooKeeperRunner.java (working copy) @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hama.zookeeper.QuorumPeer; - -/** - * This class starts and runs the ZooKeeperServer. - */ -public class ZooKeeperRunner extends Configured implements Tool { - - public static final Log LOG = LogFactory.getLog(ZooKeeperRunner.class); - - @Override - public int run(String[] args) throws Exception { - QuorumPeer.run(getConf()); - return 0; - } - - public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new ZooKeeperRunner(), args); - System.exit(exitCode); - } - -} Index: src/java/org/apache/hama/util/ClusterUtil.java =================================================================== --- src/java/org/apache/hama/util/ClusterUtil.java (revision 1152138) +++ src/java/org/apache/hama/util/ClusterUtil.java (working copy) @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSPMaster; -import org.apache.hama.bsp.GroomServer; - -public class ClusterUtil { - private static final Log LOG = LogFactory.getLog(ClusterUtil.class); - - /** - * Data Structure to hold GroomServer Thread and GroomServer instance - */ - public static class GroomServerThread extends Thread { - private final GroomServer groomServer; - - public GroomServerThread(final GroomServer r, final int index) { - super(r, "GroomServer:" + index); - this.groomServer = r; - } - - /** @return the groom server */ - public GroomServer getGroomServer() { - return this.groomServer; - } - - /** - * Block until the groom server has come online, indicating it is ready - * to be used. - */ - public void waitForServerOnline() { - while (!groomServer.isRunning()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // continue waiting - } - } - } - } - - /** - * Creates a {@link GroomServerThread}. - * Call 'start' on the returned thread to make it run. - * @param c Configuration to use. - * @param hrsc Class to create. - * @param index Used distingushing the object returned. - * @throws IOException - * @return Groom server added. - */ - public static ClusterUtil.GroomServerThread createGroomServerThread(final Configuration c, - final Class hrsc, final int index) - throws IOException { - GroomServer server; - try { - server = hrsc.getConstructor(Configuration.class).newInstance(c); - } catch (Exception e) { - IOException ioe = new IOException(); - ioe.initCause(e); - throw ioe; - } - return new ClusterUtil.GroomServerThread(server, index); - } - - /** - * Start the cluster. - * @param m - * @param conf - * @param groomservers - * @return Address to use contacting master. - * @throws InterruptedException - * @throws IOException - */ - public static String startup(final BSPMaster m, - final List groomservers, Configuration conf) throws IOException, InterruptedException { - if (m != null) { - BSPMaster.startMaster((HamaConfiguration) conf); - } - - if (groomservers != null) { - for (ClusterUtil.GroomServerThread t: groomservers) { - t.start(); - } - } - - return m == null? null: BSPMaster.getAddress(conf).getHostName(); - } - - public static void shutdown(BSPMaster master, - List groomThreads, Configuration conf) { - LOG.debug("Shutting down HAMA Cluster"); - // TODO: - } -} Index: src/java/org/apache/hama/util/RunJar.java =================================================================== --- src/java/org/apache/hama/util/RunJar.java (revision 1152138) +++ src/java/org/apache/hama/util/RunJar.java (working copy) @@ -1,151 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.reflect.Array; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.List; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.jar.Manifest; - -import org.apache.hadoop.fs.FileUtil; - -/** - * Run a Hama job jar. - */ -public class RunJar { - - /** Unpack a jar file into a directory. */ - public static void unJar(File jarFile, File toDir) throws IOException { - JarFile jar = new JarFile(jarFile); - try { - Enumeration entries = jar.entries(); - while (entries.hasMoreElements()) { - JarEntry entry = (JarEntry) entries.nextElement(); - if (!entry.isDirectory()) { - InputStream in = jar.getInputStream(entry); - try { - File file = new File(toDir, entry.getName()); - file.getParentFile().mkdirs(); - OutputStream out = new FileOutputStream(file); - try { - byte[] buffer = new byte[8192]; - int i; - while ((i = in.read(buffer)) != -1) { - out.write(buffer, 0, i); - } - } finally { - out.close(); - } - } finally { - in.close(); - } - } - } - } finally { - jar.close(); - } - } - - /** - * Run a Hama job jar. If the main class is not in the jar's manifest, then - * it must be provided on the command line. - */ - public static void main(String[] args) throws Throwable { - String usage = "Usage: hama jar [mainClass] args..."; - - if (args.length < 1) { - System.err.println(usage); - System.exit(-1); - } - - int firstArg = 0; - String fileName = args[firstArg++]; - File file = new File(fileName); - String mainClassName = null; - - JarFile jarFile = new JarFile(fileName); - Manifest manifest = jarFile.getManifest(); - if (manifest != null) { - mainClassName = manifest.getMainAttributes().getValue("Main-Class"); - } - jarFile.close(); - - if (mainClassName == null) { - if (args.length < 2) { - System.err.println(usage); - System.exit(-1); - } - mainClassName = args[firstArg++]; - } - mainClassName = mainClassName.replaceAll("/", "."); - - final File workDir = File.createTempFile("hama-unjar", ""); - workDir.delete(); - workDir.mkdirs(); - - Runtime.getRuntime().addShutdownHook(new Thread() { - public void run() { - try { - FileUtil.fullyDelete(workDir); - } catch (IOException e) { - } - } - }); - - unJar(file, workDir); - - List classPath = new ArrayList(); - classPath.add(new File(workDir + "/").toURI().toURL()); - classPath.add(file.toURI().toURL()); - classPath.add(new File(workDir, "classes/").toURI().toURL()); - File[] libs = new File(workDir, "lib").listFiles(); - if (libs != null) { - for (int i = 0; i < libs.length; i++) { - classPath.add(libs[i].toURI().toURL()); - } - } - ClassLoader loader = new URLClassLoader((URL[]) classPath - .toArray(new URL[0])); - - Thread.currentThread().setContextClassLoader(loader); - Class mainClass = loader.loadClass(mainClassName); - Method main = mainClass.getMethod("main", new Class[] { Array.newInstance( - String.class, 0).getClass() }); - String[] newArgs = (String[]) Arrays.asList(args).subList(firstArg, - args.length).toArray(new String[0]); - try { - main.invoke(null, new Object[] { newArgs }); - } catch (InvocationTargetException e) { - throw e.getTargetException(); - } - } - -} Index: src/java/org/apache/hama/util/VersionInfo.java =================================================================== --- src/java/org/apache/hama/util/VersionInfo.java (revision 1152138) +++ src/java/org/apache/hama/util/VersionInfo.java (working copy) @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -/** - * A version information class. - */ -public class VersionInfo { - - public static void main(String[] args) { - System.out.println("Apache Hama - 0.4"); - } -} Index: src/java/org/apache/hama/util/Bytes.java =================================================================== --- src/java/org/apache/hama/util/Bytes.java (revision 1152138) +++ src/java/org/apache/hama/util/Bytes.java (working copy) @@ -1,1226 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.util.Comparator; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.RawComparator; -import org.apache.hadoop.io.WritableComparator; -import org.apache.hadoop.io.WritableUtils; -import org.apache.hama.Constants; - -/** - * Utility class that handles byte arrays, conversions to/from other types, - * comparisons, hash code generation, manufacturing keys for HashMaps or - * HashSets, etc. - */ -public class Bytes { - - private static final Log LOG = LogFactory.getLog(Bytes.class); - - /** - * Size of boolean in bytes - */ - public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE; - - /** - * Size of byte in bytes - */ - public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN; - - /** - * Size of char in bytes - */ - public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE; - - /** - * Size of double in bytes - */ - public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE; - - /** - * Size of float in bytes - */ - public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE; - - /** - * Size of int in bytes - */ - public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; - - /** - * Size of long in bytes - */ - public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; - - /** - * Size of short in bytes - */ - public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; - - /** - * Estimate of size cost to pay beyond payload in jvm for instance of byte []. - * Estimate based on study of jhat and jprofiler numbers. - */ - // JHat says BU is 56 bytes. - // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?) - public static final int ESTIMATED_HEAP_TAX = 16; - - /** - * Byte array comparator class. - */ - public static class ByteArrayComparator implements RawComparator { - /** - * Constructor - */ - public ByteArrayComparator() { - super(); - } - - public int compare(byte[] left, byte[] right) { - return compareTo(left, right); - } - - public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { - return compareTo(b1, s1, l1, b2, s2, l2); - } - } - - /** - * Pass this to TreeMaps where byte [] are keys. - */ - public static Comparator BYTES_COMPARATOR = new ByteArrayComparator(); - - /** - * Use comparing byte arrays, byte-by-byte - */ - public static RawComparator BYTES_RAWCOMPARATOR = new ByteArrayComparator(); - - /** - * Read byte-array written with a WritableableUtils.vint prefix. - * - * @param in Input to read from. - * @return byte array read off in - * @throws IOException e - */ - public static byte[] readByteArray(final DataInput in) throws IOException { - int len = WritableUtils.readVInt(in); - if (len < 0) { - throw new NegativeArraySizeException(Integer.toString(len)); - } - byte[] result = new byte[len]; - in.readFully(result, 0, len); - return result; - } - - /** - * Read byte-array written with a WritableableUtils.vint prefix. IOException - * is converted to a RuntimeException. - * - * @param in Input to read from. - * @return byte array read off in - */ - public static byte[] readByteArrayThrowsRuntime(final DataInput in) { - try { - return readByteArray(in); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Write byte-array with a WritableableUtils.vint prefix. - * - * @param out output stream to be written to - * @param b array to write - * @throws IOException e - */ - public static void writeByteArray(final DataOutput out, final byte[] b) - throws IOException { - if (b == null) { - WritableUtils.writeVInt(out, 0); - } else { - writeByteArray(out, b, 0, b.length); - } - } - - /** - * Write byte-array to out with a vint length prefix. - * - * @param out output stream - * @param b array - * @param offset offset into array - * @param length length past offset - * @throws IOException e - */ - public static void writeByteArray(final DataOutput out, final byte[] b, - final int offset, final int length) throws IOException { - WritableUtils.writeVInt(out, length); - out.write(b, offset, length); - } - - /** - * Write byte-array from src to tgt with a vint length prefix. - * - * @param tgt target array - * @param tgtOffset offset into target array - * @param src source array - * @param srcOffset source offset - * @param srcLength source length - * @return New offset in src array. - */ - public static int writeByteArray(final byte[] tgt, final int tgtOffset, - final byte[] src, final int srcOffset, final int srcLength) { - byte[] vint = vintToBytes(srcLength); - System.arraycopy(vint, 0, tgt, tgtOffset, vint.length); - int offset = tgtOffset + vint.length; - System.arraycopy(src, srcOffset, tgt, offset, srcLength); - return offset + srcLength; - } - - /** - * Put bytes at the specified byte array position. - * - * @param tgtBytes the byte array - * @param tgtOffset position in the array - * @param srcBytes array to write out - * @param srcOffset source offset - * @param srcLength source length - * @return incremented offset - */ - public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, - int srcOffset, int srcLength) { - System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength); - return tgtOffset + srcLength; - } - - /** - * Write a single byte out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param b byte to write out - * @return incremented offset - */ - public static int putByte(byte[] bytes, int offset, byte b) { - bytes[offset] = b; - return offset + 1; - } - - /** - * Returns a new byte array, copied from the passed ByteBuffer. - * - * @param bb A ByteBuffer - * @return the byte array - */ - public static byte[] toBytes(ByteBuffer bb) { - int length = bb.limit(); - byte[] result = new byte[length]; - System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length); - return result; - } - - /** - * @param b Presumed UTF-8 encoded byte array. - * @return String made from b - */ - public static String toString(final byte[] b) { - if (b == null) { - return null; - } - return toString(b, 0, b.length); - } - - /** - * Joins two byte arrays together using a separator. - * - * @param b1 The first byte array. - * @param sep The separator to use. - * @param b2 The second byte array. - */ - public static String toString(final byte[] b1, String sep, final byte[] b2) { - return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length); - } - - /** - * This method will convert utf8 encoded bytes into a string. If an - * UnsupportedEncodingException occurs, this method will eat it and return - * null instead. - * - * @param b Presumed UTF-8 encoded byte array. - * @param off offset into array - * @param len length of utf-8 sequence - * @return String made from b or null - */ - public static String toString(final byte[] b, int off, int len) { - if (b == null) { - return null; - } - if (len == 0) { - return ""; - } - try { - return new String(b, off, len, Constants.UTF8_ENCODING); - } catch (UnsupportedEncodingException e) { - LOG.error("UTF-8 not supported?", e); - return null; - } - } - - /** - * Write a printable representation of a byte array. - * - * @param b byte array - * @return string - * @see #toStringBinary(byte[], int, int) - */ - public static String toStringBinary(final byte[] b) { - return toStringBinary(b, 0, b.length); - } - - /** - * Write a printable representation of a byte array. Non-printable characters - * are hex escaped in the format \\x%02X, eg: \x00 \x05 etc - * - * @param b array to write out - * @param off offset to start at - * @param len length to write - * @return string output - */ - public static String toStringBinary(final byte[] b, int off, int len) { - StringBuilder result = new StringBuilder(); - try { - String first = new String(b, off, len, "ISO-8859-1"); - for (int i = 0; i < first.length(); ++i) { - int ch = first.charAt(i) & 0xFF; - if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') - || (ch >= 'a' && ch <= 'z') || ch == ',' || ch == '_' || ch == '-' - || ch == ':' || ch == ' ' || ch == '<' || ch == '>' || ch == '=' - || ch == '/' || ch == '.') { - result.append(first.charAt(i)); - } else { - result.append(String.format("\\x%02X", ch)); - } - } - } catch (UnsupportedEncodingException e) { - LOG.error("ISO-8859-1 not supported?", e); - } - return result.toString(); - } - - private static boolean isHexDigit(char c) { - return (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9'); - } - - /** - * Takes a ASCII digit in the range A-F0-9 and returns the corresponding - * integer/ordinal value. - * - * @param ch The hex digit. - * @return The converted hex value as a byte. - */ - public static byte toBinaryFromHex(byte ch) { - if (ch >= 'A' && ch <= 'F') - return (byte) ((byte) 10 + (byte) (ch - 'A')); - // else - return (byte) (ch - '0'); - } - - public static byte[] toBytesBinary(String in) { - // this may be bigger than we need, but lets be safe. - byte[] b = new byte[in.length()]; - int size = 0; - for (int i = 0; i < in.length(); ++i) { - char ch = in.charAt(i); - if (ch == '\\') { - // begin hex escape: - char next = in.charAt(i + 1); - if (next != 'x') { - // invalid escape sequence, ignore this one. - b[size++] = (byte) ch; - continue; - } - // ok, take next 2 hex digits. - char hd1 = in.charAt(i + 2); - char hd2 = in.charAt(i + 3); - - // they need to be A-F0-9: - if (!isHexDigit(hd1) || !isHexDigit(hd2)) { - // bogus escape code, ignore: - continue; - } - // turn hex ASCII digit -> number - byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2)); - - b[size++] = d; - i += 3; // skip 3 - } else { - b[size++] = (byte) ch; - } - } - // resize: - byte[] b2 = new byte[size]; - System.arraycopy(b, 0, b2, 0, size); - return b2; - } - - /** - * Converts a string to a UTF-8 byte array. - * - * @param s string - * @return the byte array - */ - public static byte[] toBytes(String s) { - try { - return s.getBytes(Constants.UTF8_ENCODING); - } catch (UnsupportedEncodingException e) { - LOG.error("UTF-8 not supported?", e); - return null; - } - } - - /** - * Convert a boolean to a byte array. True becomes -1 and false becomes 0. - * - * @param b value - * @return b encoded in a byte array. - */ - public static byte[] toBytes(final boolean b) { - return new byte[] { b ? (byte) -1 : (byte) 0 }; - } - - /** - * Reverses {@link #toBytes(boolean)} - * - * @param b array - * @return True or false. - */ - public static boolean toBoolean(final byte[] b) { - if (b.length != 1) { - throw new IllegalArgumentException("Array has wrong size: " + b.length); - } - return b[0] != (byte) 0; - } - - /** - * Convert a long value to a byte array using big-endian. - * - * @param val value to convert - * @return the byte array - */ - public static byte[] toBytes(long val) { - byte[] b = new byte[8]; - for (int i = 7; i > 0; i--) { - b[i] = (byte) val; - val >>>= 8; - } - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to a long value. Reverses {@link #toBytes(long)} - * - * @param bytes array - * @return the long value - */ - public static long toLong(byte[] bytes) { - return toLong(bytes, 0, SIZEOF_LONG); - } - - /** - * Converts a byte array to a long value. Assumes there will be - * {@link #SIZEOF_LONG} bytes available. - * - * @param bytes bytes - * @param offset offset - * @return the long value - */ - public static long toLong(byte[] bytes, int offset) { - return toLong(bytes, offset, SIZEOF_LONG); - } - - /** - * Converts a byte array to a long value. - * - * @param bytes array of bytes - * @param offset offset into array - * @param length length of data (must be {@link #SIZEOF_LONG}) - * @return the long value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or - * if there's not enough room in the array at the offset indicated. - */ - public static long toLong(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_LONG || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); - } - long l = 0; - for (int i = offset; i < offset + length; i++) { - l <<= 8; - l ^= bytes[i] & 0xFF; - } - return l; - } - - private static IllegalArgumentException explainWrongLengthOrOffset( - final byte[] bytes, final int offset, final int length, - final int expectedLength) { - String reason; - if (length != expectedLength) { - reason = "Wrong length: " + length + ", expected " + expectedLength; - } else { - reason = "offset (" + offset + ") + length (" + length + ") exceed the" - + " capacity of the array: " + bytes.length; - } - return new IllegalArgumentException(reason); - } - - /** - * Put a long value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val long to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putLong(byte[] bytes, int offset, long val) { - if (bytes.length - offset < SIZEOF_LONG) { - throw new IllegalArgumentException("Not enough room to put a long at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - for (int i = offset + 7; i > offset; i--) { - bytes[i] = (byte) val; - val >>>= 8; - } - bytes[offset] = (byte) val; - return offset + SIZEOF_LONG; - } - - /** - * Presumes float encoded as IEEE 754 floating-point "single format" - * - * @param bytes byte array - * @return Float made from passed byte array. - */ - public static float toFloat(byte[] bytes) { - return toFloat(bytes, 0); - } - - /** - * Presumes float encoded as IEEE 754 floating-point "single format" - * - * @param bytes array to convert - * @param offset offset into array - * @return Float made from passed byte array. - */ - public static float toFloat(byte[] bytes, int offset) { - return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); - } - - /** - * @param bytes byte array - * @param offset offset to write to - * @param f float value - * @return New offset in bytes - */ - public static int putFloat(byte[] bytes, int offset, float f) { - return putInt(bytes, offset, Float.floatToRawIntBits(f)); - } - - /** - * @param f float value - * @return the float represented as byte [] - */ - public static byte[] toBytes(final float f) { - // Encode it as int - return Bytes.toBytes(Float.floatToRawIntBits(f)); - } - - /** - * @param bytes byte array - * @return Return double made from passed bytes. - */ - public static double toDouble(final byte[] bytes) { - return toDouble(bytes, 0); - } - - /** - * @param bytes byte array - * @param offset offset where double is - * @return Return double made from passed bytes. - */ - public static double toDouble(final byte[] bytes, final int offset) { - return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); - } - - /** - * @param bytes byte array - * @param offset offset to write to - * @param d value - * @return New offset into array bytes - */ - public static int putDouble(byte[] bytes, int offset, double d) { - return putLong(bytes, offset, Double.doubleToLongBits(d)); - } - - /** - * Serialize a double as the IEEE 754 double format output. The resultant - * array will be 8 bytes long. - * - * @param d value - * @return the double represented as byte [] - */ - public static byte[] toBytes(final double d) { - // Encode it as a long - return Bytes.toBytes(Double.doubleToRawLongBits(d)); - } - - /** - * Convert an int value to a byte array - * - * @param val value - * @return the byte array - */ - public static byte[] toBytes(int val) { - byte[] b = new byte[4]; - for (int i = 3; i > 0; i--) { - b[i] = (byte) val; - val >>>= 8; - } - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to an int value - * - * @param bytes byte array - * @return the int value - */ - public static int toInt(byte[] bytes) { - return toInt(bytes, 0, SIZEOF_INT); - } - - /** - * Converts a byte array to an int value - * - * @param bytes byte array - * @param offset offset into array - * @return the int value - */ - public static int toInt(byte[] bytes, int offset) { - return toInt(bytes, offset, SIZEOF_INT); - } - - /** - * Converts a byte array to an int value - * - * @param bytes byte array - * @param offset offset into array - * @param length length of int (has to be {@link #SIZEOF_INT}) - * @return the int value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or if - * there's not enough room in the array at the offset indicated. - */ - public static int toInt(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_INT || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); - } - int n = 0; - for (int i = offset; i < (offset + length); i++) { - n <<= 8; - n ^= bytes[i] & 0xFF; - } - return n; - } - - /** - * Put an int value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val int to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putInt(byte[] bytes, int offset, int val) { - if (bytes.length - offset < SIZEOF_INT) { - throw new IllegalArgumentException("Not enough room to put an int at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - for (int i = offset + 3; i > offset; i--) { - bytes[i] = (byte) val; - val >>>= 8; - } - bytes[offset] = (byte) val; - return offset + SIZEOF_INT; - } - - /** - * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long. - * - * @param val value - * @return the byte array - */ - public static byte[] toBytes(short val) { - byte[] b = new byte[SIZEOF_SHORT]; - b[1] = (byte) val; - val >>= 8; - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to a short value - * - * @param bytes byte array - * @return the short value - */ - public static short toShort(byte[] bytes) { - return toShort(bytes, 0, SIZEOF_SHORT); - } - - /** - * Converts a byte array to a short value - * - * @param bytes byte array - * @param offset offset into array - * @return the short value - */ - public static short toShort(byte[] bytes, int offset) { - return toShort(bytes, offset, SIZEOF_SHORT); - } - - /** - * Converts a byte array to a short value - * - * @param bytes byte array - * @param offset offset into array - * @param length length, has to be {@link #SIZEOF_SHORT} - * @return the short value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} or - * if there's not enough room in the array at the offset indicated. - */ - public static short toShort(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_SHORT || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); - } - short n = 0; - n ^= bytes[offset] & 0xFF; - n <<= 8; - n ^= bytes[offset + 1] & 0xFF; - return n; - } - - /** - * Put a short value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val short to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putShort(byte[] bytes, int offset, short val) { - if (bytes.length - offset < SIZEOF_SHORT) { - throw new IllegalArgumentException("Not enough room to put a short at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - bytes[offset + 1] = (byte) val; - val >>= 8; - bytes[offset] = (byte) val; - return offset + SIZEOF_SHORT; - } - - /** - * @param vint Integer to make a vint of. - * @return Vint as bytes array. - */ - public static byte[] vintToBytes(final long vint) { - long i = vint; - int size = WritableUtils.getVIntSize(i); - byte[] result = new byte[size]; - int offset = 0; - if (i >= -112 && i <= 127) { - result[offset] = (byte) i; - return result; - } - - int len = -112; - if (i < 0) { - i ^= -1L; // take one's complement' - len = -120; - } - - long tmp = i; - while (tmp != 0) { - tmp = tmp >> 8; - len--; - } - - result[offset++] = (byte) len; - - len = (len < -120) ? -(len + 120) : -(len + 112); - - for (int idx = len; idx != 0; idx--) { - int shiftbits = (idx - 1) * 8; - long mask = 0xFFL << shiftbits; - result[offset++] = (byte) ((i & mask) >> shiftbits); - } - return result; - } - - /** - * @param buffer buffer to convert - * @return vint bytes as an integer. - */ - public static long bytesToVint(final byte[] buffer) { - int offset = 0; - byte firstByte = buffer[offset++]; - int len = WritableUtils.decodeVIntSize(firstByte); - if (len == 1) { - return firstByte; - } - long i = 0; - for (int idx = 0; idx < len - 1; idx++) { - byte b = buffer[offset++]; - i = i << 8; - i = i | (b & 0xFF); - } - return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); - } - - /** - * Reads a zero-compressed encoded long from input stream and returns it. - * - * @param buffer Binary array - * @param offset Offset into array at which vint begins. - * @throws java.io.IOException e - * @return deserialized long from stream. - */ - public static long readVLong(final byte[] buffer, final int offset) - throws IOException { - byte firstByte = buffer[offset]; - int len = WritableUtils.decodeVIntSize(firstByte); - if (len == 1) { - return firstByte; - } - long i = 0; - for (int idx = 0; idx < len - 1; idx++) { - byte b = buffer[offset + 1 + idx]; - i = i << 8; - i = i | (b & 0xFF); - } - return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); - } - - /** - * @param left left operand - * @param right right operand - * @return 0 if equal, < 0 if left is less than right, etc. - */ - public static int compareTo(final byte[] left, final byte[] right) { - return compareTo(left, 0, left.length, right, 0, right.length); - } - - /** - * Lexographically compare two arrays. - * - * @param b1 left operand - * @param b2 right operand - * @param s1 Where to start comparing in the left buffer - * @param s2 Where to start comparing in the right buffer - * @param l1 How much to compare from the left buffer - * @param l2 How much to compare from the right buffer - * @return 0 if equal, < 0 if left is less than right, etc. - */ - public static int compareTo(byte[] b1, int s1, int l1, byte[] b2, int s2, - int l2) { - // Bring WritableComparator code local - int end1 = s1 + l1; - int end2 = s2 + l2; - for (int i = s1, j = s2; i < end1 && j < end2; i++, j++) { - int a = (b1[i] & 0xff); - int b = (b2[j] & 0xff); - if (a != b) { - return a - b; - } - } - return l1 - l2; - } - - /** - * @param left left operand - * @param right right operand - * @return True if equal - */ - public static boolean equals(final byte[] left, final byte[] right) { - // Could use Arrays.equals? - // noinspection SimplifiableConditionalExpression - if (left == null && right == null) { - return true; - } - return (left == null || right == null || (left.length != right.length) ? false - : compareTo(left, right) == 0); - } - - /** - * @param b bytes to hash - * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the - * passed in array. This method is what - * {@link org.apache.hadoop.io.Text} use calculating hash code. - */ - public static int hashCode(final byte[] b) { - return hashCode(b, b.length); - } - - /** - * @param b value - * @param length length of the value - * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the - * passed in array. This method is what - * {@link org.apache.hadoop.io.Text} use calculating hash code. - */ - public static int hashCode(final byte[] b, final int length) { - return WritableComparator.hashBytes(b, length); - } - - /** - * @param b bytes to hash - * @return A hash of b as an Integer that can be used as key in - * Maps. - */ - public static Integer mapKey(final byte[] b) { - return hashCode(b); - } - - /** - * @param b bytes to hash - * @param length length to hash - * @return A hash of b as an Integer that can be used as key in - * Maps. - */ - public static Integer mapKey(final byte[] b, final int length) { - return hashCode(b, length); - } - - /** - * @param a lower half - * @param b upper half - * @return New array that has a in lower half and b in upper half. - */ - public static byte[] add(final byte[] a, final byte[] b) { - return add(a, b, Constants.EMPTY_BYTE_ARRAY); - } - - /** - * @param a first third - * @param b second third - * @param c third third - * @return New array made from a, b and c - */ - public static byte[] add(final byte[] a, final byte[] b, final byte[] c) { - byte[] result = new byte[a.length + b.length + c.length]; - System.arraycopy(a, 0, result, 0, a.length); - System.arraycopy(b, 0, result, a.length, b.length); - System.arraycopy(c, 0, result, a.length + b.length, c.length); - return result; - } - - /** - * @param a array - * @param length amount of bytes to grab - * @return First length bytes from a - */ - public static byte[] head(final byte[] a, final int length) { - if (a.length < length) { - return null; - } - byte[] result = new byte[length]; - System.arraycopy(a, 0, result, 0, length); - return result; - } - - /** - * @param a array - * @param length amount of bytes to snarf - * @return Last length bytes from a - */ - public static byte[] tail(final byte[] a, final int length) { - if (a.length < length) { - return null; - } - byte[] result = new byte[length]; - System.arraycopy(a, a.length - length, result, 0, length); - return result; - } - - /** - * @param a array - * @param length new array size - * @return Value in a plus length prepended 0 bytes - */ - public static byte[] padHead(final byte[] a, final int length) { - byte[] padding = new byte[length]; - for (int i = 0; i < length; i++) { - padding[i] = 0; - } - return add(padding, a); - } - - /** - * @param a array - * @param length new array size - * @return Value in a plus length appended 0 bytes - */ - public static byte[] padTail(final byte[] a, final int length) { - byte[] padding = new byte[length]; - for (int i = 0; i < length; i++) { - padding[i] = 0; - } - return add(a, padding); - } - - /** - * Split passed range. Expensive operation relatively. Uses BigInteger math. - * Useful splitting ranges for MapReduce jobs. - * - * @param a Beginning of range - * @param b End of range - * @param num Number of times to split range. Pass 1 if you want to split the - * range in two; i.e. one split. - * @return Array of dividing values - */ - public static byte[][] split(final byte[] a, final byte[] b, final int num) { - byte[] aPadded; - byte[] bPadded; - if (a.length < b.length) { - aPadded = padTail(a, b.length - a.length); - bPadded = b; - } else if (b.length < a.length) { - aPadded = a; - bPadded = padTail(b, a.length - b.length); - } else { - aPadded = a; - bPadded = b; - } - if (compareTo(aPadded, bPadded) >= 0) { - throw new IllegalArgumentException("b <= a"); - } - if (num <= 0) { - throw new IllegalArgumentException("num cannot be < 0"); - } - byte[] prependHeader = { 1, 0 }; - BigInteger startBI = new BigInteger(add(prependHeader, aPadded)); - BigInteger stopBI = new BigInteger(add(prependHeader, bPadded)); - BigInteger diffBI = stopBI.subtract(startBI); - BigInteger splitsBI = BigInteger.valueOf(num + 1); - if (diffBI.compareTo(splitsBI) < 0) { - return null; - } - BigInteger intervalBI; - try { - intervalBI = diffBI.divide(splitsBI); - } catch (Exception e) { - LOG.error("Exception caught during division", e); - return null; - } - - byte[][] result = new byte[num + 2][]; - result[0] = a; - - for (int i = 1; i <= num; i++) { - BigInteger curBI = startBI - .add(intervalBI.multiply(BigInteger.valueOf(i))); - byte[] padded = curBI.toByteArray(); - if (padded[1] == 0) - padded = tail(padded, padded.length - 2); - else - padded = tail(padded, padded.length - 1); - result[i] = padded; - } - result[num + 1] = b; - return result; - } - - /** - * @param t operands - * @return Array of byte arrays made from passed array of Text - */ - public static byte[][] toByteArrays(final String[] t) { - byte[][] result = new byte[t.length][]; - for (int i = 0; i < t.length; i++) { - result[i] = Bytes.toBytes(t[i]); - } - return result; - } - - /** - * @param column operand - * @return A byte array of a byte array where first and only entry is - * column - */ - public static byte[][] toByteArrays(final String column) { - return toByteArrays(toBytes(column)); - } - - /** - * @param column operand - * @return A byte array of a byte array where first and only entry is - * column - */ - public static byte[][] toByteArrays(final byte[] column) { - byte[][] result = new byte[1][]; - result[0] = column; - return result; - } - - /** - * Binary search for keys in indexes. - * - * @param arr array of byte arrays to search for - * @param key the key you want to find - * @param offset the offset in the key you want to find - * @param length the length of the key - * @param comparator a comparator to compare. - * @return index of key - */ - public static int binarySearch(byte[][] arr, byte[] key, int offset, - int length, RawComparator comparator) { - int low = 0; - int high = arr.length - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - // we have to compare in this order, because the comparator order - // has special logic when the 'left side' is a special key. - int cmp = comparator.compare(key, offset, length, arr[mid], 0, - arr[mid].length); - // key lives above the midpoint - if (cmp > 0) - low = mid + 1; - // key lives below the midpoint - else if (cmp < 0) - high = mid - 1; - // BAM. how often does this really happen? - else - return mid; - } - return -(low + 1); - } - - /** - * Bytewise binary increment/deincrement of long contained in byte array on - * given amount. - * - * @param value - array of bytes containing long (length <= SIZEOF_LONG) - * @param amount value will be incremented on (deincremented if negative) - * @return array of bytes containing incremented long (length == SIZEOF_LONG) - * @throws IOException - if value.length > SIZEOF_LONG - */ - public static byte[] incrementBytes(byte[] value, long amount) - throws IOException { - byte[] val = value; - if (val.length < SIZEOF_LONG) { - // Hopefully this doesn't happen too often. - byte[] newvalue; - if (val[0] < 0) { - newvalue = new byte[] { -1, -1, -1, -1, -1, -1, -1, -1 }; - } else { - newvalue = new byte[SIZEOF_LONG]; - } - System.arraycopy(val, 0, newvalue, newvalue.length - val.length, - val.length); - val = newvalue; - } else if (val.length > SIZEOF_LONG) { - throw new IllegalArgumentException("Increment Bytes - value too big: " - + val.length); - } - if (amount == 0) - return val; - if (val[0] < 0) { - return binaryIncrementNeg(val, amount); - } - return binaryIncrementPos(val, amount); - } - - /* increment/deincrement for positive value */ - private static byte[] binaryIncrementPos(byte[] value, long amount) { - long amo = amount; - int sign = 1; - if (amount < 0) { - amo = -amount; - sign = -1; - } - for (int i = 0; i < value.length; i++) { - int cur = ((int) amo % 256) * sign; - amo = (amo >> 8); - int val = value[value.length - i - 1] & 0x0ff; - int total = val + cur; - if (total > 255) { - amo += sign; - total %= 256; - } else if (total < 0) { - amo -= sign; - } - value[value.length - i - 1] = (byte) total; - if (amo == 0) - return value; - } - return value; - } - - /* increment/deincrement for negative value */ - private static byte[] binaryIncrementNeg(byte[] value, long amount) { - long amo = amount; - int sign = 1; - if (amount < 0) { - amo = -amount; - sign = -1; - } - for (int i = 0; i < value.length; i++) { - int cur = ((int) amo % 256) * sign; - amo = (amo >> 8); - int val = ((~value[value.length - i - 1]) & 0x0ff) + 1; - int total = cur - val; - if (total >= 0) { - amo += sign; - } else if (total < -256) { - amo -= sign; - total %= 256; - } - value[value.length - i - 1] = (byte) total; - if (amo == 0) - return value; - } - return value; - } - -} Index: src/java/org/apache/hama/util/RandomVariable.java =================================================================== --- src/java/org/apache/hama/util/RandomVariable.java (revision 1152138) +++ src/java/org/apache/hama/util/RandomVariable.java (working copy) @@ -1,227 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -/** - * The RandomVaraibale Class provides static methods for generating random - * numbers. - */ -public class RandomVariable { - - /** - * Generate a random number between 0 and 1. - * - * @return a double between 0 and 1. - */ - public static double rand() { - double x = Math.random(); - return x; - } - - /** - * Generate a random integer. - * - * @param i0 min of the random variable. - * @param i1 max of the random variable. - * @return an int between i0 and i1. - */ - public static int randInt(int i0, int i1) { - double x = rand(); - int i = i0 + (int) Math.floor((i1 - i0 + 1) * x); - return i; - } - - /** - * Generate a random string using the specified prefix and a fixed length. - * @param prefix - * the specified string prefix. - * @param length - * the length of the string to be appended. - * @return random string. - */ - public static String randString(String prefix, int length) { - StringBuilder result = new StringBuilder(prefix); - for (int i = 0; i < length; i++) { - char ch = (char) ((Math.random() * 26) + 97); - result.append(ch); - } - - return result.toString(); - } - - /** - * Generate a random number from a uniform random variable. - * - * @param min min of the random variable. - * @param max max of the random variable. - * @return a double. - */ - public static double uniform(double min, double max) { - double x = min + (max - min) * rand(); - return x; - } - - /** - * Generate a random number from a discrete random variable. - * - * @param values discrete values. - * @param prob probability of each value. - * @return a double. - */ - public static double dirac(double[] values, double[] prob) { - double[] prob_cumul = new double[values.length]; - prob_cumul[0] = prob[0]; - for (int i = 1; i < values.length; i++) { - prob_cumul[i] = prob_cumul[i - 1] + prob[i]; - } - double y = rand(); - double x = 0; - for (int i = 0; i < values.length - 1; i++) { - if ((y > prob_cumul[i]) && (y < prob_cumul[i + 1])) { - x = values[i]; - } - } - return x; - } - - /** - * Generate a random number from a Gaussian (Normal) random variable. - * - * @param mu mean of the random variable. - * @param sigma standard deviation of the random variable. - * @return a double. - */ - public static double normal(double mu, double sigma) { - double x = mu + sigma * Math.cos(2 * Math.PI * rand()) - * Math.sqrt(-2 * Math.log(rand())); - return x; - } - - /** - * Generate a random number from a Chi-2 random variable. - * - * @param n degrees of freedom of the chi2 random variable. - * @return a double. - */ - public static double chi2(int n) { - double x = 0; - for (int i = 0; i < n; i++) { - double norm = normal(0, 1); - x += norm * norm; - } - return x; - } - - /** - * Generate a random number from a LogNormal random variable. - * - * @param mu mean of the Normal random variable. - * @param sigma standard deviation of the Normal random variable. - * @return a double. - */ - public static double logNormal(double mu, double sigma) { - double x = mu + sigma * Math.cos(2 * Math.PI * rand()) - * Math.sqrt(-2 * Math.log(rand())); - return x; - } - - /** - * Generate a random number from an exponantial random variable (Mean = - * 1/lambda, variance = 1/lambda^2). - * - * @param lambda parmaeter of the exponential random variable. - * @return a double. - */ - public static double exponential(double lambda) { - double x = -1 / lambda * Math.log(rand()); - return x; - } - - /** - * Generate a random number from a symetric triangular random variable. - * - * @param min min of the random variable. - * @param max max of the random variable. - * @return a double. - */ - public static double triangular(double min, double max) { - double x = min / 2 + (max - min) * rand() / 2 + min / 2 + (max - min) - * rand() / 2; - return x; - } - - /** - * Generate a random number from a non-symetric triangular random variable. - * - * @param min min of the random variable. - * @param med value of the random variable with max density. - * @param max max of the random variable. - * @return a double. - */ - public static double triangular(double min, double med, double max) { - double y = rand(); - double x = (y < ((med - min) / (max - min))) ? (min + Math.sqrt(y - * (max - min) * (med - min))) : (max - Math.sqrt((1 - y) * (max - min) - * (max - med))); - return x; - } - - /** - * Generate a random number from a beta random variable. - * - * @param a first parameter of the Beta random variable. - * @param b second parameter of the Beta random variable. - * @return a double. - */ - public static double beta(double a, double b) { - double try_x; - double try_y; - do { - try_x = Math.pow(rand(), 1 / a); - try_y = Math.pow(rand(), 1 / b); - } while ((try_x + try_y) > 1); - return try_x / (try_x + try_y); - } - - /** - * Generate a random number from a Cauchy random variable (Mean = Inf, and - * Variance = Inf). - * - * @param mu median of the Weibull random variable - * @param sigma second parameter of the Cauchy random variable. - * @return a double. - */ - public static double cauchy(double mu, double sigma) { - double x = sigma * Math.tan(Math.PI * (rand() - 0.5)) + mu; - return x; - } - - /** - * Generate a random number from a Weibull random variable. - * - * @param lambda first parameter of the Weibull random variable. - * @param c second parameter of the Weibull random variable. - * @return a double. - */ - public static double weibull(double lambda, double c) { - double x = Math.pow(-Math.log(1 - rand()), 1 / c) / lambda; - return x; - } -} Index: src/java/org/apache/hama/util/BSPServletUtil.java =================================================================== --- src/java/org/apache/hama/util/BSPServletUtil.java (revision 1152138) +++ src/java/org/apache/hama/util/BSPServletUtil.java (working copy) @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -import java.io.IOException; -import java.util.Date; -import java.util.Calendar; - -import org.apache.hadoop.util.ServletUtil; -import org.apache.hama.bsp.JobStatus; - -public class BSPServletUtil extends ServletUtil { - - public static final String HTML_TAIL = "
\n" - + "Hama, " - + Calendar.getInstance().get(Calendar.YEAR) + ".\n" + ""; - - /** - * HTML footer to be added in the jsps. - * - * @return the HTML footer. - */ - public static String htmlFooter() { - return HTML_TAIL; - } - - /** - * Method used to generate the Job table for Job pages. - * - * @param label display heading to be used in the job table. - * @param jobs vector of jobs to be displayed in table. - * @param refresh refresh interval to be used in jobdetails page. - * @param rowId beginning row id to be used in the table. - * @return generated HTML - * @throws IOException - */ - public static String generateJobTable(String label, JobStatus[] jobs, - int refresh, int rowId) throws IOException { - - StringBuffer sb = new StringBuffer(); - - if (jobs.length > 0) { - sb.append("\n"); - sb.append("" + "" + "" - + "" + "" + "\n"); - for (JobStatus status : jobs) { - sb.append("\n"); - } - sb.append("
JobidUserNameSuperStepStarttime
"); - sb.append(status.getJobID()); - sb.append(""); - sb.append(status.getUsername()); - sb.append(""); - sb.append(status.getName()); - sb.append(""); - sb.append(status.getSuperstepCount()); - sb.append(""); - sb.append(new Date(status.getStartTime())); - sb.append("
"); - } else { - sb.append("No jobs found!"); - } - - return sb.toString(); - } - -} Index: src/java/org/apache/hama/util/package.html =================================================================== --- src/java/org/apache/hama/util/package.html (revision 1152138) +++ src/java/org/apache/hama/util/package.html (working copy) @@ -1,23 +0,0 @@ - - - - - -Common utilities. - - Index: src/java/org/apache/hama/HamaConfiguration.java =================================================================== --- src/java/org/apache/hama/HamaConfiguration.java (revision 1152138) +++ src/java/org/apache/hama/HamaConfiguration.java (working copy) @@ -1,61 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import java.util.Map.Entry; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; - -/** - * Adds Hama configuration files to a Configuration. - */ -public class HamaConfiguration extends Configuration { - /** constructor */ - public HamaConfiguration() { - super(); - addHamaResources(); - } - - public HamaConfiguration(Path confFile) { - super(); - this.addResource(confFile); - } - - /** - * Create a clone of passed configuration. - * - * @param c Configuration to clone. - */ - public HamaConfiguration(final Configuration c) { - this(); - for (Entry e : c) { - set(e.getKey(), e.getValue()); - } - } - - /** - * Adds Hama configuration files to a Configuration - */ - private void addHamaResources() { - Configuration.addDefaultResource("hama-default.xml"); - Configuration.addDefaultResource("hama-site.xml"); - } -} Index: src/java/org/apache/hama/package.html =================================================================== --- src/java/org/apache/hama/package.html (revision 1152138) +++ src/java/org/apache/hama/package.html (working copy) @@ -1,23 +0,0 @@ - - - - - -Hama base package. - - Index: src/java/overview.html =================================================================== --- src/java/overview.html (revision 1152138) +++ src/java/overview.html (working copy) @@ -1,152 +0,0 @@ - - - - - - Hama - - -Hama is a distributed computing framework based on BSP (Bulk Synchronous Parallel) computing techniques for massive scientific computations. - -

Requirements

-
    -
  • hadoop-0.20.x for HDFS
  • -
  • Sun Java JDK 1.6.x or higher version
  • -
- -

Startup script

-The $HAMA_HOME/bin directory contains some script used to start up the Hama daemons. -
    -
  • -start-bspd.sh - Starts all Hama daemons, the BSPMaster, GroomServers and Zookeeper. -
  • -
- -

Configuration files

- -The $HAMA_HOME/conf directory contains some configuration files for Hama. These are: -
    -
  • -hama-env.sh - This file contains some environment variable settings used by Hama. You can use these to affect some aspects of Hama daemon behavior, -such as where log files are stored, the maximum amount of heap used etc. The only variable you should need to change in this file is JAVA_HOME, -which specifies the path to the Java 1.5.x installation used by Hama. -
  • -
  • -groomservers - This file lists the hosts, one per line, where the GroomServer daemons will run. By default this contains the single entry localhost.
  • -
  • -hama-default.xml - This file contains generic default settings for Hama daemons. Do not modify this file. -
  • -
  • -hama-site.xml - This file contains site specific settings for all Hama daemons and BSP jobs. -This file is empty by default. Settings in this file override those in hama-default.xml. -This file should contain settings that must be respected by all servers and clients in a Hama installation. -
  • -
- -

Setting up Hama

-This section describes how to get started by setting up a Hama cluster. -
    -
  • -BSPMaster and Zookeeper settings - Figure out where to run your HDFS namenode and BSPMaster. -Set the variable bsp.master.address to the BSPMaster's intended host:port. -Set the variable fs.default.name to the HDFS Namenode's intended host:port. -
  • -
- -An example of a hama-site.xml file: -
-<!--?xml version="1.0"?-->
-<!--?xml-stylesheet type="text/xsl" href="configuration.xsl"?-->
-<configuration>
-  <property>
-    <name>bsp.master.address</name>
-    <value>host1.mydomain.com:40000</value>
-    <description>The address of the bsp master server. Either the
-    literal string "local" or a host:port for distributed mode
-    </description>
-  </property>
-  
-  <property>
-    <name>fs.default.name</name>
-    <value>hdfs://host1.mydomain.com:9000/</value>
-    <description>
-      The name of the default file system. Either the literal string
-      "local" or a host:port for HDFS.
-    </description>
-  </property>
-  
-  <property>
-    <name>hama.zookeeper.quorum</name>
-    <value>host1.mydomain.com</value>
-    <description>Comma separated list of servers in the ZooKeeper Quorum.
-    For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
-    By default this is set to localhost for local and pseudo-distributed modes
-    of operation. For a fully-distributed setup, this should be set to a full
-    list of ZooKeeper quorum servers. If HAMA_MANAGES_ZK is set in hama-env.sh
-    this is the list of servers which we will start/stop zookeeper on.
-    </description>
-  </property>
-</configuration>
-
- -If you are managing your own ZooKeeper, you have to specify the port number as below: - -
-  <property>
-    <name>hama.zookeeper.property.clientPort</name>
-    <value>2181</value>
-  </property>
-
- - - -

Starting a Hama cluster

- -Run the command: -
-% $HAMA_HOME/bin/start-bspd.sh
-
- -This will startup a BSPMaster, GroomServers and Zookeeper on your machine. - -

Stopping a Hama cluster

- -Run the command: -
-% $HAMA_HOME/bin/stop-bspd.sh
-
- -to stop all the daemons running on your cluster. - -

Run the BSP Examples

-Run the command: -
-% $HAMA_HOME/bin/hama jar hama-examples-0.x.0-incubating.jar
-
- -

Hama Web Interfaces

- -The web UI provides information about BSP job statistics of the Hama cluster, running/completed/failed jobs. -

-By default, it’s available at http://localhost:40013 - - - - Index: src/webapps/bspmaster/machines.jsp =================================================================== --- src/webapps/bspmaster/machines.jsp (revision 1152138) +++ src/webapps/bspmaster/machines.jsp (working copy) @@ -1,61 +0,0 @@ - -<%@ page contentType="text/html; charset=UTF-8" import="javax.servlet.*" - import="javax.servlet.http.*" import="java.io.*" import="java.util.*" - import="java.text.DecimalFormat" import="org.apache.hama.bsp.*" - import="org.apache.hama.util.*"%> -<%!private static final long serialVersionUID = 1L;%> -<% - BSPMaster tracker = (BSPMaster) application - .getAttribute("bsp.master"); - ClusterStatus status = tracker.getClusterStatus(true); - String trackerName = tracker.getBSPMasterName(); - String type = request.getParameter("type"); -%> -<%!public void generateGroomsTable(JspWriter out, String type, - ClusterStatus status, BSPMaster master) throws IOException { - out.print("
\n"); - out.print("\n"); - out.print("\n"); - out.print("" + "" - + "\n"); - for (Map.Entry entry : status.getActiveGroomNames() - .entrySet()) { - out.print("" + "\n"); - } - out.print("
Groom Servers
NameHost# running tasks
"); - out.print(entry.getValue() + ""); - out.print(entry.getValue() + "" + 1 + "
\n"); - out.print("
\n"); - }%> - - - -<%=trackerName%> Hama Machine List - - -

<%=trackerName%> Hama Machine List

- -

Grooms

-<% - generateGroomsTable(out, type, status, tracker); -%> - -<% - out.println(BSPServletUtil.htmlFooter()); -%> \ No newline at end of file Index: src/webapps/bspmaster/bspjob.jsp =================================================================== --- src/webapps/bspmaster/bspjob.jsp (revision 1152138) +++ src/webapps/bspmaster/bspjob.jsp (working copy) @@ -1,66 +0,0 @@ - -<%@ page contentType="text/html; charset=UTF-8" import="javax.servlet.*" - import="javax.servlet.http.*" import="java.io.*" import="java.util.*" - import="java.text.DecimalFormat" import="org.apache.hama.bsp.*" - import="org.apache.hama.util.*"%> -<%!private static final long serialVersionUID = 1L;%> -<% - BSPMaster tracker = (BSPMaster) application - .getAttribute("bsp.master"); - String idString = request.getParameter("jobid"); - JobStatus status = tracker.getJobStatus(BSPJobID.forName(idString)); - JobStatus.State state = status.getState(); -%> - - - -Hama BSP Job Summary - - -

<%=status.getName()%>

- - State: <%=state.toString() %> - -

- - - - - - - - - - - - - - - - - -
NameUserSuperStepStartTimeFinishTime
<%=status.getName() %><%=status.getUsername() %><%=status.getSuperstepCount() %><%=new Date(status.getStartTime()).toString() %> - <% if(status.getFinishTime() != 0L) {out.write(new Date(status.getFinishTime()).toString());} %> -
- -
- Back to BSPMaster - - <% - out.println(BSPServletUtil.htmlFooter()); - %> \ No newline at end of file Index: src/webapps/bspmaster/index.html =================================================================== --- src/webapps/bspmaster/index.html (revision 1152138) +++ src/webapps/bspmaster/index.html (working copy) @@ -1,36 +0,0 @@ - - - - - -Hama Administration - - - - -

Hama Administration

- - - - - - Index: src/webapps/bspmaster/bspmaster.jsp =================================================================== --- src/webapps/bspmaster/bspmaster.jsp (revision 1152138) +++ src/webapps/bspmaster/bspmaster.jsp (working copy) @@ -1,82 +0,0 @@ - -<%@ page contentType="text/html; charset=UTF-8" import="javax.servlet.*" - import="javax.servlet.http.*" import="java.io.*" import="java.util.*" - import="java.text.DecimalFormat" import="org.apache.hama.bsp.*" - import="org.apache.hama.util.*"%> -<%!private static final long serialVersionUID = 1L;%> -<% - BSPMaster tracker = (BSPMaster) application - .getAttribute("bsp.master"); - ClusterStatus status = tracker.getClusterStatus(true); - String trackerName = tracker.getBSPMasterName(); - JobStatus[] runningJobs = tracker.jobsToComplete(); - JobStatus[] allJobs = tracker.getAllJobs(); -%> -<%!private static DecimalFormat percentFormat = new DecimalFormat("##0.00"); - - public void generateSummaryTable(JspWriter out, ClusterStatus status, - BSPMaster tracker) throws IOException { - String tasksPerNode = status.getGroomServers() > 0 ? percentFormat - .format(((double) (status.getMaxTasks()) / status - .getGroomServers())) : "-"; - out.print("\n" - + "" + "" - + "" - + "\n"); - out.print("
Groom ServersBSP Task CapacityAvg. Tasks/NodeBlacklisted Nodes
" - + status.getActiveGroomNames().size() + "" - + status.getMaxTasks() + "" + tasksPerNode - + "" + 0 - + "" + "
\n"); - - out.print("
"); - }%> - - - - -<%=trackerName%> Hama BSP Administration - - - - -

<%=trackerName%> Hama BSP Administration

- -State: -<%=status.getBSPMasterState()%>
-Started: -<%=new Date(tracker.getStartTime())%>
-Identifier: -<%=tracker.getBSPMasterIdentifier()%>
- -
-<% - generateSummaryTable(out, status, tracker); -%> -
- -

Running Jobs

-<%=BSPServletUtil.generateJobTable("Running", runningJobs, - 30, 0)%> -
-

All Jobs History

-<%=BSPServletUtil.generateJobTable("All", allJobs, - 30, 0)%> -<% - out.println(BSPServletUtil.htmlFooter()); -%> \ No newline at end of file Index: src/docs/forrest.properties =================================================================== --- src/docs/forrest.properties (revision 1152138) +++ src/docs/forrest.properties (working copy) @@ -1,104 +0,0 @@ -# Copyright 2002-2004 The Apache Software Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -############## -# Properties used by forrest.build.xml for building the website -# These are the defaults, un-comment them if you need to change them. -############## - -# Prints out a summary of Forrest settings for this project -#forrest.echo=true - -# Project name (used to name .war file) -#project.name=my-project - -# Specifies name of Forrest skin to use -#project.skin=tigris -#project.skin=pelt - -# comma separated list, file:// is supported -#forrest.skins.descriptors=http://forrest.apache.org/skins/skins.xml,file:///c:/myskins/skins.xml - -############## -# behavioural properties -#project.menu-scheme=tab_attributes -#project.menu-scheme=directories - -############## -# layout properties - -# Properties that can be set to override the default locations -# -# Parent properties must be set. This usually means uncommenting -# project.content-dir if any other property using it is uncommented - -#project.status=status.xml -#project.content-dir=src/documentation -#project.raw-content-dir=${project.content-dir}/content -#project.conf-dir=${project.content-dir}/conf -#project.sitemap-dir=${project.content-dir} -#project.xdocs-dir=${project.content-dir}/content/xdocs -#project.resources-dir=${project.content-dir}/resources -#project.stylesheets-dir=${project.resources-dir}/stylesheets -#project.images-dir=${project.resources-dir}/images -#project.schema-dir=${project.resources-dir}/schema -#project.skins-dir=${project.content-dir}/skins -#project.skinconf=${project.content-dir}/skinconf.xml -#project.lib-dir=${project.content-dir}/lib -#project.classes-dir=${project.content-dir}/classes -#project.translations-dir=${project.content-dir}/translations - -############## -# validation properties - -# This set of properties determine if validation is performed -# Values are inherited unless overridden. -# e.g. if forrest.validate=false then all others are false unless set to true. -forrest.validate=false -#forrest.validate.xdocs=${forrest.validate} -#forrest.validate.skinconf=${forrest.validate} -#forrest.validate.sitemap=${forrest.validate} -#forrest.validate.stylesheets=${forrest.validate} -#forrest.validate.skins=${forrest.validate} -#forrest.validate.skins.stylesheets=${forrest.validate.skins} - -# *.failonerror=(true|false) - stop when an XML file is invalid -#forrest.validate.failonerror=true - -# *.excludes=(pattern) - comma-separated list of path patterns to not validate -# e.g. -#forrest.validate.xdocs.excludes=samples/subdir/**, samples/faq.xml -#forrest.validate.xdocs.excludes= - - -############## -# General Forrest properties - -# The URL to start crawling from -#project.start-uri=linkmap.html -# Set logging level for messages printed to the console -# (DEBUG, INFO, WARN, ERROR, FATAL_ERROR) -#project.debuglevel=ERROR -# Max memory to allocate to Java -#forrest.maxmemory=64m -# Any other arguments to pass to the JVM. For example, to run on an X-less -# server, set to -Djava.awt.headless=true -#forrest.jvmargs= -# The bugtracking URL - the issue number will be appended -#project.bugtracking-url=http://issues.apache.org/bugzilla/show_bug.cgi?id= -#project.bugtracking-url=http://issues.apache.org/jira/browse/ -# The issues list as rss -#project.issues-rss-url= -#I18n Property only works for the "forrest run" target. -#project.i18n=true Index: src/docs/status.xml =================================================================== --- src/docs/status.xml (revision 1152138) +++ src/docs/status.xml (working copy) @@ -1,74 +0,0 @@ - - - - - - - - - - - - - - - - Initial Import - - - - - - - - - Customize this template project with your project's details. This - TODO list is generated from 'status.xml'. - - - Add lots of content. XML content goes in - src/documentation/content/xdocs, or wherever the - ${project.xdocs-dir} property (set in - forrest.properties) points. - - - Mail forrest-dev@xml.apache.org - with feedback. - - - - - - Index: src/docs/src/documentation/skinconf.xml =================================================================== --- src/docs/src/documentation/skinconf.xml (revision 1152138) +++ src/docs/src/documentation/skinconf.xml (working copy) @@ -1,354 +0,0 @@ - - - - - - - - - - - - - true - - false - - true - - true - - - true - - - true - - - true - - - false - - - true - - - Apache Hama - - index.html - images/hama_paint_logo_120x120.png - - - Apache Incubator - - http://incubator.apache.org/ - images/apache-incubator-logo.png - - - - - - - images/favicon.ico - - - 2008 - The Apache Software Foundation. - http://www.apache.org/licenses/ - - - - - - - - - - - - - - - - - - - - - #footer a { color: black; } - img.logo { - float: left; - margin-right: 0.5em; - } - - - - - - - - - - - - - - - - - - - - - - - - 1in - 1in - 1.25in - 1in - - - - false - - - false - - - - - - Built with Apache Forrest - http://forrest.apache.org/ - images/built-with-forrest-button.png - 88 - 31 - - - - - - Index: src/docs/src/documentation/content/xdocs/site.xml =================================================================== --- src/docs/src/documentation/content/xdocs/site.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/site.xml (working copy) @@ -1,84 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Index: src/docs/src/documentation/content/xdocs/irc_channel.xml =================================================================== --- src/docs/src/documentation/content/xdocs/irc_channel.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/irc_channel.xml (working copy) @@ -1,46 +0,0 @@ - - - - - - - -
- IRC Channel -
- - - -
- IRC Channel -

- There is an IRC channel dedicated to Hama at irc.freenode.org. - The name of the channel is #hama. -

- The IRC channel can be used for online discussion about hama related stuff, - but developers should be careful to transfer all the official decisions - or useful discussions to the issue tracking - system. -

- -
- -
Index: src/docs/src/documentation/content/xdocs/index.xml =================================================================== --- src/docs/src/documentation/content/xdocs/index.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/index.xml (working copy) @@ -1,91 +0,0 @@ - - - - - - - -
- Welcome to Hama project -
- -
- Overview -

- Hama is a distributed computing framework based on - BSP (Bulk Synchronous Parallel) computing techniques for massive scientific computations, such as Google's Pregel, - Currently being incubated as one of the incubator project by the Apache Software Foundation. -

-
- -
- Getting Started -

- Start by installing Hama on a Hadoop cluster. -
-

- -
- -
- Getting Involved -

- Hama is an open source volunteer project under the Apache Software Foundation. We encourage you to learn about the project and contribute your expertise. - Here are some starter links:
-

- -
- -
- Recent News -
- July 28, 2011: release 0.3.0 available -
    -
  • Added LockBSPRunner
  • -
  • Added web UI for BSP cluster and job monitoring
  • -
  • Added more practical examples e.g., Shortest Path Problem, PageRank
  • -
  • Performance has improved by more than 25% with BSPMessageBundle
  • -
  • Switched from Ant to Maven
  • -
-
-
- June 2, 2011: release 0.2.0 available -
    -
  • BSP computing framework and its examples -
    • Pi Estimator and Serialize Printing
  • -
  • CLI-based managing and monitoring tool of BSP job
  • -
-
-
- Apr 30, 2010: Introduced BSP Worldwide -

We're introduced in the BSP Worldwide.

-
-
- May 20, 2008: Accept Hama to be a Apache incubator project -

The Incubator PMC have voted to accept Hama to be a Apache incubator project.

-
-
- - -
Index: src/docs/src/documentation/content/xdocs/developers.xml =================================================================== --- src/docs/src/documentation/content/xdocs/developers.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/developers.xml (working copy) @@ -1,68 +0,0 @@ - - - - - - - -
- Developers -
- - -
- Contribution -

- We are always looking for contributions. See the - Wiki for details on contributing. -

-
-
- Source Code -

- The source files are now stored on Subversion : - https://svn.apache.org/repos/asf/incubator/hama/trunk -
- And, git mirros : git://git.apache.org/hama.git - [ http, github ] -

-
-
- Code Style -

- The following sections show how to set up the code style for Hama in Eclipse. - It is strongly preferred that patches use this style before they are supplied. -

- - Download hama-eclipse-formatter.xml. -

- After this, select Window > Preferences, and open up the configuration for - Java > Code Style > Code Formatter. Click on the button labeled Import... - and select the file you downloaded. Give the style a name, and click OK. -

-
-
- Issues -

- All bugs, improvements, patches, etc. should be logged in - JIRA -

-
- - -
Index: src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml =================================================================== --- src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/docs/r0.2.0/index.xml (working copy) @@ -1,37 +0,0 @@ - - - - - - - -
- Welcome to Hama project -
- - -
- Documentations - -
- - -
\ No newline at end of file Index: src/docs/src/documentation/content/xdocs/docs/r0.2.0/ApacheHama-0.2_UserGuide.pdf =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml =================================================================== --- src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/docs/r0.3.0/index.xml (working copy) @@ -1,36 +0,0 @@ - - - - - - - -
- Welcome to Hama project -
- - -
- Documentations - -
- - -
\ No newline at end of file Index: src/docs/src/documentation/content/xdocs/tabs.xml =================================================================== --- src/docs/src/documentation/content/xdocs/tabs.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/tabs.xml (working copy) @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - Index: src/docs/src/documentation/content/xdocs/downloads.xml =================================================================== --- src/docs/src/documentation/content/xdocs/downloads.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/downloads.xml (working copy) @@ -1,64 +0,0 @@ - - - - - - - -
- Apache Hama Downloads -
- - -
- Download -

Use the links below to download Apache Hama releases from one of our mirrors. - You should verify the integrity of the files using the signatures and checksums available from this page. -

- Apache Hama releases are available under the - Apache License, Version 2.0. - See the NOTICE.txt file contained in each release artifact for applicable copyright attribution notices. -

- Download a release now! -

-
- -
- PGP Signatures -

All of the release distribution package have been digitally signed (using PGP or GPG) by - the Apache Group members that constructed them. There will be an accompanying distribution.asc file - in the same directory as the distribution. - The PGP keys can be found in the MIT key repository (hkp://pgp.mit.edu), - the OpenPGP Public Key repository (hkp://subkeys.pgp.net), and within this project's KEYS file, - which is located within each download directory. -

- Always use the signature files to verify the authenticity of the distribution, e.g., -

  % pgpk -a KEYS
-  % pgpv hama-x.x.x.tar.gz.asc
- or, -
  % gpg --import KEYS
-  % gpg --verify hama-x.x.x.tar.gz.asc
- We offer MD5 hashes as an alternative to validate the integrity of the downloaded files. - A unix program called md5 or md5sum is included in many unix distributions. - It is also available as part of GNU Core Utilities package. -

-
- - - -
Index: src/docs/src/documentation/content/xdocs/issue_tracking.xml =================================================================== --- src/docs/src/documentation/content/xdocs/issue_tracking.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/issue_tracking.xml (working copy) @@ -1,44 +0,0 @@ - - - - - - - -
- Issue Tracking -
- - - -
- Issue Tracking -

- Hama tracks both bugs and enhancement requests - here using Apache JIRA. - We welcome input, however, before filing a request, - please make sure you do the following: -

-
    -
  • Search the JIRA database.
  • -
  • Check the user mailing list, - both by searching the archives and by asking questions.
  • -
-
- -
Index: src/docs/src/documentation/content/xdocs/privacy_policy.xml =================================================================== --- src/docs/src/documentation/content/xdocs/privacy_policy.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/privacy_policy.xml (working copy) @@ -1,61 +0,0 @@ - - - - - - - -
- Privacy Policy -
- - - -
- Privacy Policy -

- Information about your use of this website is collected using - server access logs and a tracking cookie. - The collected information consists of the following:

-

-
    -
  • The IP address from which you access the website;
  • -
  • The type of browser and operating system you use to access our site;
  • -
  • The date and time you access our site;
  • -
  • The pages you visit; and
  • -
  • The addresses of pages from where you followed a link to our site.
  • -
-



- Part of this information is gathered using a tracking cookie set by the - Google Analytics service - and handled by Google as described in their - privacy policy. - See your browser documentation for instructions on how to disable - the cookie if you prefer not to share this data with Google.

- We use the gathered information to help us make our site more - useful to visitors and to better understand how and when - our site is used. We do not track or collect personally - identifiable information or associate gathered data - with any personally identifying information from other sources.

- By using this website, you consent to the collection of this data - in the manner and for the purpose described above. -

-
- - -
Index: src/docs/src/documentation/content/xdocs/mailing_lists.xml =================================================================== --- src/docs/src/documentation/content/xdocs/mailing_lists.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/mailing_lists.xml (working copy) @@ -1,70 +0,0 @@ - - - - - - - -
- Mailing Lists -
- - - -
- Hama User List -

- If you use Hama, please subscribe to the Hama user mailing list.
- The Hama user mailing list is : - hama-user@incubator.apache.org. -

- -
- -
- Hama Developer List -

- If you'd like to contribute to Hama, please subscribe to the Hama developer mailing list.
- The Hama developer mailing list is : - hama-dev@incubator.apache.org. -

- -
- -
- Hama Commit List -

- If you'd like to see changes made in Hama's version control system then subscribe - to the Hama commit mailing list.
- The Hama commit mailing list is : - hama-commits@incubator.apache.org. -

- -
- - -
Index: src/docs/src/documentation/content/xdocs/who_we_are.xml =================================================================== --- src/docs/src/documentation/content/xdocs/who_we_are.xml (revision 1152138) +++ src/docs/src/documentation/content/xdocs/who_we_are.xml (working copy) @@ -1,80 +0,0 @@ - - - - - - - -
- Who we are -
- - - -
- Active Committers - -

Hama's active committers are:

- - - - - - - - - - - - - - - - - - - - - - - -
usernamenameorganizationrolestimezone
edwardyoonEdward J. Yoon+9
chanwitChanwit KaewkasiThe University of Manchester+1
-
- -
- Mentors -

Mentors while in incubation include:

-
    -
  • Ian Holsman
  • -
  • Jeff Eastman
  • -
  • Tommaso Toefili
  • -
  • Steve Loughran
  • -
-
- -
- Contributors -

Contributors are:

-
    -
  • Soo-kyung Kim, a contributor of Hama logo sketch.
  • -
  • Morakot, a contributor of Hama logo paint.
  • -
-
- - -
Index: src/docs/src/documentation/resources/images/hama_art__arthur_300x220.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/hama_paint_logo_120x120.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/hama_art_arthur.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/hama_paint_logo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/apache-incubator-logo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/MapReduce.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/hama_logo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/apache-incubator-hama-white.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/apache-incubator-hama.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/headerlogo.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/block.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/resources/images/favicon.ico =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/docs/src/documentation/classes/CatalogManager.properties =================================================================== --- src/docs/src/documentation/classes/CatalogManager.properties (revision 1152138) +++ src/docs/src/documentation/classes/CatalogManager.properties (working copy) @@ -1,37 +0,0 @@ -# Copyright 2002-2004 The Apache Software Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#======================================================================= -# CatalogManager.properties -# -# This is the default properties file for Apache Forrest. -# This facilitates local configuration of application-specific catalogs. -# -# See the Apache Forrest documentation: -# http://forrest.apache.org/docs/your-project.html -# http://forrest.apache.org/docs/validation.html - -# verbosity ... level of messages for status/debug -# See forrest/src/core/context/WEB-INF/cocoon.xconf - -# catalogs ... list of additional catalogs to load -# (Note that Apache Forrest will automatically load its own default catalog -# from src/core/context/resources/schema/catalog.xcat) -# use full pathnames -# pathname separator is always semi-colon (;) regardless of operating system -# directory separator is always slash (/) regardless of operating system -# -#catalogs=/home/me/forrest/my-site/src/documentation/resources/schema/catalog.xcat -catalogs= - Index: src/examples/org/apache/hama/examples/PiEstimator.java =================================================================== --- src/examples/org/apache/hama/examples/PiEstimator.java (revision 1152138) +++ src/examples/org/apache/hama/examples/PiEstimator.java (working copy) @@ -1,158 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSP; -import org.apache.hama.bsp.BSPJob; -import org.apache.hama.bsp.BSPJobClient; -import org.apache.hama.bsp.BSPPeer; -import org.apache.hama.bsp.ClusterStatus; -import org.apache.hama.bsp.DoubleMessage; -import org.apache.zookeeper.KeeperException; - -public class PiEstimator { - private static String MASTER_TASK = "master.task."; - private static Path TMP_OUTPUT = new Path("/tmp/pi-example/output"); - - public static class MyEstimator extends BSP { - public static final Log LOG = LogFactory.getLog(MyEstimator.class); - private Configuration conf; - private String masterTask; - private static final int iterations = 10000; - - public void bsp(BSPPeer bspPeer) throws IOException, - KeeperException, InterruptedException { - int in = 0, out = 0; - for (int i = 0; i < iterations; i++) { - double x = 2.0 * Math.random() - 1.0, y = 2.0 * Math.random() - 1.0; - if ((Math.sqrt(x * x + y * y) < 1.0)) { - in++; - } else { - out++; - } - } - - double data = 4.0 * (double) in / (double) iterations; - DoubleMessage estimate = new DoubleMessage(bspPeer.getPeerName(), data); - - bspPeer.send(masterTask, estimate); - bspPeer.sync(); - - if (bspPeer.getPeerName().equals(masterTask)) { - double pi = 0.0; - int numPeers = bspPeer.getNumCurrentMessages(); - DoubleMessage received; - while ((received = (DoubleMessage) bspPeer.getCurrentMessage()) != null) { - pi += received.getData(); - } - - pi = pi / numPeers; - writeResult(pi); - } - } - - private void writeResult(double pi) throws IOException { - FileSystem fileSys = FileSystem.get(conf); - - SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, - TMP_OUTPUT, DoubleWritable.class, DoubleWritable.class, - CompressionType.NONE); - writer.append(new DoubleWritable(pi), new DoubleWritable(0)); - writer.close(); - } - - public Configuration getConf() { - return conf; - } - - public void setConf(Configuration conf) { - this.conf = conf; - this.masterTask = conf.get(MASTER_TASK); - } - - } - - private static void initTempDir(FileSystem fileSys) throws IOException { - if (fileSys.exists(TMP_OUTPUT)) { - fileSys.delete(TMP_OUTPUT, true); - } - } - - private static void printOutput(FileSystem fileSys, HamaConfiguration conf) - throws IOException { - SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, TMP_OUTPUT, - conf); - DoubleWritable output = new DoubleWritable(); - DoubleWritable zero = new DoubleWritable(); - reader.next(output, zero); - reader.close(); - - System.out.println("Estimated value of PI is " + output); - } - - public static void main(String[] args) throws InterruptedException, - IOException, ClassNotFoundException { - // BSP job configuration - HamaConfiguration conf = new HamaConfiguration(); - - BSPJob bsp = new BSPJob(conf, PiEstimator.class); - // Set the job name - bsp.setJobName("Pi Estimation Example"); - bsp.setBspClass(MyEstimator.class); - - BSPJobClient jobClient = new BSPJobClient(conf); - ClusterStatus cluster = jobClient.getClusterStatus(true); - - if (args.length > 0) { - bsp.setNumBspTask(Integer.parseInt(args[0])); - } else { - // Set to maximum - bsp.setNumBspTask(cluster.getMaxTasks()); - } - - // Choose one as a master - for (String peerName : cluster.getActiveGroomNames().values()) { - conf.set(MASTER_TASK, peerName); - break; - } - - FileSystem fileSys = FileSystem.get(conf); - initTempDir(fileSys); - - long startTime = System.currentTimeMillis(); - - if (bsp.waitForCompletion(true)) { - printOutput(fileSys, conf); - - System.out.println("Job Finished in " - + (double) (System.currentTimeMillis() - startTime) / 1000.0 - + " seconds"); - } - } -} Index: src/examples/org/apache/hama/examples/graph/PageRankBase.java =================================================================== --- src/examples/org/apache/hama/examples/graph/PageRankBase.java (revision 1152138) +++ src/examples/org/apache/hama/examples/graph/PageRankBase.java (working copy) @@ -1,236 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.graph; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSP; -import org.apache.hama.bsp.BSPPeer; - -public abstract class PageRankBase extends BSP { - public static final Log LOG = LogFactory.getLog(PageRankBase.class); - - protected static int MAX_ITERATIONS = 30; - protected static final String MASTER_TASK = "master.groom"; - protected static double ALPHA; - protected static int numOfVertices; - protected static double DAMPING_FACTOR = 0.85; - protected static double EPSILON = 0.001; - - static HashMap> mapAdjacencyList( - Configuration conf, BSPPeer peer) throws FileNotFoundException, - IOException { - FileSystem fs = FileSystem.get(conf); - HashMap> adjacencyList = new HashMap>(); - Path p = new Path(conf.get("in.path." + peer.getPeerName().split(":")[0])); - LOG.info(p.toString()); - SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf); - Text key = new Text(); - Text value = new Text(); - while (reader.next(key, value)) { - PageRankVertex k = new PageRankVertex(key.toString()); - PageRankVertex v = new PageRankVertex(value.toString()); - if (!adjacencyList.containsKey(k)) { - adjacencyList.put(k, new LinkedList()); - adjacencyList.get(k).add(v); - } else { - adjacencyList.get(k).add(v); - } - } - reader.close(); - return adjacencyList; - } - - static HamaConfiguration partitionExample(Path out, HamaConfiguration conf) - throws IOException { - - String[] groomNames = conf.get(ShortestPaths.BSP_PEERS).split(";"); - int sizeOfCluster = groomNames.length; - - // setup the paths where the grooms can find their input - List partPaths = new ArrayList(sizeOfCluster); - List writers = new ArrayList( - sizeOfCluster); - FileSystem fs = FileSystem.get(conf); - for (String entry : groomNames) { - partPaths.add(new Path(out.getParent().toString() + Path.SEPARATOR - + ShortestPaths.PARTED + Path.SEPARATOR - + entry.split(ShortestPaths.NAME_VALUE_SEPARATOR)[0])); - } - // create a seq writer for that - for (Path p : partPaths) { - // LOG.info(p.toString()); - fs.delete(p, true); - writers.add(SequenceFile - .createWriter(fs, conf, p, Text.class, Text.class)); - } - - /** - * 1:twitter.com
- * 2:google.com
- * 3:facebook.com
- * 4:yahoo.com
- * 5:nasa.gov
- * 6:stackoverflow.com
- * 7:youtube.com - */ - - String[] realNames = new String[] { null, "twitter.com", "google.com", - "facebook.com", "yahoo.com", "nasa.gov", "stackoverflow.com", - "youtube.com" }; - - String[] lineArray = new String[] { "1;2;3", "2", "3;1;2;5", "4;5;6", - "5;4;6", "6;4", "7;2;4" }; - - int numLines = 0; - for (String line : lineArray) { - String[] arr = line.split(";"); - String vId = arr[0]; - int indexKey = Integer.valueOf(vId); - LinkedList list = new LinkedList(); - for (int i = 0; i < arr.length; i++) { - int index = Integer.valueOf(arr[i]); - list.add(realNames[index]); - } - - int mod = Math.abs(realNames[indexKey].hashCode() % sizeOfCluster); - for (String value : list) { - writers.get(mod).append(new Text(realNames[indexKey]), new Text(value)); - } - numLines++; - } - - for (SequenceFile.Writer w : writers) - w.close(); - - for (Path p : partPaths) { - conf.set("in.path." + p.getName(), p.toString()); - } - conf.set("num.vertices", "" + numLines); - - return conf; - } - - static HamaConfiguration partitionTextFile(Path in, HamaConfiguration conf) - throws IOException { - - String[] groomNames = conf.get(ShortestPaths.BSP_PEERS).split(";"); - int sizeOfCluster = groomNames.length; - - // setup the paths where the grooms can find their input - List partPaths = new ArrayList(sizeOfCluster); - List writers = new ArrayList( - sizeOfCluster); - FileSystem fs = FileSystem.get(conf); - for (String entry : groomNames) { - partPaths.add(new Path(in.getParent().toString() + Path.SEPARATOR - + ShortestPaths.PARTED + Path.SEPARATOR - + entry.split(ShortestPaths.NAME_VALUE_SEPARATOR)[0])); - } - // create a seq writer for that - for (Path p : partPaths) { - // LOG.info(p.toString()); - fs.delete(p, true); - writers.add(SequenceFile - .createWriter(fs, conf, p, Text.class, Text.class)); - } - - BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(in))); - - String line = null; - long numLines = 0; - while ((line = br.readLine()) != null) { - String[] arr = line.split("\t"); - String vId = arr[0]; - LinkedList list = new LinkedList(); - for (int i = 0; i < arr.length; i++) { - list.add(arr[i]); - } - - int mod = Math.abs(vId.hashCode() % sizeOfCluster); - for (String value : list) { - writers.get(mod).append(new Text(vId), new Text(value)); - } - numLines++; - } - - for (SequenceFile.Writer w : writers) - w.close(); - - for (Path p : partPaths) { - conf.set("in.path." + p.getName(), p.toString()); - } - conf.set("num.vertices", "" + numLines); - - return conf; - } - - static void savePageRankMap(BSPPeer peer, Configuration conf, - Map tentativePagerank) throws IOException { - FileSystem fs = FileSystem.get(conf); - Path outPath = new Path(conf.get("out.path") + Path.SEPARATOR + "temp" - + Path.SEPARATOR - + peer.getPeerName().split(ShortestPaths.NAME_VALUE_SEPARATOR)[0]); - fs.delete(outPath, true); - final SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, - outPath, Text.class, DoubleWritable.class); - for (Entry row : tentativePagerank.entrySet()) { - out.append(new Text(row.getKey().getUrl()), new DoubleWritable(row - .getValue())); - } - LOG.info("Closing..."); - out.close(); - LOG.info("Closed!"); - } - - static void printOutput(FileSystem fs, Configuration conf) throws IOException { - LOG.info("-------------------- RESULTS --------------------"); - FileStatus[] stati = fs.listStatus(new Path(conf.get("out.path") - + Path.SEPARATOR + "temp")); - for (FileStatus status : stati) { - Path path = status.getPath(); - SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); - Text key = new Text(); - DoubleWritable value = new DoubleWritable(); - while (reader.next(key, value)) { - LOG.info(key.toString() + " | " + value.get()); - } - reader.close(); - } - } - -} Index: src/examples/org/apache/hama/examples/graph/ShortestPathsGraphLoader.java =================================================================== --- src/examples/org/apache/hama/examples/graph/ShortestPathsGraphLoader.java (revision 1152138) +++ src/examples/org/apache/hama/examples/graph/ShortestPathsGraphLoader.java (working copy) @@ -1,94 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.graph; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class ShortestPathsGraphLoader { - - static Map> loadGraph() { - - Map> adjacencyList = new HashMap>(); - String[] cities = new String[] { "Frankfurt", "Mannheim", "Wuerzburg", - "Stuttgart", "Kassel", "Karlsruhe", "Erfurt", "Nuernberg", "Augsburg", - "Muenchen" }; - - int id = 1; - for (String city : cities) { - if (city.equals("Frankfurt")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(85, "Mannheim")); - list.add(new ShortestPathVertex(173, "Kassel")); - list.add(new ShortestPathVertex(217, "Wuerzburg")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Stuttgart")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(183, "Nuernberg")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Kassel")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(502, "Muenchen")); - list.add(new ShortestPathVertex(173, "Frankfurt")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Erfurt")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(186, "Wuerzburg")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Wuerzburg")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(217, "Frankfurt")); - list.add(new ShortestPathVertex(168, "Erfurt")); - list.add(new ShortestPathVertex(103, "Nuernberg")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Mannheim")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(80, "Karlsruhe")); - list.add(new ShortestPathVertex(85, "Frankfurt")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Karlsruhe")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(250, "Augsburg")); - list.add(new ShortestPathVertex(80, "Mannheim")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Augsburg")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(250, "Karlsruhe")); - list.add(new ShortestPathVertex(84, "Muenchen")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Nuernberg")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(183, "Stuttgart")); - list.add(new ShortestPathVertex(167, "Muenchen")); - list.add(new ShortestPathVertex(103, "Wuerzburg")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } else if (city.equals("Muenchen")) { - List list = new LinkedList(); - list.add(new ShortestPathVertex(167, "Nuernberg")); - list.add(new ShortestPathVertex(173, "Kassel")); - list.add(new ShortestPathVertex(84, "Augsburg")); - adjacencyList.put(new ShortestPathVertex(0, city), list); - } - id++; - } - return adjacencyList; - } - -} Index: src/examples/org/apache/hama/examples/graph/PageRankVertex.java =================================================================== --- src/examples/org/apache/hama/examples/graph/PageRankVertex.java (revision 1152138) +++ src/examples/org/apache/hama/examples/graph/PageRankVertex.java (working copy) @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.graph; - -public class PageRankVertex { - - private final int id; - private final String url; - - public PageRankVertex(String url) { - super(); - this.id = url.hashCode(); - this.url = url; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + id; - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - PageRankVertex other = (PageRankVertex) obj; - if (id != other.id) - return false; - return true; - } - - @Override - public String toString() { - return url; - } - - public int getId() { - return id; - } - - public String getUrl() { - return url; - } - -} Index: src/examples/org/apache/hama/examples/graph/ShortestPathVertex.java =================================================================== --- src/examples/org/apache/hama/examples/graph/ShortestPathVertex.java (revision 1152138) +++ src/examples/org/apache/hama/examples/graph/ShortestPathVertex.java (working copy) @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.graph; - -public final class ShortestPathVertex { - - private final int id; - private final String name; - private final int weight; - private Integer cost; - - public ShortestPathVertex(int weight, String name) { - super(); - this.id = name.hashCode(); - this.weight = weight; - this.name = name; - } - - public ShortestPathVertex(int weight, String name, Integer cost) { - super(); - this.id = name.hashCode(); - this.weight = weight; - this.name = name; - this.cost = cost; - } - - public String getName() { - return name; - } - - public Integer getCost() { - return cost; - } - - public void setCost(Integer cost) { - this.cost = cost; - } - - public int getId() { - return id; - } - - public int getWeight() { - return weight; - } - - @Override - public String toString() { - return name; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((name == null) ? 0 : name.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - ShortestPathVertex other = (ShortestPathVertex) obj; - if (name == null) { - if (other.name != null) - return false; - } else if (!name.equals(other.name)) - return false; - return true; - } - -} Index: src/examples/org/apache/hama/examples/graph/ShortestPathsBase.java =================================================================== --- src/examples/org/apache/hama/examples/graph/ShortestPathsBase.java (revision 1152138) +++ src/examples/org/apache/hama/examples/graph/ShortestPathsBase.java (working copy) @@ -1,277 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.graph; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSP; -import org.apache.hama.bsp.BSPPeer; - -public abstract class ShortestPathsBase extends BSP { - - public static final String BSP_PEERS = "bsp.peers"; - public static final String SHORTEST_PATHS_START_VERTEX_ID = "shortest.paths.start.vertex.id"; - public static final String PARTED = "parted"; - public static final String IN_PATH = "in.path."; - public static final String OUT_PATH = "out.path"; - public static final String NAME_VALUE_SEPARATOR = ":"; - public static final String MASTER_TASK = "master.groom"; - - /** - * When finished we just writing a sequencefile of the vertex name and the - * cost. - * - * @param peer The peer we got through the BSP method. - * @param adjacencyList - * @throws IOException - */ - static void saveVertexMap(Configuration conf, BSPPeer peer, - Map> adjacencyList) - throws IOException { - Path outPath = new Path(conf.get(OUT_PATH) + Path.SEPARATOR - + peer.getPeerName().split(":")[0]); - FileSystem fs = FileSystem.get(conf); - fs.delete(outPath, true); - final SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, - outPath, Text.class, IntWritable.class); - for (ShortestPathVertex vertex : adjacencyList.keySet()) { - out.append(new Text(vertex.getName()), new IntWritable(vertex.getCost())); - } - out.close(); - } - - /** - * Just a reader of the vertexMap in DFS. Output going to STDOUT. - * - * @param fs - * @param conf - * @throws IOException - */ - static void printOutput(FileSystem fs, Configuration conf) throws IOException { - System.out.println("-------------------- RESULTS --------------------"); - FileStatus[] stati = fs.listStatus(new Path(conf.get(OUT_PATH))); - for (FileStatus status : stati) { - Path path = status.getPath(); - SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); - Text key = new Text(); - IntWritable value = new IntWritable(); - while (reader.next(key, value)) { - System.out.println(key.toString() + " | " + value.get()); - } - reader.close(); - } - } - - /** - * - * The adjacencylist contains two text fields on each line. The key component - * is the name of a vertex, the value is a ":" separated Text field that - * contains the name of the adjacent vertex leftmost and the weight on the - * rightmost side. - * - *
-   *    K               V 
- * Vertex[Text] AdjacentVertex : Weight [Text] - *
- * - * @param adjacencyList - * @param vertexLookupMap - */ - static void mapAdjacencyList(Configuration conf, BSPPeer peer, - Map> adjacencyList, - Map vertexLookupMap) - throws FileNotFoundException, IOException { - FileSystem fs = FileSystem.get(conf); - Path p = new Path(conf.get(IN_PATH - + peer.getPeerName().split(NAME_VALUE_SEPARATOR)[0])); - SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf); - Text key = new Text(); // name of the vertex - Text value = new Text(); // name of the adjacent vertex : weight - while (reader.next(key, value)) { - // a key vertex has weight 0 to itself - ShortestPathVertex keyVertex = new ShortestPathVertex(0, key.toString(), - Integer.MAX_VALUE); - String[] nameWeight = value.toString().split(NAME_VALUE_SEPARATOR); - if (!adjacencyList.containsKey(keyVertex)) { - LinkedList list = new LinkedList(); - list.add(new ShortestPathVertex(Integer.valueOf(nameWeight[1]), - nameWeight[0], Integer.MAX_VALUE)); - adjacencyList.put(keyVertex, list); - vertexLookupMap.put(keyVertex.getName(), keyVertex); - } else { - adjacencyList.get(keyVertex).add( - new ShortestPathVertex(Integer.valueOf(nameWeight[1]), - nameWeight[0], Integer.MAX_VALUE)); - } - } - reader.close(); - } - - /** - * Partitioning for in memory adjacency lists. - * - * @param adjacencyList - * @param status - * @param conf - * @return - * @throws IOException - */ - static HamaConfiguration partition( - Map> adjacencyList, - HamaConfiguration conf) throws IOException { - - String[] groomNames = conf.get(BSP_PEERS).split(";"); - - int sizeOfCluster = groomNames.length; - - // setup the paths where the grooms can find their input - List partPaths = new ArrayList(sizeOfCluster); - List writers = new ArrayList( - sizeOfCluster); - FileSystem fs = FileSystem.get(conf); - Path fileToPartition = new Path(conf.get(OUT_PATH)); - for (String entry : groomNames) { - partPaths.add(new Path(fileToPartition.getParent().toString() - + Path.SEPARATOR + PARTED + Path.SEPARATOR - + entry.split(NAME_VALUE_SEPARATOR)[0])); - } - // create a seq writer for that - for (Path p : partPaths) { - // System.out.println(p.toString()); - fs.delete(p, true); - writers.add(SequenceFile - .createWriter(fs, conf, p, Text.class, Text.class)); - } - - for (Entry> entry : adjacencyList - .entrySet()) { - // a key vertex has weight 0 to itself - ShortestPathVertex keyVertex = entry.getKey(); - // just mod the id - int mod = Math.abs(keyVertex.getId() % sizeOfCluster); - // append it to the right sequenceFile - for (ShortestPathVertex value : entry.getValue()) - writers.get(mod) - .append( - new Text(keyVertex.getName()), - new Text(value.getName() + NAME_VALUE_SEPARATOR - + value.getWeight())); - } - - for (SequenceFile.Writer w : writers) - w.close(); - - for (Path p : partPaths) { - conf.set(IN_PATH + p.getName(), p.toString()); - } - return conf; - } - - /** - * Partitioning for sequencefile partitioned adjacency lists. - * - * The adjacencylist contains two text fields on each line. The key component - * is the name of a vertex, the value is a ":" separated Text field that - * contains the name of the adjacent vertex leftmost and the weight on the - * rightmost side. - * - *
-   *    K               V 
- * Vertex[Text] AdjacentVertex : Weight [Text] - *
- * - * @param fileToPartition - * @param status - * @param conf - * @return - * @throws IOException - */ - static HamaConfiguration partition(Path fileToPartition, - HamaConfiguration conf, boolean skipPartitioning) throws IOException { - - String[] groomNames = conf.get(BSP_PEERS).split(";"); - int sizeOfCluster = groomNames.length; - - // setup the paths where the grooms can find their input - List partPaths = new ArrayList(sizeOfCluster); - List writers = new ArrayList( - sizeOfCluster); - FileSystem fs = FileSystem.get(conf); - for (String entry : groomNames) { - partPaths.add(new Path(fileToPartition.getParent().toString() - + Path.SEPARATOR + PARTED + Path.SEPARATOR - + entry.split(NAME_VALUE_SEPARATOR)[0])); - } - - if (!skipPartitioning) { - - // create a seq writer for that - for (Path p : partPaths) { - // System.out.println(p.toString()); - fs.delete(p, true); - writers.add(SequenceFile.createWriter(fs, conf, p, Text.class, - Text.class)); - } - - // parse our file - if (!fs.exists(fileToPartition)) - throw new FileNotFoundException("File " + fileToPartition - + " wasn't found!"); - - SequenceFile.Reader reader = new SequenceFile.Reader(fs, fileToPartition, - conf); - Text key = new Text(); // name of the vertex - Text value = new Text(); // name of the adjacent vertex : weight - - while (reader.next(key, value)) { - // a key vertex has weight 0 to itself - ShortestPathVertex keyVertex = new ShortestPathVertex(0, key.toString()); - // just mod the id - int mod = Math.abs(keyVertex.getId() % sizeOfCluster); - // append it to the right sequenceFile - writers.get(mod).append(new Text(keyVertex.getName()), new Text(value)); - } - - reader.close(); - - for (SequenceFile.Writer w : writers) - w.close(); - } - - for (Path p : partPaths) { - conf.set(IN_PATH + p.getName(), p.toString()); - } - - return conf; - } - -} Index: src/examples/org/apache/hama/examples/graph/PageRank.java =================================================================== --- src/examples/org/apache/hama/examples/graph/PageRank.java (revision 1152138) +++ src/examples/org/apache/hama/examples/graph/PageRank.java (working copy) @@ -1,251 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.graph; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map.Entry; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSPJob; -import org.apache.hama.bsp.BSPJobClient; -import org.apache.hama.bsp.BSPPeer; -import org.apache.hama.bsp.ClusterStatus; -import org.apache.hama.bsp.DoubleMessage; -import org.apache.zookeeper.KeeperException; - -public class PageRank extends PageRankBase { - public static final Log LOG = LogFactory.getLog(PageRank.class); - - private Configuration conf; - - private HashMap> adjacencyList; - private final HashMap lookupMap = new HashMap(); - private final HashMap tentativePagerank = new HashMap(); - // backup of the last pagerank to determine the error - private final HashMap lastTentativePagerank = new HashMap(); - private String[] peerNames; - - @Override - public void bsp(BSPPeer peer) throws IOException, KeeperException, - InterruptedException { - String master = conf.get(MASTER_TASK); - // setup the datasets - adjacencyList = PageRankBase.mapAdjacencyList(getConf(), peer); - // init the pageranks to 1/n where n is the number of all vertices - for (PageRankVertex vertex : adjacencyList.keySet()) { - tentativePagerank.put(vertex, Double.valueOf(1.0 / numOfVertices)); - lookupMap.put(vertex.getUrl(), vertex); - } - // while the error not converges against epsilon do the pagerank stuff - double error = 1.0; - int iteration = 0; - // if MAX_ITERATIONS are set to 0, ignore the iterations and just go - // with the error - while ((MAX_ITERATIONS > 0 && iteration < MAX_ITERATIONS) - || error >= EPSILON) { - peer.sync(); - - if (iteration >= 1) { - // copy the old pagerank to the backup - copyTentativePageRankToBackup(); - // sum up all incoming messages for a vertex - HashMap sumMap = new HashMap(); - DoubleMessage msg = null; - while ((msg = (DoubleMessage) peer.getCurrentMessage()) != null) { - PageRankVertex k = lookupMap.get(msg.getTag()); - if (!sumMap.containsKey(k)) { - sumMap.put(k, msg.getData()); - } else { - sumMap.put(k, msg.getData() + sumMap.get(k)); - } - } - // pregel formula: - // ALPHA = 0.15 / NumVertices() - // P(i) = ALPHA + 0.85 * sum - for (Entry entry : sumMap.entrySet()) { - tentativePagerank.put(entry.getKey(), ALPHA - + (entry.getValue() * DAMPING_FACTOR)); - } - - // determine the error and send this to the master - double err = determineError(); - error = broadcastError(peer, master, err); - } - // in every step send the tentative pagerank of a vertex to its - // adjacent vertices - for (PageRankVertex vertex : adjacencyList.keySet()) { - sendMessageToNeighbors(peer, vertex); - } - - iteration++; - } - - // Clears all queues entries. - peer.clear(); - // finally save the chunk of pageranks - PageRankBase.savePageRankMap(peer, conf, lastTentativePagerank); - LOG.info("Finished with iteration " + iteration + "!"); - } - - private double broadcastError(BSPPeer peer, String master, - double error) throws IOException, KeeperException, InterruptedException { - peer.send(master, new DoubleMessage("", error)); - peer.sync(); - if (peer.getPeerName().equals(master)) { - double errorSum = 0.0; - int count = 0; - DoubleMessage message; - while ((message = (DoubleMessage) peer.getCurrentMessage()) != null) { - errorSum += message.getData(); - count++; - } - double avgError = errorSum / (double) count; - // LOG.info("Average error: " + avgError); - for (String name : peer.getAllPeerNames()) { - peer.send(name, new DoubleMessage("", avgError)); - } - } - - peer.sync(); - DoubleMessage message = (DoubleMessage) peer.getCurrentMessage(); - return message.getData(); - } - - private double determineError() { - double error = 0.0; - for (Entry entry : tentativePagerank.entrySet()) { - error += Math.abs(lastTentativePagerank.get(entry.getKey()) - - entry.getValue()); - } - return error; - } - - private void copyTentativePageRankToBackup() { - for (Entry entry : tentativePagerank.entrySet()) { - lastTentativePagerank.put(entry.getKey(), entry.getValue()); - } - } - - private void sendMessageToNeighbors(BSPPeer peer, PageRankVertex v) - throws IOException { - List outgoingEdges = adjacencyList.get(v); - for (PageRankVertex adjacent : outgoingEdges) { - int mod = Math.abs(adjacent.getId() % peerNames.length); - // send a message of the tentative pagerank divided by the size of - // the outgoing edges to all adjacents - peer.send(peerNames[mod], new DoubleMessage(adjacent.getUrl(), - tentativePagerank.get(v) / outgoingEdges.size())); - } - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - numOfVertices = Integer.parseInt(conf.get("num.vertices")); - DAMPING_FACTOR = Double.parseDouble(conf.get("damping.factor")); - ALPHA = (1 - DAMPING_FACTOR) / (double) numOfVertices; - EPSILON = Double.parseDouble(conf.get("epsilon.error")); - MAX_ITERATIONS = Integer.parseInt(conf.get("max.iterations")); - peerNames = conf.get(ShortestPaths.BSP_PEERS).split(";"); - } - - @Override - public Configuration getConf() { - return conf; - } - - public static void printUsage() { - System.out.println("PageRank Example:"); - System.out - .println(" "); - } - - public static void main(String[] args) throws IOException, - InterruptedException, ClassNotFoundException { - if (args.length == 0) { - printUsage(); - System.exit(-1); - } - - HamaConfiguration conf = new HamaConfiguration(new Configuration()); - // set the defaults - conf.set("damping.factor", "0.85"); - conf.set("epsilon.error", "0.000001"); - - if (args.length < 2) { - System.out.println("You have to provide a damping factor and an error!"); - System.out.println("Try using 0.85 0.001 as parameter!"); - System.exit(-1); - } else { - conf.set("damping.factor", args[0]); - conf.set("epsilon.error", args[1]); - LOG.info("Set damping factor to " + args[0]); - LOG.info("Set epsilon error to " + args[1]); - if (args.length > 2) { - conf.set("out.path", args[2]); - LOG.info("Set output path to " + args[2]); - if (args.length == 4) { - conf.set("in.path", args[3]); - LOG.info("Using custom input at " + args[3]); - } else { - LOG.info("Running default example graph!"); - } - } else { - conf.set("out.path", "pagerank/output"); - LOG.info("Set output path to default of pagerank/output!"); - } - } - - BSPJobClient jobClient = new BSPJobClient(conf); - ClusterStatus cluster = jobClient.getClusterStatus(true); - StringBuilder sb = new StringBuilder(); - for (String peerName : cluster.getActiveGroomNames().values()) { - conf.set(MASTER_TASK, peerName); - sb.append(peerName + ";"); - } - - // put every peer into the configuration - conf.set(ShortestPaths.BSP_PEERS, sb.toString()); - // leave the iterations on default - conf.set("max.iterations", "0"); - - if (conf.get("in.path") == null) { - conf = PageRankBase - .partitionExample(new Path(conf.get("out.path")), conf); - } else { - conf = PageRankBase - .partitionTextFile(new Path(conf.get("in.path")), conf); - } - - BSPJob job = new BSPJob(conf); - job.setNumBspTask(cluster.getGroomServers()); - job.setBspClass(PageRank.class); - job.setJarByClass(PageRank.class); - job.setJobName("Pagerank"); - if (job.waitForCompletion(true)) { - PageRankBase.printOutput(FileSystem.get(conf), conf); - } - } -} Index: src/examples/org/apache/hama/examples/graph/ShortestPaths.java =================================================================== --- src/examples/org/apache/hama/examples/graph/ShortestPaths.java (revision 1152138) +++ src/examples/org/apache/hama/examples/graph/ShortestPaths.java (working copy) @@ -1,254 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.graph; - -import java.io.IOException; -import java.util.Deque; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSPJob; -import org.apache.hama.bsp.BSPJobClient; -import org.apache.hama.bsp.BSPPeer; -import org.apache.hama.bsp.BooleanMessage; -import org.apache.hama.bsp.ClusterStatus; -import org.apache.hama.bsp.IntegerMessage; -import org.apache.hama.examples.RandBench; -import org.apache.zookeeper.KeeperException; - -public class ShortestPaths extends ShortestPathsBase { - public static final Log LOG = LogFactory.getLog(ShortestPaths.class); - - private Configuration conf; - private Map> adjacencyList = new HashMap>(); - private Map vertexLookupMap = new HashMap(); - private String[] peerNames; - - @Override - public void bsp(BSPPeer peer) throws IOException, KeeperException, - InterruptedException { - LOG.info("Mapping graph into ram..."); - // map our input into ram - mapAdjacencyList(conf, peer, adjacencyList, vertexLookupMap); - LOG.info("Finished! Starting graph initialization..."); - // parse the configuration to get the peerNames - parsePeerNames(conf); - // get our master groom - String master = conf.get(MASTER_TASK); - - // initial message bypass - ShortestPathVertex v = vertexLookupMap.get(conf - .get(SHORTEST_PATHS_START_VERTEX_ID)); - if (v != null) { - v.setCost(0); - sendMessageToNeighbors(peer, v); - } - - LOG.info("Finished! Starting main loop..."); - boolean updated = true; - while (updated) { - int updatesMade = 0; - peer.sync(); - - IntegerMessage msg = null; - Deque updatedQueue = new LinkedList(); - while ((msg = (IntegerMessage) peer.getCurrentMessage()) != null) { - ShortestPathVertex vertex = vertexLookupMap.get(msg.getTag()); - // check if we need an distance update - if (vertex.getCost() > msg.getData()) { - updatesMade++; - updatedQueue.add(vertex); - vertex.setCost(msg.getData()); - } - } - // synchonize with all grooms if there were updates - updated = broadcastUpdatesMade(peer, master, updatesMade); - // send updates to the adjacents of the updated vertices - for (ShortestPathVertex vertex : updatedQueue) { - sendMessageToNeighbors(peer, vertex); - } - } - LOG.info("Finished!"); - // finished, finally save our map to DFS. - saveVertexMap(conf, peer, adjacencyList); - } - - /** - * Parses the peer names to fix inconsistency in bsp peer names from context. - * - * @param conf - */ - private void parsePeerNames(Configuration conf) { - peerNames = conf.get(BSP_PEERS).split(";"); - } - - /** - * This method broadcasts to a master groom how many updates were made. He - * simply sums them up and sends a message back to the grooms if sum is - * greater than zero. - * - * @param peer The peer we got through the BSP method. - * @param master The assigned master groom name. - * @param updates How many updates were made? - * @return True if we need another iteration, False if no updates can be made - * anymore. - * @throws IOException - * @throws KeeperException - * @throws InterruptedException - */ - private boolean broadcastUpdatesMade(BSPPeer peer, String master, - int updates) throws IOException, KeeperException, InterruptedException { - peer.send(master, new IntegerMessage(peer.getPeerName(), updates)); - peer.sync(); - if (peer.getPeerName().equals(master)) { - int count = 0; - IntegerMessage message; - while ((message = (IntegerMessage) peer.getCurrentMessage()) != null) { - count += message.getData(); - LOG.info("Received " + message.getData() + " updates from " - + message.getTag() + " in SuperStep " + peer.getSuperstepCount()); - } - - for (String name : peer.getAllPeerNames()) { - peer.send(name, new BooleanMessage("", count > 0 ? true : false)); - } - } - - peer.sync(); - BooleanMessage message = (BooleanMessage) peer.getCurrentMessage(); - return message.getData(); - } - - /** - * This method takes advantage of our partitioning: it uses the vertexID - * (simply hash of the name) to determine the host where the message belongs - * to.
- * It sends the current cost to the adjacent vertex + the edge weight. If cost - * will be infinity we just going to send infinity, because summing the weight - * will cause an integer overflow resulting in negative weights. - * - * @param peer The peer we got through the BSP method. - * @param id The vertex to all adjacent vertices the new cost has to be send. - * @throws IOException - */ - private void sendMessageToNeighbors(BSPPeer peer, - ShortestPathVertex id) throws IOException { - List outgoingEdges = adjacencyList.get(id); - for (ShortestPathVertex adjacent : outgoingEdges) { - int mod = Math.abs((adjacent.getId() % peer.getAllPeerNames().length)); - peer.send(peerNames[mod], new IntegerMessage(adjacent.getName(), id - .getCost() == Integer.MAX_VALUE ? id.getCost() : id.getCost() - + adjacent.getWeight())); - } - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - } - - @Override - public Configuration getConf() { - return conf; - } - - public static void printUsage() { - System.out.println("Single Source Shortest Path Example:"); - System.out - .println(" "); - } - - public static void main(String[] args) throws IOException, - InterruptedException, ClassNotFoundException { - - printUsage(); - - // BSP job configuration - HamaConfiguration conf = new HamaConfiguration(); - conf.set(SHORTEST_PATHS_START_VERTEX_ID, "Frankfurt"); - System.out.println("Setting default start vertex to \"Frankfurt\"!"); - conf.set(OUT_PATH, "sssp/output"); - boolean skipPartitioning = false; - Path adjacencyListPath = null; - - if (args.length > 0) { - conf.set(SHORTEST_PATHS_START_VERTEX_ID, args[0]); - System.out.println("Setting start vertex to " + args[0] + "!"); - - if (args.length > 1) { - conf.set(OUT_PATH, args[1]); - System.out.println("Using new output folder: " + args[1]); - } - - if (args.length > 2) { - adjacencyListPath = new Path(args[2]); - } - - if (args.length > 3) { - skipPartitioning = Boolean.valueOf(args[3]); - } - - } - - Map> adjacencyList = null; - if (adjacencyListPath == null) - adjacencyList = ShortestPathsGraphLoader.loadGraph(); - - BSPJob bsp = new BSPJob(conf, RandBench.class); - // Set the job name - bsp.setJobName("Single Source Shortest Path"); - bsp.setBspClass(ShortestPaths.class); - - // Set the task size as a number of GroomServer - BSPJobClient jobClient = new BSPJobClient(conf); - ClusterStatus cluster = jobClient.getClusterStatus(true); - StringBuilder sb = new StringBuilder(); - for (String peerName : cluster.getActiveGroomNames().values()) { - conf.set(MASTER_TASK, peerName); - sb.append(peerName); - sb.append(";"); - } - LOG.info("Master is: " + conf.get(MASTER_TASK)); - conf.set(BSP_PEERS, sb.toString()); - LOG.info("Starting data partitioning..."); - if (adjacencyList == null) - conf = partition(adjacencyListPath, conf, skipPartitioning); - else - conf = partition(adjacencyList, conf); - LOG.info("Finished!"); - - bsp.setNumBspTask(cluster.getGroomServers()); - - long startTime = System.currentTimeMillis(); - if (bsp.waitForCompletion(true)) { - System.out.println("Job Finished in " - + (double) (System.currentTimeMillis() - startTime) / 1000.0 - + " seconds"); - printOutput(FileSystem.get(conf), conf); - } - } - -} Index: src/examples/org/apache/hama/examples/SerializePrinting.java =================================================================== --- src/examples/org/apache/hama/examples/SerializePrinting.java (revision 1152138) +++ src/examples/org/apache/hama/examples/SerializePrinting.java (working copy) @@ -1,134 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; -import java.util.Date; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.io.Text; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSP; -import org.apache.hama.bsp.BSPJob; -import org.apache.hama.bsp.BSPJobClient; -import org.apache.hama.bsp.BSPPeer; -import org.apache.hama.bsp.ClusterStatus; -import org.apache.zookeeper.KeeperException; - -public class SerializePrinting { - private static String TMP_OUTPUT = "/tmp/test-example/"; - - public static class HelloBSP extends BSP { - public static final Log LOG = LogFactory.getLog(HelloBSP.class); - private Configuration conf; - private final static int PRINT_INTERVAL = 1000; - private FileSystem fileSys; - private int num; - - public void bsp(BSPPeer bspPeer) throws IOException, - KeeperException, InterruptedException { - - int i = 0; - for (String otherPeer : bspPeer.getAllPeerNames()) { - String peerName = bspPeer.getPeerName(); - if (peerName.equals(otherPeer)) { - writeLogToFile(peerName, i); - } - - Thread.sleep(PRINT_INTERVAL); - bspPeer.sync(); - i++; - } - } - - private void writeLogToFile(String string, int i) throws IOException { - SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, - new Path(TMP_OUTPUT + i), LongWritable.class, Text.class, - CompressionType.NONE); - writer.append(new LongWritable(System.currentTimeMillis()), new Text( - "Hello BSP from " + (i + 1) + " of " + num + ": " + string)); - writer.close(); - } - - public Configuration getConf() { - return conf; - } - - public void setConf(Configuration conf) { - this.conf = conf; - num = Integer.parseInt(conf.get("bsp.peers.num")); - try { - fileSys = FileSystem.get(conf); - } catch (IOException e) { - e.printStackTrace(); - } - } - - } - - private static void printOutput(FileSystem fileSys, ClusterStatus cluster, - HamaConfiguration conf) throws IOException { - System.out.println("Each task printed the \"Hello World\" as below:"); - for (int i = 0; i < cluster.getGroomServers(); i++) { - SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, new Path( - TMP_OUTPUT + i), conf); - LongWritable timestamp = new LongWritable(); - Text message = new Text(); - reader.next(timestamp, message); - System.out.println(new Date(timestamp.get()) + ": " + message); - reader.close(); - } - } - - private static void initTempDir(FileSystem fileSys) throws IOException { - if (fileSys.exists(new Path(TMP_OUTPUT))) { - fileSys.delete(new Path(TMP_OUTPUT), true); - } - } - - public static void main(String[] args) throws InterruptedException, - IOException, ClassNotFoundException { - // BSP job configuration - HamaConfiguration conf = new HamaConfiguration(); - - BSPJob bsp = new BSPJob(conf, SerializePrinting.class); - // Set the job name - bsp.setJobName("Serialize Printing"); - bsp.setBspClass(HelloBSP.class); - - // Set the task size as a number of GroomServer - BSPJobClient jobClient = new BSPJobClient(conf); - ClusterStatus cluster = jobClient.getClusterStatus(false); - bsp.setNumBspTask(cluster.getGroomServers()); - - FileSystem fileSys = FileSystem.get(conf); - initTempDir(fileSys); - - if (bsp.waitForCompletion(true)) { - printOutput(fileSys, cluster, conf); - } - } - -} Index: src/examples/org/apache/hama/examples/RandBench.java =================================================================== --- src/examples/org/apache/hama/examples/RandBench.java (revision 1152138) +++ src/examples/org/apache/hama/examples/RandBench.java (working copy) @@ -1,122 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSP; -import org.apache.hama.bsp.BSPJob; -import org.apache.hama.bsp.BSPJobClient; -import org.apache.hama.bsp.BSPMessage; -import org.apache.hama.bsp.BSPPeer; -import org.apache.hama.bsp.ByteMessage; -import org.apache.hama.bsp.ClusterStatus; -import org.apache.hama.util.Bytes; -import org.apache.zookeeper.KeeperException; - -public class RandBench { - private static final String SIZEOFMSG = "msg.size"; - private static final String N_COMMUNICATIONS = "communications.num"; - private static final String N_SUPERSTEPS = "supersteps.num"; - - public static class RandBSP extends BSP { - public static final Log LOG = LogFactory.getLog(RandBSP.class); - private Configuration conf; - private Random r = new Random(); - private int sizeOfMsg; - private int nCommunications; - private int nSupersteps; - - @Override - public void bsp(BSPPeer bspPeer) throws IOException, - KeeperException, InterruptedException { - byte[] dummyData = new byte[sizeOfMsg]; - BSPMessage msg = null; - String[] peers = bspPeer.getAllPeerNames(); - String peerName = bspPeer.getPeerName(); - - for (int i = 0; i < nSupersteps; i++) { - - for (int j = 0; j < nCommunications; j++) { - String tPeer = peers[r.nextInt(peers.length)]; - String tag = peerName + " to " + tPeer; - msg = new ByteMessage(Bytes.toBytes(tag), dummyData); - bspPeer.send(tPeer, msg); - } - - bspPeer.sync(); - - ByteMessage received; - while ((received = (ByteMessage) bspPeer.getCurrentMessage()) != null) { - LOG.info(Bytes.toString(received.getTag()) + " : " - + received.getData().length); - } - - } - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - this.sizeOfMsg = conf.getInt(SIZEOFMSG, 1); - this.nCommunications = conf.getInt(N_COMMUNICATIONS, 1); - this.nSupersteps = conf.getInt(N_SUPERSTEPS, 1); - } - - @Override - public Configuration getConf() { - return conf; - } - - } - - public static void main(String[] args) throws Exception { - if (args.length < 3) { - System.out.println("Usage: "); - System.exit(-1); - } - - // BSP job configuration - HamaConfiguration conf = new HamaConfiguration(); - - conf.setInt(SIZEOFMSG, Integer.parseInt(args[0])); - conf.setInt(N_COMMUNICATIONS, Integer.parseInt(args[1])); - conf.setInt(N_SUPERSTEPS, Integer.parseInt(args[2])); - - BSPJob bsp = new BSPJob(conf, RandBench.class); - // Set the job name - bsp.setJobName("Random Communication Benchmark"); - bsp.setBspClass(RandBSP.class); - - // Set the task size as a number of GroomServer - BSPJobClient jobClient = new BSPJobClient(conf); - ClusterStatus cluster = jobClient.getClusterStatus(false); - bsp.setNumBspTask(cluster.getMaxTasks()); - - long startTime = System.currentTimeMillis(); - bsp.waitForCompletion(true); - System.out.println("Job Finished in " - + (double) (System.currentTimeMillis() - startTime) / 1000.0 - + " seconds"); - } -} Index: src/examples/org/apache/hama/examples/ExampleDriver.java =================================================================== --- src/examples/org/apache/hama/examples/ExampleDriver.java (revision 1152138) +++ src/examples/org/apache/hama/examples/ExampleDriver.java (working copy) @@ -1,42 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import org.apache.hadoop.util.ProgramDriver; -import org.apache.hama.examples.graph.PageRank; -import org.apache.hama.examples.graph.ShortestPaths; - -public class ExampleDriver { - - public static void main(String[] args) { - ProgramDriver pgd = new ProgramDriver(); - try { - pgd.addClass("pi", PiEstimator.class, "Pi Estimator"); - pgd.addClass("bench", RandBench.class, "Random Communication Benchmark"); - pgd.addClass("test", SerializePrinting.class, "Serialize Printing Test"); - pgd.addClass("sssp", ShortestPaths.class, "Single Source Shortest Path"); - pgd.addClass("pagerank", PageRank.class, "PageRank"); - - pgd.driver(args); - } catch (Throwable e) { - e.printStackTrace(); - } - } -} Index: bin/grooms.sh =================================================================== --- bin/grooms.sh (revision 1152138) +++ bin/grooms.sh (working copy) @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Run a shell command on all slave hosts. -# -# Environment Variables -# -# HAMA_GROOMS File naming remote hosts. -# Default is ${HAMA_CONF_DIR}/groomservers. -# HAMA_CONF_DIR Alternate conf dir. Default is ${HAMA_HOME}/conf. -# HAMA_GROOM_SLEEP Seconds to sleep between spawning remote commands. -# HAMA_SSH_OPTS Options passed to ssh when running remote commands. -## - -usage="Usage: grooms.sh [--config confdir] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. "$bin"/hama-config.sh - -# If the groomservers file is specified in the command line, -# then it takes precedence over the definition in -# hama-env.sh. Save it here. -HOSTLIST=$HAMA_GROOMS - -if [ -f "${HAMA_CONF_DIR}/hama-env.sh" ]; then - . "${HAMA_CONF_DIR}/hama-env.sh" -fi - -if [ "$HOSTLIST" = "" ]; then - if [ "$HAMA_GROOMS" = "" ]; then - export HOSTLIST="${HAMA_CONF_DIR}/groomservers" - else - export HOSTLIST="${HAMA_GROOMS}" - fi -fi - -for groom in `cat "$HOSTLIST"|sed "s/#.*$//;/^$/d"`; do - ssh $HAMA_SSH_OPTS $groom $"${@// /\\ }" \ - 2>&1 | sed "s/^/$groom: /" & - if [ "$HAMA_GROOM_SLEEP" != "" ]; then - sleep $HAMA_GROOM_SLEEP - fi -done - -wait Index: bin/zookeepers.sh =================================================================== --- bin/zookeepers.sh (revision 1152138) +++ bin/zookeepers.sh (working copy) @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2009 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all zookeeper hosts. -# -# Environment Variables -# -# HAMA_CONF_DIR Alternate hama conf dir. Default is ${HAMA_HOME}/conf. -# HAMA_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: zookeepers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. "$bin"/hama-config.sh - -if [ "$HAMA_MANAGES_ZK" = "" ]; then - HAMA_MANAGES_ZK=true -fi - -if [ "$HAMA_MANAGES_ZK" = "true" ]; then - hosts=`"$bin"/hama org.apache.hama.zookeeper.ZKServerTool` - cmd=$"${@// /\\ }" - for zookeeper in $hosts; do - ssh $HAMA_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & - done -fi - -wait Index: bin/hama-daemon.sh =================================================================== --- bin/hama-daemon.sh (revision 1152138) +++ bin/hama-daemon.sh (working copy) @@ -1,141 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Runs a Hama command as a daemon. -# -# Environment Variables -# -# HAMA_CONF_DIR Alternate conf dir. Default is ${HAMA_HOME}/conf. -# HAMA_LOG_DIR Where log files are stored. PWD by default. -# HAMA_MASTER host:path where hama code should be rsync'd from -# HAMA_PID_DIR The pid files are stored. /tmp by default. -# HAMA_IDENT_STRING A string representing this instance of hama. $USER by default -# HAMA_NICENESS The scheduling priority for daemons. Defaults to 0. -## - -usage="Usage: hama-daemon.sh [--config ] [--hosts hostlistfile] (start|stop) " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. "$bin"/hama-config.sh - -# get arguments -startStop=$1 -shift -command=$1 -shift - -hama_rotate_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" - num=$prev - done - mv "$log" "$log.$num"; - fi -} - -if [ -f "${HAMA_CONF_DIR}/hama-env.sh" ]; then - . "${HAMA_CONF_DIR}/hama-env.sh" -fi - -# get log directory -if [ "$HAMA_LOG_DIR" = "" ]; then - export HAMA_LOG_DIR="$HAMA_HOME/logs" -fi -mkdir -p "$HAMA_LOG_DIR" - -if [ "$HAMA_PID_DIR" = "" ]; then - HAMA_PID_DIR=/tmp -fi - -if [ "$HAMA_IDENT_STRING" = "" ]; then - export HAMA_IDENT_STRING="$USER" -fi - -# some variables -export HAMA_LOGFILE=hama-$HAMA_IDENT_STRING-$command-$HOSTNAME.log -export HAMA_ROOT_LOGGER="INFO,DRFA" -log=$HAMA_LOG_DIR/hama-$HAMA_IDENT_STRING-$command-$HOSTNAME.out -pid=$HAMA_PID_DIR/hama-$HAMA_IDENT_STRING-$command.pid - -# Set default scheduling priority -if [ "$HAMA_NICENESS" = "" ]; then - export HAMA_NICENESS=0 -fi - -case $startStop in - - (start) - - mkdir -p "$HAMA_PID_DIR" - - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi - - if [ "$HAMA_MASTER" != "" ]; then - echo rsync from $HAMA_MASTER - rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $HAMA_MASTER/ "$HAMA_HOME" - fi - - hama_rotate_log $log - echo starting $command, logging to $log - cd "$HAMA_HOME" - nohup nice -n $HAMA_NICENESS "$HAMA_HOME"/bin/hama --config $HAMA_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & - echo $! > $pid - sleep 1; head "$log" - ;; - - (stop) - - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo stopping $command - kill `cat $pid` - else - echo no $command to stop - fi - else - echo no $command to stop - fi - ;; - - (*) - echo $usage - exit 1 - ;; - -esac Index: bin/hama-config.sh =================================================================== --- bin/hama-config.sh (revision 1152138) +++ bin/hama-config.sh (working copy) @@ -1,74 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# included in all the hama scripts with source command -# should not be executable directly -# also should not be passed any arguments, since we need original $* - -# resolve links - $0 may be a softlink - -this="$0" -while [ -h "$this" ]; do - ls=`ls -ld "$this"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=`dirname "$this"`/"$link" - fi -done - -# convert relative path to absolute path -bin=`dirname "$this"` -script=`basename "$this"` -bin=`cd "$bin"; pwd` -this="$bin/$script" - -# the root of the Hama installation -export HAMA_HOME=`dirname "$this"`/.. - -#check to see if the conf dir is given as an optional argument -if [ $# -gt 1 ] -then - if [ "--config" = "$1" ] - then - shift - confdir=$1 - shift - HAMA_CONF_DIR=$confdir - fi -fi - -# Allow alternate conf dir location. -HAMA_CONF_DIR="${HAMA_CONF_DIR:-$HAMA_HOME/conf}" - -# Source the hama-env.sh. -# Will have JAVA_HOME and HAMA_MANAGES_ZK defined. -if [ -f "${HAMA_CONF_DIR}/hama-env.sh" ]; then - . "${HAMA_CONF_DIR}/hama-env.sh" -fi - -#check to see it is specified whether to use the grooms or the -# masters file -if [ $# -gt 1 ] -then - if [ "--hosts" = "$1" ] - then - shift - grooms=$1 - shift - export HAMA_SLAVES="${HAMA_CONF_DIR}/$grooms" - fi -fi Index: bin/hama-daemons.sh =================================================================== --- bin/hama-daemons.sh (revision 1152138) +++ bin/hama-daemons.sh (working copy) @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Run a Hama command on all slave hosts. - -usage="Usage: hama-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..." - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. $bin/hama-config.sh - -remote_cmd="cd ${HAMA_HOME}; $bin/hama-daemon.sh --config ${HAMA_CONF_DIR} $@" -args="--config ${HAMA_CONF_DIR} $remote_cmd" -command=$2 - -case $command in - (zookeeper) - exec "$bin/zookeepers.sh" $args - ;; - (*) - exec "$bin/grooms.sh" $args - ;; -esac \ No newline at end of file Index: bin/stop-bspd.sh =================================================================== --- bin/stop-bspd.sh (revision 1152138) +++ bin/stop-bspd.sh (working copy) @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Stop hama map reduce daemons. Run this on master node. - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. "$bin"/hama-config.sh - -"$bin"/hama-daemon.sh --config $HAMA_CONF_DIR stop bspmaster -"$bin"/hama-daemons.sh --config $HAMA_CONF_DIR stop groom -"$bin"/hama-daemons.sh --config "${HAMA_CONF_DIR}" stop zookeeper Index: bin/hama =================================================================== --- bin/hama (revision 1152138) +++ bin/hama (working copy) @@ -1,186 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# The Hama command script -# -# Environment Variables -# -# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -# -# HAMA_CLASSPATH Extra Java CLASSPATH entries. -# -# HAMA_HEAPSIZE The maximum amount of heap to use, in MB. -# Default is 1000. -# -# HAMA_OPTS Extra Java runtime options. -# -# HAMA_NAMENODE_OPTS These options are added to HAMA_OPTS -# HAMA_CLIENT_OPTS when the respective command is run. -# HAMA_{COMMAND}_OPTS etc HAMA_JT_OPTS applies to JobTracker -# for e.g. HAMA_CLIENT_OPTS applies to -# more than one command (fs, dfs, fsck, -# dfsadmin etc) -# -# HAMA_CONF_DIR Alternate conf dir. Default is ${HAMA_HOME}/conf. -# -# HAMA_ROOT_LOGGER The root appender. Default is INFO,console -# - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. "$bin"/hama-config.sh - -cygwin=false -case "`uname`" in -CYGWIN*) cygwin=true;; -esac - -# if no args specified, show usage -if [ $# = 0 ]; then - echo "Usage: hama [--config confdir] COMMAND" - echo "where COMMAND is one of:" - echo " bspmaster run the BSP Master node" - echo " groom run the Groom node" - echo " zookeeper run a Zookeeper server" - echo " job manipulate BSP jobs" - echo " jar run a jar file" - echo " or" - echo " CLASSNAME run the class named CLASSNAME" - echo "Most commands print help when invoked w/o parameters." - exit 1 -fi - -# get arguments -COMMAND=$1 -shift - -if [ -f "${HAMA_CONF_DIR}/hama-env.sh" ]; then - . "${HAMA_CONF_DIR}/hama-env.sh" -fi - -# some Java parameters -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi - -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi - -JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m - -# check envvars which might override default args -if [ "$HAMA_HEAPSIZE" != "" ]; then - #echo "run with heapsize $HAMA_HEAPSIZE" - JAVA_HEAP_MAX="-Xmx""$HAMA_HEAPSIZE""m" - #echo $JAVA_HEAP_MAX -fi - -# CLASSPATH initially contains $HAMA_CONF_DIR -CLASSPATH="${HAMA_CONF_DIR}" -CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar - -# for developers, add Hama classes to CLASSPATH -if [ -d "$HAMA_HOME/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HAMA_HOME/target/classes -fi -if [ -d "$HAMA_HOME/target/test-classes/classes" ]; then - CLASSPATH=${CLASSPATH}:$HAMA_HOME/target/test-classes -fi - -# so that filenames w/ spaces are handled correctly in loops below -IFS= - -# for releases, add core hama jar to CLASSPATH -for f in $HAMA_HOME/hama-**.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# add libs to CLASSPATH -for f in $HAMA_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# add user-specified CLASSPATH last -if [ "$HAMA_CLASSPATH" != "" ]; then - CLASSPATH=${CLASSPATH}:${HAMA_CLASSPATH} -fi - -# default log directory & file -if [ "$HAMA_LOG_DIR" = "" ]; then - HAMA_LOG_DIR="$HAMA_HOME/logs" -fi -if [ "$HAMA_LOGFILE" = "" ]; then - HAMA_LOGFILE='hama.log' -fi - -# default policy file for service-level authorization -if [ "$HAMA_POLICYFILE" = "" ]; then - HAMA_POLICYFILE="hama-policy.xml" -fi - -# restore ordinary behaviour -unset IFS - -# figure out which class to run -if [ "$COMMAND" = "bspmaster" ] ; then - CLASS='org.apache.hama.BSPMasterRunner' - BSP_OPTS="$BSP_OPTS $BSP_BSPMASTER_OPTS" -elif [ "$COMMAND" = "groom" ] ; then - CLASS='org.apache.hama.GroomServerRunner' - BSP_OPTS="$BSP_OPTS $BSP_GROOMSERVER_OPTS" -elif [ "$COMMAND" = "zookeeper" ] ; then - CLASS='org.apache.hama.ZooKeeperRunner' -elif [ "$COMMAND" = "job" ] ; then - CLASS='org.apache.hama.bsp.BSPJobClient' -elif [ "$COMMAND" = "jar" ] ; then - CLASS=org.apache.hama.util.RunJar - BSP_OPTS="$BSP_OPTS" -else - CLASS=$COMMAND -fi - -# cygwin path translation -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - HAMA_HOME=`cygpath -w "$HAMA_HOME"` - HAMA_LOG_DIR=`cygpath -w "$HAMA_LOG_DIR"` - TOOL_PATH=`cygpath -p -w "$TOOL_PATH"` -fi - -# cygwin path translation -if $cygwin; then - JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` -fi - -HAMA_OPTS="$HAMA_OPTS -Dhama.log.dir=$HAMA_LOG_DIR" -HAMA_OPTS="$HAMA_OPTS -Dhama.log.file=$HAMA_LOGFILE" -HAMA_OPTS="$HAMA_OPTS -Dhama.home.dir=$HAMA_HOME" -HAMA_OPTS="$HAMA_OPTS -Dhama.id.str=$HAMA_IDENT_STRING" -HAMA_OPTS="$HAMA_OPTS -Dhama.root.logger=${HAMA_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - HAMA_OPTS="$HAMA_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" -fi -HAMA_OPTS="$HAMA_OPTS -Dhama.policy.file=$HAMA_POLICYFILE" - -# run it -exec "$JAVA" $JAVA_HEAP_MAX $HAMA_OPTS -classpath "$CLASSPATH" $CLASS "$@" Index: bin/start-bspd.sh =================================================================== --- bin/start-bspd.sh (revision 1152138) +++ bin/start-bspd.sh (working copy) @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Start hama map reduce daemons. Run this on master node. - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. "$bin"/hama-config.sh - -# start bsp daemons -# start zookeeper first to minimize connection errors at startup -"$bin"/hama-daemons.sh --config "${HAMA_CONF_DIR}" start zookeeper -"$bin"/hama-daemon.sh --config $HAMA_CONF_DIR start bspmaster -"$bin"/hama-daemons.sh --config $HAMA_CONF_DIR start groom Property changes on: core ___________________________________________________________________ Added: svn:ignore + *.iml target lib Index: core/conf/hama-env.sh.template =================================================================== --- core/conf/hama-env.sh.template (revision 0) +++ core/conf/hama-env.sh.template (revision 0) @@ -0,0 +1,37 @@ +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# The java implementation to use. Required. +# export JAVA_HOME=/usr/lib/jvm/java-6-sun + +# Where log files are stored. $HAMA_HOME/logs by default. +# export HAMA_LOG_DIR=${HAMA_HOME}/logs + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HAMA_HEAPSIZE=1000 + +# Extra ssh options. Empty by default. +# export HAMA_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HAMA_CONF_DIR" + +# Tell Hama whether it should manage it's own instance of Zookeeper or not. +# export HAMA_MANAGES_ZK=true Index: core/conf/hama-site.xml.template =================================================================== --- core/conf/hama-site.xml.template (revision 0) +++ core/conf/hama-site.xml.template (revision 0) @@ -0,0 +1,25 @@ + + + + + Index: core/conf/hama-env.sh =================================================================== --- core/conf/hama-env.sh (revision 0) +++ core/conf/hama-env.sh (revision 0) @@ -0,0 +1,34 @@ +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# The java implementation to use. Required. +# export JAVA_HOME=/usr/lib/jvm/java-6-sun + +# Where log files are stored. $HAMA_HOME/logs by default. +# export HAMA_LOG_DIR=${HAMA_HOME}/logs + +# Extra ssh options. Empty by default. +# export HAMA_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HAMA_CONF_DIR" + +# Tell Hama whether it should manage it's own instance of Zookeeper or not. +# export HAMA_MANAGES_ZK=true Index: core/conf/groomservers.template =================================================================== --- core/conf/groomservers.template (revision 0) +++ core/conf/groomservers.template (revision 0) @@ -0,0 +1 @@ +localhost \ No newline at end of file Index: core/conf/hama-default.xml =================================================================== --- core/conf/hama-default.xml (revision 0) +++ core/conf/hama-default.xml (revision 0) @@ -0,0 +1,165 @@ + + + + + + bsp.master.address + local + The address of the bsp master server. Either the + literal string "local" or a host[:port] (where host is a name or + IP address) for distributed mode. + + + + bsp.master.port + 40000 + The port master should bind to. + + + bsp.peer.port + 61000 + The port an groom server binds to. + + + bsp.groom.rpc.port + 50000 + The port an groom rpc binds to. + + + bsp.http.infoserver.port + 40013 + The port where the web-interface can be seen. + + + + bsp.groom.report.address + 127.0.0.1:0 + The interface and port that groom server listens on. + Since it is only connected to by the tasks, it uses the local interface. + EXPERT ONLY. Should only be changed if your host does not have the loopback + interface. + + + bsp.system.dir + ${hadoop.tmp.dir}/bsp/system + The shared directory where BSP stores control files. + + + + bsp.local.dir + ${hadoop.tmp.dir}/bsp/local + local directory for temporal store. + + + hama.tmp.dir + /tmp/hama-${user.name} + Temporary directory on the local filesystem. + + + bsp.child.java.opts + -Xmx1024m + Java opts for the groom server child processes. + The following symbol, if present, will be interpolated: @taskid@ is replaced + by current TaskID. Any other occurrences of '@' will go unchanged. + For example, to enable verbose gc logging to a file named for the taskid in + /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: + -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc + The configuration variable bsp.child.ulimit can be used to control the + maximum virtual memory of the child processes. + + + + bsp.local.tasks.maximum + 20 + Number of tasks that run in parallel when in local mode. + + + + + hama.zookeeper.property.initLimit + 10 + Property from ZooKeeper's config zoo.cfg. + The number of ticks that the initial synchronization phase can take. + + + + hama.zookeeper.property.syncLimit + 5 + Property from ZooKeeper's config zoo.cfg. + The number of ticks that can pass between sending a request and getting an + acknowledgment. + + + + hama.zookeeper.property.dataDir + ${hama.tmp.dir}/zookeeper + Property from ZooKeeper's config zoo.cfg. + The directory where the snapshot is stored. + + + + hama.zookeeper.property.clientPort + 21810 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + hama.zookeeper.property.tickTime + 3000 + + + + hama.zookeeper.property.maxClientCnxns + 30 + Property from ZooKeeper's config zoo.cfg. + Limit on number of concurrent connections (at the socket level) that a + single client, identified by IP address, may make to a single member of + the ZooKeeper ensemble. Set high to avoid zk connection issues running + standalone and pseudo-distributed. + + + + + hama.zookeeper.quorum + localhost + Comma separated list of servers in the ZooKeeper quorum. + For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". + By default this is set to localhost for local and pseudo-distributed modes + of operation. For a fully-distributed setup, this should be set to a full + list of ZooKeeper quorum servers. If HAMA_MANAGES_ZK is set in hama-env.sh + this is the list of servers which we will start/stop ZooKeeper on. + + + + + Index: core/conf/hama-site.xml =================================================================== --- core/conf/hama-site.xml (revision 0) +++ core/conf/hama-site.xml (revision 0) @@ -0,0 +1,25 @@ + + + + + Index: core/conf/log4j.properties =================================================================== --- core/conf/log4j.properties (revision 0) +++ core/conf/log4j.properties (revision 0) @@ -0,0 +1,81 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Define some default values that can be overridden by system properties +hama.root.logger=INFO,console +hama.log.dir=. +hama.log.file=hama.log + +# Define the root logger to the system property "hama.root.logger". +log4j.rootLogger=${hama.root.logger} + +# Logging Threshold +log4j.threshhold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hama.log.dir}/${hama.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# +# TaskLog Appender +# + +#Default values +hama.tasklog.taskid=null +hama.tasklog.noKeepSplits=4 +hama.tasklog.totalLogFileSize=100 +hama.tasklog.purgeLogSplits=true +hama.tasklog.logsRetainHours=12 + +log4j.appender.TLA=org.apache.hama.bsp.TaskLogAppender +log4j.appender.TLA.taskId=${hama.tasklog.taskid} +log4j.appender.TLA.totalLogFileSize=${hama.tasklog.totalLogFileSize} + +log4j.appender.TLA.layout=org.apache.log4j.PatternLayout +log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# Custom Logging levels + +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +#log4j.logger.org.apache.hadoop.dfs=DEBUG +#log4j.logger.org.apache.hama=DEBUG Index: core/conf/groomservers =================================================================== --- core/conf/groomservers (revision 0) +++ core/conf/groomservers (revision 0) @@ -0,0 +1 @@ +localhost \ No newline at end of file Index: core/src/test/java/org/apache/hama/HamaTestCase.java =================================================================== --- core/src/test/java/org/apache/hama/HamaTestCase.java (revision 0) +++ core/src/test/java/org/apache/hama/HamaTestCase.java (revision 0) @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import java.io.File; +import java.io.IOException; + +import junit.framework.AssertionFailedError; +import junit.framework.TestCase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hama.util.Bytes; + +public abstract class HamaTestCase extends TestCase { + private static Log LOG = LogFactory.getLog(HamaTestCase.class); + + /** configuration parameter name for test directory */ + public static final String TEST_DIRECTORY_KEY = "test.build.data"; + + private boolean localfs = false; + protected Path testDir = null; + protected FileSystem fs = null; + + static { + initialize(); + } + + public volatile HamaConfiguration conf; + + /** constructor */ + public HamaTestCase() { + super(); + init(); + } + + /** + * @param name + */ + public HamaTestCase(String name) { + super(name); + init(); + } + + private void init() { + conf = new HamaConfiguration(); + conf.setStrings("bsp.local.dir", "/tmp/hama-test"); + conf.set("bsp.master.address", "localhost"); + conf.set("bsp.groom.report.address", "127.0.0.1:0"); + } + + /** + * Note that this method must be called after the mini hdfs cluster has + * started or we end up with a local file system. + */ + @Override + protected void setUp() throws Exception { + super.setUp(); + localfs = + (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0); + + if (fs == null) { + this.fs = FileSystem.get(conf); + } + try { + if (localfs) { + this.testDir = getUnitTestdir(getName()); + if (fs.exists(testDir)) { + fs.delete(testDir, true); + } + } else { + this.testDir = + this.fs.makeQualified(new Path("/tmp/hama-test")); + } + } catch (Exception e) { + LOG.fatal("error during setup", e); + throw e; + } + } + + @Override + protected void tearDown() throws Exception { + try { + if (localfs) { + if (this.fs.exists(testDir)) { + this.fs.delete(testDir, true); + } + } + } catch (Exception e) { + LOG.fatal("error during tear down", e); + } + super.tearDown(); + } + + protected Path getUnitTestdir(String testName) { + return new Path( + conf.get(TEST_DIRECTORY_KEY, "/tmp/hama-test/build/data"), testName); + } + + /** + * Initializes parameters used in the test environment: + * + * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set. + * Sets the boolean debugging if "DEBUGGING" is set in the environment. + * If debugging is enabled, reconfigures logging so that the root log level is + * set to WARN and the logging level for the package is set to DEBUG. + */ + public static void initialize() { + if (System.getProperty(TEST_DIRECTORY_KEY) == null) { + System.setProperty(TEST_DIRECTORY_KEY, new File( + "build/hama/test").getAbsolutePath()); + } + } + + /** + * Common method to close down a MiniDFSCluster and the associated file system + * + * @param cluster + */ + public static void shutdownDfs(MiniDFSCluster cluster) { + if (cluster != null) { + LOG.info("Shutting down Mini DFS "); + try { + cluster.shutdown(); + } catch (Exception e) { + /// Can get a java.lang.reflect.UndeclaredThrowableException thrown + // here because of an InterruptedException. Don't let exceptions in + // here be cause of test failure. + } + try { + FileSystem fs = cluster.getFileSystem(); + if (fs != null) { + LOG.info("Shutting down FileSystem"); + fs.close(); + } + FileSystem.closeAll(); + } catch (IOException e) { + LOG.error("error closing file system", e); + } + } + } + + public void assertByteEquals(byte[] expected, + byte[] actual) { + if (Bytes.compareTo(expected, actual) != 0) { + throw new AssertionFailedError("expected:<" + + Bytes.toString(expected) + "> but was:<" + + Bytes.toString(actual) + ">"); + } + } + + public static void assertEquals(byte[] expected, + byte[] actual) { + if (Bytes.compareTo(expected, actual) != 0) { + throw new AssertionFailedError("expected:<" + + Bytes.toStringBinary(expected) + "> but was:<" + + Bytes.toStringBinary(actual) + ">"); + } + } + +} Index: core/src/test/java/org/apache/hama/MiniBSPCluster.java =================================================================== --- core/src/test/java/org/apache/hama/MiniBSPCluster.java (revision 0) +++ core/src/test/java/org/apache/hama/MiniBSPCluster.java (revision 0) @@ -0,0 +1,252 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import java.io.IOException; + +import java.net.ServerSocket; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import static java.util.concurrent.TimeUnit.*; + +import static junit.framework.Assert.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hama.bsp.BSPMaster; +import org.apache.hama.bsp.GroomServer; +import org.apache.hama.HamaConfiguration; + + +public class MiniBSPCluster { + + public static final Log LOG = LogFactory.getLog(MiniBSPCluster.class); + + private ScheduledExecutorService scheduler; + + private HamaConfiguration configuration; + private BSPMasterRunner master; + private List groomServerList = + new CopyOnWriteArrayList(); + private int grooms; + + public class BSPMasterRunner implements Runnable{ + BSPMaster bspm; + HamaConfiguration conf; + + public BSPMasterRunner(HamaConfiguration conf){ + this.conf = conf; + if(null == this.conf) + throw new NullPointerException("No Configuration for BSPMaster."); + } + + public void run(){ + try{ + LOG.info("Starting BSP Master."); + this.bspm = BSPMaster.startMaster(this.conf); + this.bspm.offerService(); + }catch(IOException ioe){ + LOG.error("Fail to startup BSP Master.", ioe); + }catch(InterruptedException ie){ + LOG.error("BSP Master fails in offerService().", ie); + Thread.currentThread().interrupt(); + } + } + + public void shutdown(){ + if(null != this.bspm) this.bspm.shutdown(); + } + + public boolean isRunning(){ + if(null == this.bspm) return false; + + if(this.bspm.currentState().equals(BSPMaster.State.RUNNING)){ + return true; + } + return false; + } + + public BSPMaster getMaster(){ + return this.bspm; + } + } + + public class GroomServerRunner implements Runnable{ + GroomServer gs; + HamaConfiguration conf; + + public GroomServerRunner(HamaConfiguration conf){ + this.conf = conf; + } + + public void run(){ + try{ + this.gs = GroomServer.constructGroomServer(GroomServer.class, conf); + GroomServer.startGroomServer(this.gs).join(); + }catch(InterruptedException ie){ + LOG.error("Fail to start GroomServer. ", ie); + Thread.currentThread().interrupt(); + } + } + + public void shutdown(){ + try{ + if(null != this.gs) this.gs.shutdown(); + }catch(IOException ioe){ + LOG.info("Fail to shutdown GroomServer.", ioe); + } + } + + public boolean isRunning(){ + if(null == this.gs) return false; + return this.gs.isRunning(); + } + + public GroomServer getGroomServer(){ + return this.gs; + } + } + + public MiniBSPCluster(HamaConfiguration conf, int groomServers) { + this.configuration = conf; + this.grooms = groomServers; + if(1 > this.grooms) { + this.grooms = 2; + } + LOG.info("Groom server number "+this.grooms); + int threadpool = conf.getInt("bsp.test.threadpool", 10); + LOG.info("Thread pool value "+threadpool); + scheduler = Executors.newScheduledThreadPool(threadpool); + } + + public void startBSPCluster(){ + startMaster(); + startGroomServers(); + } + + public void shutdownBSPCluster(){ + if(null != this.master && this.master.isRunning()) + this.master.shutdown(); + if(0 < groomServerList.size()){ + for(GroomServerRunner groom: groomServerList){ + if(groom.isRunning()) groom.shutdown(); + } + } + } + + + public void startMaster(){ + if(null == this.scheduler) + throw new NullPointerException("No ScheduledExecutorService exists."); + this.master = new BSPMasterRunner(this.configuration); + scheduler.schedule(this.master, 0, SECONDS); + } + + public void startGroomServers(){ + if(null == this.scheduler) + throw new NullPointerException("No ScheduledExecutorService exists."); + if(null == this.master) + throw new NullPointerException("No BSPMaster exists."); + int cnt=0; + while(!this.master.isRunning()){ + LOG.info("Waiting BSPMaster up."); + try{ + Thread.sleep(1000); + cnt++; + if(100 < cnt){ + fail("Fail to launch BSPMaster."); + } + }catch(InterruptedException ie){ + LOG.error("Fail to check BSP Master's state.", ie); + Thread.currentThread().interrupt(); + } + } + for(int i=0; i < this.grooms; i++){ + HamaConfiguration c = new HamaConfiguration(this.configuration); + randomPort(c); + GroomServerRunner gsr = new GroomServerRunner(c); + groomServerList.add(gsr); + scheduler.schedule(gsr, 0, SECONDS); + cnt = 0; + while(!gsr.isRunning()){ + LOG.info("Waitin for GroomServer up."); + try{ + Thread.sleep(1000); + cnt++; + if(10 < cnt){ + fail("Fail to launch groom server."); + } + }catch(InterruptedException ie){ + LOG.error("Fail to check Groom Server's state.", ie); + Thread.currentThread().interrupt(); + } + } + } + + } + + private void randomPort(HamaConfiguration conf){ + try{ + ServerSocket skt = new ServerSocket(0); + int p = skt.getLocalPort(); + skt.close(); + conf.set(Constants.PEER_PORT, new Integer(p).toString()); + conf.setInt(Constants.GROOM_RPC_PORT, p+100); + }catch(IOException ioe){ + LOG.error("Can not find a free port for BSPPeer.", ioe); + } + } + + public void shutdown() { + shutdownBSPCluster(); + scheduler.shutdown(); + } + + public List getGroomServerThreads() { + List list = new ArrayList(); + for(GroomServerRunner gsr: groomServerList){ + list.add(new Thread(gsr)); + } + return list; + } + + public Thread getMaster() { + return new Thread(this.master); + } + + public List getGroomServers(){ + List list = new ArrayList(); + for(GroomServerRunner gsr: groomServerList){ + list.add(gsr.getGroomServer()); + } + return list; + } + + public BSPMaster getBSPMaster(){ + if(null != this.master) + return this.master.getMaster(); + return null; + } + + public ScheduledExecutorService getScheduler(){ + return this.scheduler; + } +} Index: core/src/test/java/org/apache/hama/zookeeper/TestZKTools.java =================================================================== --- core/src/test/java/org/apache/hama/zookeeper/TestZKTools.java (revision 0) +++ core/src/test/java/org/apache/hama/zookeeper/TestZKTools.java (revision 0) @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.zookeeper; + +import org.apache.hama.Constants; +import org.apache.hama.HamaConfiguration; + +import junit.framework.TestCase; + +public class TestZKTools extends TestCase { + + public void testZKProps() { + HamaConfiguration conf = new HamaConfiguration(); + conf.set(Constants.ZOOKEEPER_QUORUM, "test.com:123"); + conf.set(Constants.ZOOKEEPER_CLIENT_PORT, "2222"); + + assertEquals("test.com:2222", QuorumPeer.getZKQuorumServersString(conf)); + } +} Index: core/src/test/java/org/apache/hama/HamaClusterTestCase.java =================================================================== --- core/src/test/java/org/apache/hama/HamaClusterTestCase.java (revision 0) +++ core/src/test/java/org/apache/hama/HamaClusterTestCase.java (revision 0) @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import java.io.File; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; + +public abstract class HamaClusterTestCase extends HamaTestCase { + public static final Log LOG = LogFactory.getLog(HamaClusterTestCase.class); + protected MiniDFSCluster dfsCluster; + protected MiniBSPCluster bspCluster; + protected MiniZooKeeperCluster zooKeeperCluster; + protected boolean startDfs; + protected int numOfGroom = 2; + + /** default constructor */ + public HamaClusterTestCase() { + this(false); + } + + public HamaClusterTestCase(boolean startDfs) { + super(); + this.startDfs = startDfs; + } + + /** + * Actually start the MiniBSP instance. + */ + protected void hamaClusterSetup() throws Exception { + File testDir = new File(getUnitTestdir(getName()).toString()); + + // Note that this is done before we create the MiniHamaCluster because we + // need to edit the config to add the ZooKeeper servers. + this.zooKeeperCluster = new MiniZooKeeperCluster(); + int clientPort = this.zooKeeperCluster.startup(testDir); + conf.set("hama.zookeeper.property.clientPort", Integer.toString(clientPort)); + bspCluster = new MiniBSPCluster(this.conf, numOfGroom); + bspCluster.startBSPCluster(); + } + + @Override + protected void setUp() throws Exception { + try { + if (this.startDfs) { + // This spews a bunch of warnings about missing scheme. TODO: fix. + this.dfsCluster = new MiniDFSCluster(0, this.conf, 2, true, true, true, + null, null, null, null); + + // mangle the conf so that the fs parameter points to the minidfs we + // just started up + FileSystem filesystem = dfsCluster.getFileSystem(); + conf.set("fs.defaultFS", filesystem.getUri().toString()); + Path parentdir = filesystem.getHomeDirectory(); + + filesystem.mkdirs(parentdir); + } + + // do the super setup now. if we had done it first, then we would have + // gotten our conf all mangled and a local fs started up. + super.setUp(); + + // start the instance + hamaClusterSetup(); + } catch (Exception e) { + if (zooKeeperCluster != null) { + zooKeeperCluster.shutdown(); + } + if (dfsCluster != null) { + shutdownDfs(dfsCluster); + } + throw e; + } + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + try { + if (startDfs) { + shutdownDfs(dfsCluster); + } + bspCluster.shutdown(); + } catch (Exception e) { + LOG.error(e); + } + } +} Index: core/src/test/java/org/apache/hama/bsp/TestBSPMessageBundle.java =================================================================== --- core/src/test/java/org/apache/hama/bsp/TestBSPMessageBundle.java (revision 0) +++ core/src/test/java/org/apache/hama/bsp/TestBSPMessageBundle.java (revision 0) @@ -0,0 +1,81 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; +import junit.framework.TestCase; + +public class TestBSPMessageBundle extends TestCase { + + public void testEmpty() throws IOException { + BSPMessageBundle bundle = new BSPMessageBundle(); + // Serialize it. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + bundle.write(new DataOutputStream(baos)); + baos.close(); + // Deserialize it. + BSPMessageBundle readBundle = new BSPMessageBundle(); + readBundle.readFields(new DataInputStream(new ByteArrayInputStream(baos + .toByteArray()))); + assertEquals(0, readBundle.getMessages().size()); + } + + public void testSerializationDeserialization() throws IOException { + BSPMessageBundle bundle = new BSPMessageBundle(); + ByteMessage[] testMessages = new ByteMessage[16]; + for (int i = 0; i < testMessages.length; ++i) { + // Create a one byte tag containing the number of the message. + byte[] tag = new byte[1]; + tag[0] = (byte) i; + // Create a four bytes data part containing serialized number of the + // message. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(i); + baos.close(); + byte[] data = baos.toByteArray(); + testMessages[i] = new ByteMessage(tag, data); + bundle.addMessage(testMessages[i]); + } + // Serialize it. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + bundle.write(new DataOutputStream(baos)); + baos.close(); + // Deserialize it. + BSPMessageBundle readBundle = new BSPMessageBundle(); + readBundle.readFields(new DataInputStream(new ByteArrayInputStream(baos + .toByteArray()))); + // Check contents. + int messageNumber = 0; + for (BSPMessage message : readBundle.getMessages()) { + ByteMessage byteMessage = (ByteMessage) message; + assertTrue(Arrays.equals(testMessages[messageNumber].getTag(), + byteMessage.getTag())); + assertTrue(Arrays.equals(testMessages[messageNumber].getData(), + byteMessage.getData())); + ++messageNumber; + } + assertEquals(testMessages.length, messageNumber); + } +} Index: core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java =================================================================== --- core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java (revision 0) +++ core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java (revision 0) @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import junit.framework.TestCase; + +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hama.bsp.BSPMaster; +import org.apache.hama.bsp.ClusterStatus; + +public class TestClusterStatus extends TestCase { + Random rnd = new Random(); + + protected void setUp() throws Exception { + super.setUp(); + } + + public final void testWriteAndReadFields() throws IOException { + DataOutputBuffer out = new DataOutputBuffer(); + DataInputBuffer in = new DataInputBuffer(); + + ClusterStatus status1; + Map grooms = new HashMap(); + + for (int i = 0; i < 10; i++) { + int num = rnd.nextInt(); + String groomName = "groom_" + num; + String peerName = "peerhost:" + num; + grooms.put(groomName, peerName); + } + + int tasks = rnd.nextInt(100); + int maxTasks = rnd.nextInt(100); + BSPMaster.State state = BSPMaster.State.RUNNING; + + status1 = new ClusterStatus(grooms, tasks, maxTasks, state); + status1.write(out); + + in.reset(out.getData(), out.getLength()); + + ClusterStatus status2 = new ClusterStatus(); + status2.readFields(in); + + Map grooms_s = new HashMap(status1 + .getActiveGroomNames()); + Map grooms_o = new HashMap(status2 + .getActiveGroomNames()); + + assertEquals(status1.getGroomServers(), status2.getGroomServers()); + + assertTrue(grooms_s.entrySet().containsAll(grooms_o.entrySet())); + assertTrue(grooms_o.entrySet().containsAll(grooms_s.entrySet())); + + assertEquals(status1.getTasks(), status2.getTasks()); + assertEquals(status1.getMaxTasks(), status2.getMaxTasks()); + } +} Index: core/src/test/java/org/apache/hama/bsp/TestMessages.java =================================================================== --- core/src/test/java/org/apache/hama/bsp/TestMessages.java (revision 0) +++ core/src/test/java/org/apache/hama/bsp/TestMessages.java (revision 0) @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import junit.framework.TestCase; + +import org.apache.hama.util.Bytes; + +public class TestMessages extends TestCase { + + public void testByteMessage() { + int dataSize = (int) (Runtime.getRuntime().maxMemory() * 0.60); + ByteMessage msg = new ByteMessage(Bytes.toBytes("tag"), new byte[dataSize]); + assertEquals(msg.getData().length, dataSize); + msg = null; + + byte[] dummyData = new byte[1024]; + ByteMessage msg2 = new ByteMessage(Bytes.tail(dummyData, 128), dummyData); + assertEquals( + Bytes.compareTo(msg2.getTag(), 0, 128, msg2.getData(), + msg2.getData().length - 128, 128), 0); + } +} Index: core/src/test/java/org/apache/hama/bsp/TestBSPMasterGroomServer.java =================================================================== --- core/src/test/java/org/apache/hama/bsp/TestBSPMasterGroomServer.java (revision 0) +++ core/src/test/java/org/apache/hama/bsp/TestBSPMasterGroomServer.java (revision 0) @@ -0,0 +1,94 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hama.Constants; +import org.apache.hama.HamaCluster; +import org.apache.hama.HamaConfiguration; + +public class TestBSPMasterGroomServer extends HamaCluster { + + private static Log LOG = LogFactory.getLog(TestBSPMasterGroomServer.class); + private static String TMP_OUTPUT = "/tmp/test-example/"; + private HamaConfiguration configuration; + private String TEST_JOB = "src/test/java/testjar/testjob.jar"; + + public TestBSPMasterGroomServer() { + configuration = new HamaConfiguration(); + configuration.set("bsp.master.address", "localhost"); + assertEquals("Make sure master addr is set to localhost:", "localhost", + configuration.get("bsp.master.address")); + configuration.setStrings("bsp.local.dir", "/tmp/hama-test"); + configuration.set(Constants.ZOOKEEPER_QUORUM, "localhost"); + configuration.setInt(Constants.ZOOKEEPER_CLIENT_PORT, 21810); + } + + public void setUp() throws Exception { + super.setUp(); + } + + public void testSubmitJob() throws Exception { + BSPJob bsp = new BSPJob(configuration); + bsp.setJobName("Test Serialize Printing"); + bsp.setBspClass(testjar.ClassSerializePrinting.HelloBSP.class); + bsp.setJar(System.getProperty("user.dir")+"/"+TEST_JOB); + + // Set the task size as a number of GroomServer + BSPJobClient jobClient = new BSPJobClient(configuration); + configuration.setInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 600); + ClusterStatus cluster = jobClient.getClusterStatus(false); + assertEquals(this.numOfGroom, cluster.getMaxTasks()); + + // TODO test the multi-tasks + bsp.setNumBspTask(1); + + FileSystem fileSys = FileSystem.get(conf); + + if (bsp.waitForCompletion(true)) { + checkOutput(fileSys, cluster, conf); + } + LOG.info("Client finishes execution job."); + } + + private static void checkOutput(FileSystem fileSys, ClusterStatus cluster, + HamaConfiguration conf) throws Exception { + for (int i = 0; i < 1; i++) { // TODO test the multi-tasks + SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, new Path( + TMP_OUTPUT + i), conf); + LongWritable timestamp = new LongWritable(); + Text message = new Text(); + reader.next(timestamp, message); + assertTrue("Check if `Hello BSP' gets printed.", message.toString() + .indexOf("Hello BSP from") >= 0); + reader.close(); + } + } + + public void tearDown() throws Exception { + super.tearDown(); + } +} Index: core/src/test/java/org/apache/hama/HamaCluster.java =================================================================== --- core/src/test/java/org/apache/hama/HamaCluster.java (revision 0) +++ core/src/test/java/org/apache/hama/HamaCluster.java (revision 0) @@ -0,0 +1,47 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Forming up the miniDfs and miniZooKeeper + */ +public abstract class HamaCluster extends HamaClusterTestCase { + public static final Log LOG = LogFactory.getLog(HamaCluster.class); + protected final static HamaConfiguration conf = new HamaConfiguration(); + + public HamaCluster(){ + super(); + } + + public HamaCluster(boolean startDfs) { + super(startDfs); + } + + protected void setUp() throws Exception { + super.setUp(); + } + + protected static HamaConfiguration getConf() { + return conf; + } +} Index: core/src/test/java/org/apache/hama/ipc/TestRPC.java =================================================================== --- core/src/test/java/org/apache/hama/ipc/TestRPC.java (revision 0) +++ core/src/test/java/org/apache/hama/ipc/TestRPC.java (revision 0) @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ipc; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.InetSocketAddress; +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.VersionedProtocol; + +import junit.framework.TestCase; + +public class TestRPC extends TestCase { + private static final int PORT = 1234; + private static final String ADDRESS = "0.0.0.0"; + + public static final Log LOG = LogFactory + .getLog("org.apache.hadoop.ipc.TestRPC"); + + private static Configuration conf = new Configuration(); + + public TestRPC(String name) { + super(name); + } + + public interface TestProtocol extends VersionedProtocol { + public static final long versionID = 1L; + + void ping() throws IOException; + + String echo(String value) throws IOException; + + String[] echo(String[] value) throws IOException; + + Writable echo(Writable value) throws IOException; + + int add(int v1, int v2) throws IOException; + + int add(int[] values) throws IOException; + + int error() throws IOException; + + void testServerGet() throws IOException; + } + + public class TestImpl implements TestProtocol { + + public long getProtocolVersion(String protocol, long clientVersion) { + return TestProtocol.versionID; + } + + public void ping() { + } + + public String echo(String value) throws IOException { + return value; + } + + public String[] echo(String[] values) throws IOException { + return values; + } + + public Writable echo(Writable writable) { + return writable; + } + + public int add(int v1, int v2) { + return v1 + v2; + } + + public int add(int[] values) { + int sum = 0; + for (int i = 0; i < values.length; i++) { + sum += values[i]; + } + return sum; + } + + public int error() throws IOException { + throw new IOException("bobo"); + } + + public void testServerGet() throws IOException { + if (!(Server.get() instanceof RPC.Server)) { + throw new IOException("Server.get() failed"); + } + } + + } + + public void testCalls() throws Exception { + Server server = RPC.getServer(new TestImpl(), ADDRESS, PORT, conf); + server.start(); + + InetSocketAddress addr = new InetSocketAddress(PORT); + TestProtocol proxy = (TestProtocol) RPC.getProxy(TestProtocol.class, + TestProtocol.versionID, addr, conf); + + proxy.ping(); + + String stringResult = proxy.echo("foo"); + assertEquals(stringResult, "foo"); + + stringResult = proxy.echo((String) null); + assertEquals(stringResult, null); + + String[] stringResults = proxy.echo(new String[] { "foo", "bar" }); + assertTrue(Arrays.equals(stringResults, new String[] { "foo", "bar" })); + + stringResults = proxy.echo((String[]) null); + assertTrue(Arrays.equals(stringResults, null)); + + int intResult = proxy.add(1, 2); + assertEquals(intResult, 3); + + intResult = proxy.add(new int[] { 1, 2 }); + assertEquals(intResult, 3); + + boolean caught = false; + try { + proxy.error(); + } catch (IOException e) { + LOG.debug("Caught " + e); + caught = true; + } + assertTrue(caught); + + proxy.testServerGet(); + + // try some multi-calls + Method echo = TestProtocol.class.getMethod("echo", + new Class[] { String.class }); + String[] strings = (String[]) RPC.call(echo, new String[][] { { "a" }, + { "b" } }, new InetSocketAddress[] { addr, addr }, null, conf); + assertTrue(Arrays.equals(strings, new String[] { "a", "b" })); + + Method ping = TestProtocol.class.getMethod("ping", new Class[] {}); + Object[] voids = (Object[]) RPC.call(ping, new Object[][] { {}, {} }, + new InetSocketAddress[] { addr, addr }, null, conf); + assertEquals(voids, null); + + server.stop(); + } + + public static void main(String[] args) throws Exception { + + new TestRPC("test").testCalls(); + + } + +} Index: core/src/test/java/org/apache/hama/ipc/TestIPC.java =================================================================== --- core/src/test/java/org/apache/hama/ipc/TestIPC.java (revision 0) +++ core/src/test/java/org/apache/hama/ipc/TestIPC.java (revision 0) @@ -0,0 +1,232 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ipc; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Random; + +import junit.framework.TestCase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.ipc.Client; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.StringUtils; + +public class TestIPC extends TestCase { + public static final Log LOG = LogFactory.getLog(TestIPC.class); + + final private static Configuration conf = new Configuration(); + final static private int PING_INTERVAL = 1000; + + static { + Client.setPingInterval(conf, PING_INTERVAL); + } + + public TestIPC(String name) { + super(name); + } + + private static final Random RANDOM = new Random(); + + private static final String ADDRESS = "0.0.0.0"; + + private static class TestServer extends Server { + private boolean sleep; + + public TestServer(int handlerCount, boolean sleep) throws IOException { + super(ADDRESS, 0, LongWritable.class, handlerCount, conf); + this.sleep = sleep; + } + + @Override + public Writable call(Class protocol, Writable param, long receiveTime) + throws IOException { + if (sleep) { + try { + Thread.sleep(RANDOM.nextInt(2 * PING_INTERVAL)); // sleep a bit + } catch (InterruptedException e) { + } + } + return param; // echo param as result + } + } + + private static class SerialCaller extends Thread { + private Client client; + private InetSocketAddress server; + private int count; + private boolean failed; + + public SerialCaller(Client client, InetSocketAddress server, int count) { + this.client = client; + this.server = server; + this.count = count; + } + + public void run() { + for (int i = 0; i < count; i++) { + try { + LongWritable param = new LongWritable(RANDOM.nextLong()); + LongWritable value = (LongWritable) client.call(param, server, null, + null); + if (!param.equals(value)) { + LOG.fatal("Call failed!"); + failed = true; + break; + } + } catch (Exception e) { + LOG.fatal("Caught: " + StringUtils.stringifyException(e)); + failed = true; + } + } + } + } + + private static class ParallelCaller extends Thread { + private Client client; + private int count; + private InetSocketAddress[] addresses; + private boolean failed; + + public ParallelCaller(Client client, InetSocketAddress[] addresses, + int count) { + this.client = client; + this.addresses = addresses; + this.count = count; + } + + public void run() { + for (int i = 0; i < count; i++) { + try { + Writable[] params = new Writable[addresses.length]; + for (int j = 0; j < addresses.length; j++) + params[j] = new LongWritable(RANDOM.nextLong()); + Writable[] values = client.call(params, addresses, null, null); + for (int j = 0; j < addresses.length; j++) { + if (!params[j].equals(values[j])) { + LOG.fatal("Call failed!"); + failed = true; + break; + } + } + } catch (Exception e) { + LOG.fatal("Caught: " + StringUtils.stringifyException(e)); + failed = true; + } + } + } + } + + public void testSerial() throws Exception { + testSerial(3, false, 2, 5, 100); + } + + public void testSerial(int handlerCount, boolean handlerSleep, + int clientCount, int callerCount, int callCount) throws Exception { + Server server = new TestServer(handlerCount, handlerSleep); + InetSocketAddress addr = NetUtils.getConnectAddress(server); + server.start(); + + Client[] clients = new Client[clientCount]; + for (int i = 0; i < clientCount; i++) { + clients[i] = new Client(LongWritable.class, conf); + } + + SerialCaller[] callers = new SerialCaller[callerCount]; + for (int i = 0; i < callerCount; i++) { + callers[i] = new SerialCaller(clients[i % clientCount], addr, callCount); + callers[i].start(); + } + for (int i = 0; i < callerCount; i++) { + callers[i].join(); + assertFalse(callers[i].failed); + } + for (int i = 0; i < clientCount; i++) { + clients[i].stop(); + } + server.stop(); + } + + public void testParallel() throws Exception { + testParallel(10, false, 2, 4, 2, 4, 100); + } + + public void testParallel(int handlerCount, boolean handlerSleep, + int serverCount, int addressCount, int clientCount, int callerCount, + int callCount) throws Exception { + Server[] servers = new Server[serverCount]; + for (int i = 0; i < serverCount; i++) { + servers[i] = new TestServer(handlerCount, handlerSleep); + servers[i].start(); + } + + InetSocketAddress[] addresses = new InetSocketAddress[addressCount]; + for (int i = 0; i < addressCount; i++) { + addresses[i] = NetUtils.getConnectAddress(servers[i % serverCount]); + } + + Client[] clients = new Client[clientCount]; + for (int i = 0; i < clientCount; i++) { + clients[i] = new Client(LongWritable.class, conf); + } + + ParallelCaller[] callers = new ParallelCaller[callerCount]; + for (int i = 0; i < callerCount; i++) { + callers[i] = new ParallelCaller(clients[i % clientCount], addresses, + callCount); + callers[i].start(); + } + for (int i = 0; i < callerCount; i++) { + callers[i].join(); + assertFalse(callers[i].failed); + } + for (int i = 0; i < clientCount; i++) { + clients[i].stop(); + } + for (int i = 0; i < serverCount; i++) { + servers[i].stop(); + } + } + + public void testStandAloneClient() throws Exception { + testParallel(10, false, 2, 4, 2, 4, 100); + Client client = new Client(LongWritable.class, conf); + InetSocketAddress address = new InetSocketAddress("127.0.0.1", 10); + try { + client.call(new LongWritable(RANDOM.nextLong()), address, null, null); + fail("Expected an exception to have been thrown"); + } catch (IOException e) { + String message = e.getMessage(); + String addressText = address.toString(); + assertTrue("Did not find " + addressText + " in " + message, message + .contains(addressText)); + Throwable cause = e.getCause(); + assertNotNull("No nested exception in " + e, cause); + String causeText = cause.getMessage(); + assertTrue("Did not find " + causeText + " in " + message, message + .contains(causeText)); + } + } + +} Index: core/src/test/java/org/apache/hama/util/TestNumeric.java =================================================================== --- core/src/test/java/org/apache/hama/util/TestNumeric.java (revision 0) +++ core/src/test/java/org/apache/hama/util/TestNumeric.java (revision 0) @@ -0,0 +1,34 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +import junit.framework.TestCase; + +public class TestNumeric extends TestCase { + final static int TEST_INT = 3; + final static double TEST_DOUBLE = 0.4; + + /** + * Double conversion test + */ + public void testDouble() { + assertEquals(Bytes.toDouble(Bytes.toBytes(TEST_DOUBLE)), TEST_DOUBLE); + } +} Index: core/src/test/java/org/apache/hama/util/TestBytes.java =================================================================== --- core/src/test/java/org/apache/hama/util/TestBytes.java (revision 0) +++ core/src/test/java/org/apache/hama/util/TestBytes.java (revision 0) @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +import java.io.IOException; +import java.util.Arrays; + +import junit.framework.TestCase; + +public class TestBytes extends TestCase { + public void testNullHashCode() { + byte [] b = null; + Exception ee = null; + try { + Bytes.hashCode(b); + } catch (Exception e) { + ee = e; + } + assertNotNull(ee); + } + + public void testSplit() throws Exception { + byte [] lowest = Bytes.toBytes("AAA"); + byte [] middle = Bytes.toBytes("CCC"); + byte [] highest = Bytes.toBytes("EEE"); + byte [][] parts = Bytes.split(lowest, highest, 1); + for (int i = 0; i < parts.length; i++) { + System.out.println(Bytes.toString(parts[i])); + } + assertEquals(3, parts.length); + assertTrue(Bytes.equals(parts[1], middle)); + // Now divide into three parts. Change highest so split is even. + highest = Bytes.toBytes("DDD"); + parts = Bytes.split(lowest, highest, 2); + for (int i = 0; i < parts.length; i++) { + System.out.println(Bytes.toString(parts[i])); + } + assertEquals(4, parts.length); + // Assert that 3rd part is 'CCC'. + assertTrue(Bytes.equals(parts[2], middle)); + } + + public void testSplit2() throws Exception { + // More split tests. + byte [] lowest = Bytes.toBytes("http://A"); + byte [] highest = Bytes.toBytes("http://z"); + byte [] middle = Bytes.toBytes("http://]"); + byte [][] parts = Bytes.split(lowest, highest, 1); + for (int i = 0; i < parts.length; i++) { + System.out.println(Bytes.toString(parts[i])); + } + assertEquals(3, parts.length); + assertTrue(Bytes.equals(parts[1], middle)); + } + + public void testToLong() throws Exception { + long [] longs = {-1l, 123l, 122232323232l}; + for (int i = 0; i < longs.length; i++) { + byte [] b = Bytes.toBytes(longs[i]); + assertEquals(longs[i], Bytes.toLong(b)); + } + } + + public void testToFloat() throws Exception { + float [] floats = {-1f, 123.123f, Float.MAX_VALUE}; + for (int i = 0; i < floats.length; i++) { + byte [] b = Bytes.toBytes(floats[i]); + assertEquals(floats[i], Bytes.toFloat(b)); + } + } + + public void testToDouble() throws Exception { + double [] doubles = {Double.MIN_VALUE, Double.MAX_VALUE}; + for (int i = 0; i < doubles.length; i++) { + byte [] b = Bytes.toBytes(doubles[i]); + assertEquals(doubles[i], Bytes.toDouble(b)); + } + } + + public void testBinarySearch() throws Exception { + byte [][] arr = { + {1}, + {3}, + {5}, + {7}, + {9}, + {11}, + {13}, + {15}, + }; + byte [] key1 = {3,1}; + byte [] key2 = {4,9}; + byte [] key2_2 = {4}; + byte [] key3 = {5,11}; + + assertEquals(1, Bytes.binarySearch(arr, key1, 0, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(0, Bytes.binarySearch(arr, key1, 1, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(-(2+1), Arrays.binarySearch(arr, key2_2, + Bytes.BYTES_COMPARATOR)); + assertEquals(-(2+1), Bytes.binarySearch(arr, key2, 0, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(4, Bytes.binarySearch(arr, key2, 1, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(2, Bytes.binarySearch(arr, key3, 0, 1, + Bytes.BYTES_RAWCOMPARATOR)); + assertEquals(5, Bytes.binarySearch(arr, key3, 1, 1, + Bytes.BYTES_RAWCOMPARATOR)); + } + + public void testIncrementBytes() throws IOException { + + assertTrue(checkTestIncrementBytes(10, 1)); + assertTrue(checkTestIncrementBytes(12, 123435445)); + assertTrue(checkTestIncrementBytes(124634654, 1)); + assertTrue(checkTestIncrementBytes(10005460, 5005645)); + assertTrue(checkTestIncrementBytes(1, -1)); + assertTrue(checkTestIncrementBytes(10, -1)); + assertTrue(checkTestIncrementBytes(10, -5)); + assertTrue(checkTestIncrementBytes(1005435000, -5)); + assertTrue(checkTestIncrementBytes(10, -43657655)); + assertTrue(checkTestIncrementBytes(-1, 1)); + assertTrue(checkTestIncrementBytes(-26, 5034520)); + assertTrue(checkTestIncrementBytes(-10657200, 5)); + assertTrue(checkTestIncrementBytes(-12343250, 45376475)); + assertTrue(checkTestIncrementBytes(-10, -5)); + assertTrue(checkTestIncrementBytes(-12343250, -5)); + assertTrue(checkTestIncrementBytes(-12, -34565445)); + assertTrue(checkTestIncrementBytes(-1546543452, -34565445)); + } + + private static boolean checkTestIncrementBytes(long val, long amount) + throws IOException { + byte[] value = Bytes.toBytes(val); + byte [] testValue = {-1, -1, -1, -1, -1, -1, -1, -1}; + if (value[0] > 0) { + testValue = new byte[Bytes.SIZEOF_LONG]; + } + System.arraycopy(value, 0, testValue, testValue.length - value.length, + value.length); + + long incrementResult = Bytes.toLong(Bytes.incrementBytes(value, amount)); + + return (Bytes.toLong(testValue) + amount) == incrementResult; + } +} Index: core/src/test/java/org/apache/hama/util/TestRandomVariable.java =================================================================== --- core/src/test/java/org/apache/hama/util/TestRandomVariable.java (revision 0) +++ core/src/test/java/org/apache/hama/util/TestRandomVariable.java (revision 0) @@ -0,0 +1,74 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +import org.apache.log4j.Logger; + +import junit.framework.TestCase; + +/** + * Random variable generation test + */ +public class TestRandomVariable extends TestCase { + static final Logger LOG = Logger.getLogger(TestRandomVariable.class); + final static int COUNT = 50; + + /** + * Random object test + * + * @throws Exception + */ + public void testRand() throws Exception { + for (int i = 0; i < COUNT; i++) { + double result = RandomVariable.rand(); + assertTrue(result >= 0.0d && result <= 1.0); + } + } + + /** + * Random integer test + * + * @throws Exception + */ + public void testRandInt() throws Exception { + final int min = 122; + final int max = 561; + + for (int i = 0; i < COUNT; i++) { + int result = RandomVariable.randInt(min, max); + assertTrue(result >= min && result <= max); + } + } + + /** + * Uniform test + * + * @throws Exception + */ + public void testUniform() throws Exception { + final double min = 1.0d; + final double max = 3.0d; + + for (int i = 0; i < COUNT; i++) { + double result = RandomVariable.uniform(min, max); + assertTrue(result >= min && result <= max); + } + } +} Index: core/src/test/java/testjar/testjob.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: core/src/test/java/testjar/testjob.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: core/src/test/java/testjar/ClassSerializePrinting.java =================================================================== --- core/src/test/java/testjar/ClassSerializePrinting.java (revision 0) +++ core/src/test/java/testjar/ClassSerializePrinting.java (revision 0) @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package testjar; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.apache.hadoop.io.Text; +import org.apache.hama.bsp.BSP; +import org.apache.hama.bsp.BSPPeer; +import org.apache.zookeeper.KeeperException; + +public class ClassSerializePrinting { + private static String TMP_OUTPUT = "/tmp/test-example/"; + + public static class HelloBSP extends BSP { + public static final Log LOG = LogFactory.getLog(HelloBSP.class); + private Configuration conf; + private final static int PRINT_INTERVAL = 1000; + private FileSystem fileSys; + private int num; + + public void bsp(BSPPeer bspPeer) throws IOException, + KeeperException, InterruptedException { + + int i = 0; + for (String otherPeer : bspPeer.getAllPeerNames()) { + String peerName = bspPeer.getPeerName(); + if (peerName.equals(otherPeer)) { + writeLogToFile(peerName, i); + } + + Thread.sleep(PRINT_INTERVAL); + bspPeer.sync(); + i++; + } + } + + private void writeLogToFile(String string, int i) throws IOException { + SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, + new Path(TMP_OUTPUT + i), LongWritable.class, Text.class, + CompressionType.NONE); + writer.append(new LongWritable(System.currentTimeMillis()), new Text( + "Hello BSP from " + (i + 1) + " of " + num + ": " + string)); + writer.close(); + } + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration conf) { + this.conf = conf; + num = Integer.parseInt(conf.get("bsp.peers.num")); + try { + fileSys = FileSystem.get(conf); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + +} Index: core/src/main/java/org/apache/hama/http/HttpServer.java =================================================================== --- core/src/main/java/org/apache/hama/http/HttpServer.java (revision 0) +++ core/src/main/java/org/apache/hama/http/HttpServer.java (revision 0) @@ -0,0 +1,501 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.http; + +import java.io.IOException; +import java.io.PrintWriter; +import java.net.BindException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.log.LogLevel; +import org.apache.hadoop.util.ReflectionUtils; +import org.mortbay.jetty.Connector; +import org.mortbay.jetty.Handler; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.handler.ContextHandlerCollection; +import org.mortbay.jetty.nio.SelectChannelConnector; +import org.mortbay.jetty.security.SslSocketConnector; +import org.mortbay.jetty.servlet.Context; +import org.mortbay.jetty.servlet.DefaultServlet; +import org.mortbay.jetty.servlet.FilterHolder; +import org.mortbay.jetty.servlet.FilterMapping; +import org.mortbay.jetty.servlet.ServletHandler; +import org.mortbay.jetty.servlet.ServletHolder; +import org.mortbay.jetty.webapp.WebAppContext; +import org.mortbay.thread.BoundedThreadPool; +import org.mortbay.util.MultiException; + +/** + * Create a Jetty embedded server to answer http requests. The primary goal is + * to serve up status information for the server. There are three contexts: + * "/logs/" -> points to the log directory "/static/" -> points to common static + * files (src/main/webapp/static) "/" -> the jsp server code from + * (src/main/webapp/) + */ +public class HttpServer { + public static final Log LOG = LogFactory.getLog(HttpServer.class); + + static final String FILTER_INITIALIZER_PROPERTY = "hama.http.filter.initializers"; + + protected final Server webServer; + protected final Connector listener; + protected final WebAppContext webAppContext; + protected final boolean findPort; + protected final Map defaultContexts = new HashMap(); + protected final List filterNames = new ArrayList(); + private static final int MAX_RETRIES = 10; + + /** Same as this(name, bindAddress, port, findPort, null); */ + public HttpServer(String name, String bindAddress, int port, boolean findPort) + throws IOException { + this(name, bindAddress, port, findPort, new Configuration()); + } + + /** + * Create a status server on the given port. The jsp scripts are taken from + * src/main/webapp/. + * + * @param name The name of the server + * @param port The port to use on the server + * @param findPort whether the server should start at the given port and + * increment by 1 until it finds a free port. + * @param conf Configuration + */ + public HttpServer(String name, String bindAddress, int port, + boolean findPort, Configuration conf) throws IOException { + webServer = new Server(); + this.findPort = findPort; + + listener = createBaseListener(conf); + listener.setHost(bindAddress); + listener.setPort(port); + webServer.addConnector(listener); + + webServer.setThreadPool(new BoundedThreadPool()); + + final String appDir = getWebAppsPath(); + ContextHandlerCollection contexts = new ContextHandlerCollection(); + webServer.setHandler(contexts); + webAppContext = new WebAppContext(); + + System.setProperty("java.naming.factory.initial", + "org.mortbay.naming.InitialContextFactory"); + System.setProperty("java.naming.factory.url.pkgs", "org.mortbay.naming"); + + webAppContext.setContextPath("/"); + webAppContext.setWar(appDir + "/" + name); + webServer.addHandler(webAppContext); + + addDefaultApps(contexts, appDir); + + addDefaultServlets(); + } + + /** + * Create a required listener for the Jetty instance listening on the port + * provided. This wrapper and all subclasses must create at least one + * listener. + */ + protected Connector createBaseListener(Configuration conf) throws IOException { + SelectChannelConnector ret = new SelectChannelConnector(); + ret.setLowResourceMaxIdleTime(10000); + ret.setAcceptQueueSize(128); + ret.setResolveNames(false); + ret.setUseDirectBuffers(false); + return ret; + } + + /** + * Add default apps. + * + * @param appDir The application directory + * @throws IOException + */ + protected void addDefaultApps(ContextHandlerCollection parent, + final String appDir) throws IOException { + // set up the context for "/logs/" if "hama.log.dir" property is defined. + String logDir = System.getProperty("hama.log.dir"); + if (logDir != null) { + Context logContext = new Context(parent, "/logs"); + logContext.setResourceBase(logDir); + logContext.addServlet(DefaultServlet.class, "/"); + defaultContexts.put(logContext, true); + } + // set up the context for "/static/*" + Context staticContext = new Context(parent, "/static"); + staticContext.setResourceBase(appDir + "/static"); + staticContext.addServlet(DefaultServlet.class, "/*"); + defaultContexts.put(staticContext, true); + } + + /** + * Add default servlets. + */ + protected void addDefaultServlets() { + // set up default servlets + addServlet("stacks", "/stacks", StackServlet.class); + addServlet("logLevel", "/logLevel", LogLevel.Servlet.class); + } + + public void addContext(Context ctxt, boolean isFiltered) throws IOException { + webServer.addHandler(ctxt); + defaultContexts.put(ctxt, isFiltered); + } + + /** + * Add a context + * + * @param pathSpec The path spec for the context + * @param dir The directory containing the context + * @param isFiltered if true, the servlet is added to the filter path mapping + * @throws IOException + */ + protected void addContext(String pathSpec, String dir, boolean isFiltered) + throws IOException { + if (0 == webServer.getHandlers().length) { + throw new RuntimeException("Couldn't find handler"); + } + WebAppContext webAppCtx = new WebAppContext(); + webAppCtx.setContextPath(pathSpec); + webAppCtx.setWar(dir); + addContext(webAppCtx, true); + } + + /** + * Set a value in the webapp context. These values are available to the jsp + * pages as "application.getAttribute(name)". + * + * @param name The name of the attribute + * @param value The value of the attribute + */ + public void setAttribute(String name, Object value) { + webAppContext.setAttribute(name, value); + } + + /** + * Add a servlet in the server. + * + * @param name The name of the servlet (can be passed as null) + * @param pathSpec The path spec for the servlet + * @param clazz The servlet class + */ + public void addServlet(String name, String pathSpec, + Class clazz) { + addInternalServlet(name, pathSpec, clazz); + addFilterPathMapping(pathSpec, webAppContext); + } + + /** + * Add an internal servlet in the server. + * + * @param name The name of the servlet (can be passed as null) + * @param pathSpec The path spec for the servlet + * @param clazz The servlet class + * @deprecated this is a temporary method + */ + @Deprecated + public void addInternalServlet(String name, String pathSpec, + Class clazz) { + ServletHolder holder = new ServletHolder(clazz); + if (name != null) { + holder.setName(name); + } + webAppContext.addServlet(holder, pathSpec); + } + + /** + * Define a filter for a context and set up default url mappings. + */ + protected void defineFilter(Context ctx, String name, String classname, + Map parameters, String[] urls) { + + FilterHolder holder = new FilterHolder(); + holder.setName(name); + holder.setClassName(classname); + holder.setInitParameters(parameters); + FilterMapping fmap = new FilterMapping(); + fmap.setPathSpecs(urls); + fmap.setDispatches(Handler.ALL); + fmap.setFilterName(name); + ServletHandler handler = ctx.getServletHandler(); + handler.addFilter(holder, fmap); + } + + /** + * Add the path spec to the filter path mapping. + * + * @param pathSpec The path spec + * @param webAppCtx The WebApplicationContext to add to + */ + protected void addFilterPathMapping(String pathSpec, Context webAppCtx) { + ServletHandler handler = webAppCtx.getServletHandler(); + for (String name : filterNames) { + FilterMapping fmap = new FilterMapping(); + fmap.setPathSpec(pathSpec); + fmap.setFilterName(name); + fmap.setDispatches(Handler.ALL); + handler.addFilterMapping(fmap); + } + } + + /** + * Get the value in the webapp context. + * + * @param name The name of the attribute + * @return The value of the attribute + */ + public Object getAttribute(String name) { + return webAppContext.getAttribute(name); + } + + /** + * Get the pathname to the webapps files. + * + * @return the pathname as a URL + * @throws IOException if 'webapps' directory cannot be found on CLASSPATH. + */ + protected String getWebAppsPath() throws IOException { + // URL url = BSPMaster.class.getClassLoader().getResource("webapps"); + // if (url == null) + // throw new IOException("webapps not found in CLASSPATH"); + // return url.toString(); + return "src/main/webapp"; + } + + /** + * Get the port that the server is on + * + * @return the port + */ + public int getPort() { + return webServer.getConnectors()[0].getLocalPort(); + } + + /** + * Set the min, max number of worker threads (simultaneous connections). + */ + public void setThreads(int min, int max) { + BoundedThreadPool pool = (BoundedThreadPool) webServer.getThreadPool(); + pool.setMinThreads(min); + pool.setMaxThreads(max); + } + + /** + * Configure an ssl listener on the server. + * + * @param addr address to listen on + * @param keystore location of the keystore + * @param storPass password for the keystore + * @param keyPass password for the key + * @deprecated Use + * {@link #addSslListener(InetSocketAddress, Configuration, boolean)} + */ + @Deprecated + public void addSslListener(InetSocketAddress addr, String keystore, + String storPass, String keyPass) throws IOException { + if (webServer.isStarted()) { + throw new IOException("Failed to add ssl listener"); + } + SslSocketConnector sslListener = new SslSocketConnector(); + sslListener.setHost(addr.getHostName()); + sslListener.setPort(addr.getPort()); + sslListener.setKeystore(keystore); + sslListener.setPassword(storPass); + sslListener.setKeyPassword(keyPass); + webServer.addConnector(sslListener); + } + + /** + * Configure an ssl listener on the server. + * + * @param addr address to listen on + * @param sslConf conf to retrieve ssl options + * @param needClientAuth whether client authentication is required + */ + public void addSslListener(InetSocketAddress addr, Configuration sslConf, + boolean needClientAuth) throws IOException { + if (webServer.isStarted()) { + throw new IOException("Failed to add ssl listener"); + } + if (needClientAuth) { + // setting up SSL truststore for authenticating clients + System.setProperty("javax.net.ssl.trustStore", sslConf.get( + "ssl.server.truststore.location", "")); + System.setProperty("javax.net.ssl.trustStorePassword", sslConf.get( + "ssl.server.truststore.password", "")); + System.setProperty("javax.net.ssl.trustStoreType", sslConf.get( + "ssl.server.truststore.type", "jks")); + } + SslSocketConnector sslListener = new SslSocketConnector(); + sslListener.setHost(addr.getHostName()); + sslListener.setPort(addr.getPort()); + sslListener.setKeystore(sslConf.get("ssl.server.keystore.location")); + sslListener.setPassword(sslConf.get("ssl.server.keystore.password", "")); + sslListener.setKeyPassword(sslConf.get("ssl.server.keystore.keypassword", + "")); + sslListener.setKeystoreType(sslConf.get("ssl.server.keystore.type", "jks")); + sslListener.setNeedClientAuth(needClientAuth); + webServer.addConnector(sslListener); + } + + /** + * Start the server. Does not wait for the server to start. + */ + public void start() throws IOException { + try { + int port = 0; + int oriPort = listener.getPort(); // The original requested port + while (true) { + try { + port = webServer.getConnectors()[0].getLocalPort(); + LOG.info("Port returned by webServer.getConnectors()[0]." + + "getLocalPort() before open() is " + port + + ". Opening the listener on " + oriPort); + listener.open(); + port = listener.getLocalPort(); + LOG.info("listener.getLocalPort() returned " + + listener.getLocalPort() + + " webServer.getConnectors()[0].getLocalPort() returned " + + webServer.getConnectors()[0].getLocalPort()); + // Workaround to handle the problem reported in HADOOP-4744 + if (port < 0) { + Thread.sleep(100); + int numRetries = 1; + while (port < 0) { + LOG.warn("listener.getLocalPort returned " + port); + if (numRetries++ > MAX_RETRIES) { + throw new Exception(" listener.getLocalPort is returning " + + "less than 0 even after " + numRetries + " resets"); + } + for (int i = 0; i < 2; i++) { + LOG.info("Retrying listener.getLocalPort()"); + port = listener.getLocalPort(); + if (port > 0) { + break; + } + Thread.sleep(200); + } + if (port > 0) { + break; + } + LOG.info("Bouncing the listener"); + listener.close(); + Thread.sleep(1000); + listener.setPort(oriPort == 0 ? 0 : (oriPort += 1)); + listener.open(); + Thread.sleep(100); + port = listener.getLocalPort(); + } + } // Workaround end + LOG.info("Jetty bound to port " + port); + webServer.start(); + // Workaround for HADOOP-6386 + port = listener.getLocalPort(); + if (port < 0) { + LOG.warn("Bounds port is " + port + " after webserver start"); + for (int i = 0; i < MAX_RETRIES / 2; i++) { + try { + webServer.stop(); + } catch (Exception e) { + LOG.warn("Can't stop web-server", e); + } + Thread.sleep(1000); + + listener.setPort(oriPort == 0 ? 0 : (oriPort += 1)); + listener.open(); + Thread.sleep(100); + webServer.start(); + LOG.info(i + "attempts to restart webserver"); + port = listener.getLocalPort(); + if (port > 0) + break; + } + if (port < 0) + throw new Exception("listener.getLocalPort() is returning " + + "less than 0 even after " + MAX_RETRIES + " resets"); + } + // End of HADOOP-6386 workaround + break; + } catch (IOException ex) { + // if this is a bind exception, + // then try the next port number. + if (ex instanceof BindException) { + if (!findPort) { + throw (BindException) ex; + } + } else { + LOG.info("HttpServer.start() threw a non Bind IOException"); + throw ex; + } + } catch (MultiException ex) { + LOG.info("HttpServer.start() threw a MultiException"); + throw ex; + } + listener.setPort((oriPort += 1)); + } + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("Problem starting http server", e); + } + } + + /** + * stop the server + */ + public void stop() throws Exception { + listener.close(); + webServer.stop(); + } + + public void join() throws InterruptedException { + webServer.join(); + } + + /** + * A very simple servlet to serve up a text representation of the current + * stack traces. It both returns the stacks to the caller and logs them. + * Currently the stack traces are done sequentially rather than exactly the + * same data. + */ + public static class StackServlet extends HttpServlet { + private static final long serialVersionUID = -6284183679759467039L; + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + PrintWriter out = new PrintWriter(response.getOutputStream()); + ReflectionUtils.printThreadInfo(out, ""); + out.close(); + ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1); + } + } +} Index: core/src/main/java/org/apache/hama/http/package.html =================================================================== --- core/src/main/java/org/apache/hama/http/package.html (revision 0) +++ core/src/main/java/org/apache/hama/http/package.html (revision 0) @@ -0,0 +1,23 @@ + + + + + +Contains the administrative web interfaces. + + Index: core/src/main/java/org/apache/hama/MiniZooKeeperCluster.java =================================================================== --- core/src/main/java/org/apache/hama/MiniZooKeeperCluster.java (revision 0) +++ core/src/main/java/org/apache/hama/MiniZooKeeperCluster.java (revision 0) @@ -0,0 +1,216 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.net.BindException; +import java.net.InetSocketAddress; +import java.net.Socket; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.apache.zookeeper.server.NIOServerCnxn; +import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.persistence.FileTxnLog; + +/** + * This class starts and runs the MiniZookeeperCluster. + */ +public class MiniZooKeeperCluster { + private static final Log LOG = LogFactory.getLog(MiniZooKeeperCluster.class); + + private static final int TICK_TIME = 2000; + private static final int CONNECTION_TIMEOUT = 30000; + + private boolean started; + private int clientPort = 21810; // use non-standard port + + private NIOServerCnxn.Factory standaloneServerFactory; + private int tickTime = 0; + + /** Create mini Zookeeper cluster. */ + public MiniZooKeeperCluster() { + this.started = false; + } + + public void setClientPort(int clientPort) { + this.clientPort = clientPort; + } + + public void setTickTime(int tickTime) { + this.tickTime = tickTime; + } + + private static void setupTestEnv() { + // during the tests we run with 100K prealloc in the logs. + // on windows systems prealloc of 64M was seen to take ~15seconds + // resulting in test failure (client timeout on first session). + // set env and directly in order to handle static init/gc issues + System.setProperty("zookeeper.preAllocSize", "100"); + FileTxnLog.setPreallocSize(100); + } + + /** + * @param baseDir + * @return ClientPort server bound to. + * @throws IOException + * @throws InterruptedException + */ + public int startup(File baseDir) throws IOException, + InterruptedException { + + setupTestEnv(); + + shutdown(); + + File dir = new File(baseDir, "zookeeper").getAbsoluteFile(); + recreateDir(dir); + + int tickTimeToUse; + if (this.tickTime > 0) { + tickTimeToUse = this.tickTime; + } else { + tickTimeToUse = TICK_TIME; + } + ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse); + while (true) { + try { + standaloneServerFactory = + new NIOServerCnxn.Factory(new InetSocketAddress(clientPort)); + } catch (BindException e) { + LOG.info("Faild binding ZK Server to client port: " + clientPort); + //this port is already in use. try to use another + clientPort++; + continue; + } + break; + } + standaloneServerFactory.startup(server); + + if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for startup of standalone server"); + } + + started = true; + + return clientPort; + } + + private void recreateDir(File dir) throws IOException { + if (dir.exists()) { + FileUtil.fullyDelete(dir); + } + try { + dir.mkdirs(); + } catch (SecurityException e) { + throw new IOException("creating dir: " + dir, e); + } + } + + /** + * @throws IOException + */ + public void shutdown() throws IOException { + if (!started) { + return; + } + + standaloneServerFactory.shutdown(); + if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for shutdown of standalone server"); + } + + started = false; + } + + // XXX: From o.a.zk.t.ClientBase + private static boolean waitForServerDown(int port, long timeout) { + long start = System.currentTimeMillis(); + while (true) { + try { + Socket sock = new Socket("localhost", port); + try { + OutputStream outstream = sock.getOutputStream(); + outstream.write("stat".getBytes()); + outstream.flush(); + } finally { + sock.close(); + } + } catch (IOException e) { + return true; + } + + if (System.currentTimeMillis() > start + timeout) { + break; + } + try { + Thread.sleep(250); + } catch (InterruptedException e) { + // ignore + } + } + return false; + } + + // XXX: From o.a.zk.t.ClientBase + private static boolean waitForServerUp(int port, long timeout) { + long start = System.currentTimeMillis(); + while (true) { + try { + Socket sock = new Socket("localhost", port); + BufferedReader reader = null; + try { + OutputStream outstream = sock.getOutputStream(); + outstream.write("stat".getBytes()); + outstream.flush(); + + Reader isr = new InputStreamReader(sock.getInputStream()); + reader = new BufferedReader(isr); + String line = reader.readLine(); + if (line != null && line.startsWith("Zookeeper version:")) { + return true; + } + } finally { + sock.close(); + if (reader != null) { + reader.close(); + } + } + } catch (IOException e) { + // ignore as this is expected + LOG.info("server localhost:" + port + " not up " + e); + } + + if (System.currentTimeMillis() > start + timeout) { + break; + } + try { + Thread.sleep(250); + } catch (InterruptedException e) { + // ignore + } + } + return false; + } +} Index: core/src/main/java/org/apache/hama/zookeeper/QuorumPeer.java =================================================================== --- core/src/main/java/org/apache/hama/zookeeper/QuorumPeer.java (revision 0) +++ core/src/main/java/org/apache/hama/zookeeper/QuorumPeer.java (revision 0) @@ -0,0 +1,354 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.zookeeper; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Properties; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.DNS; +import org.apache.hadoop.util.StringUtils; +import org.apache.hama.Constants; +import org.apache.hama.HamaConfiguration; +import org.apache.zookeeper.server.ServerConfig; +import org.apache.zookeeper.server.ZooKeeperServerMain; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.apache.zookeeper.server.quorum.QuorumPeerMain; + +/** + * This class starts and runs the QuorumPeers + */ +public class QuorumPeer implements Constants { + private static final Log LOG = LogFactory.getLog(QuorumPeer.class); + + private static final String VARIABLE_START = "${"; + private static final int VARIABLE_START_LENGTH = VARIABLE_START.length(); + private static final String VARIABLE_END = "}"; + private static final int VARIABLE_END_LENGTH = VARIABLE_END.length(); + + private static final String ZK_CFG_PROPERTY = "hama.zookeeper.property."; + private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length(); + + /** + * Parse ZooKeeper configuration from Hama XML config and run a QuorumPeer. + * @param baseConf Hadoop Configuration. + */ + public static void run(Configuration baseConf) { + Configuration conf = new HamaConfiguration(baseConf); + try { + Properties zkProperties = makeZKProps(conf); + writeMyID(zkProperties); + QuorumPeerConfig zkConfig = new QuorumPeerConfig(); + zkConfig.parseProperties(zkProperties); + runZKServer(zkConfig); + } catch (Exception e) { + LOG.error("Exception during ZooKeeper startup - exiting...",e); + System.exit(-1); + } + } + + private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException { + if (zkConfig.isDistributed()) { + QuorumPeerMain qp = new QuorumPeerMain(); + qp.runFromConfig(zkConfig); + } else { + ZooKeeperServerMain zk = new ZooKeeperServerMain(); + ServerConfig serverConfig = new ServerConfig(); + serverConfig.readFrom(zkConfig); + zk.runFromConfig(serverConfig); + } + } + + private static boolean addressIsLocalHost(String address) { + return address.equals("localhost") || address.equals("127.0.0.1"); + } + + private static void writeMyID(Properties properties) throws IOException { + long myId = -1; + + Configuration conf = new HamaConfiguration(); + String myAddress = DNS.getDefaultHost( + conf.get("hama.zookeeper.dns.interface","default"), + conf.get("hama.zookeeper.dns.nameserver","default")); + + List ips = new ArrayList(); + + // Add what could be the best (configured) match + ips.add(myAddress.contains(".") ? + myAddress : + StringUtils.simpleHostname(myAddress)); + + // For all nics get all hostnames and IPs + Enumeration nics = NetworkInterface.getNetworkInterfaces(); + while(nics.hasMoreElements()) { + Enumeration rawAdrs = + ((NetworkInterface)nics.nextElement()).getInetAddresses(); + while(rawAdrs.hasMoreElements()) { + InetAddress inet = (InetAddress) rawAdrs.nextElement(); + ips.add(StringUtils.simpleHostname(inet.getHostName())); + ips.add(inet.getHostAddress()); + } + } + + for (Entry entry : properties.entrySet()) { + String key = entry.getKey().toString().trim(); + String value = entry.getValue().toString().trim(); + + if (key.startsWith("server.")) { + int dot = key.indexOf('.'); + long id = Long.parseLong(key.substring(dot + 1)); + String[] parts = value.split(":"); + String address = parts[0]; + if (addressIsLocalHost(address) || ips.contains(address)) { + myId = id; + break; + } + } + } + + if (myId == -1) { + throw new IOException("Could not find my address: " + myAddress + + " in list of ZooKeeper quorum servers"); + } + + String dataDirStr = properties.get("dataDir").toString().trim(); + File dataDir = new File(dataDirStr); + if (!dataDir.isDirectory()) { + if (!dataDir.mkdirs()) { + throw new IOException("Unable to create data dir " + dataDir); + } + } + + File myIdFile = new File(dataDir, "myid"); + PrintWriter w = new PrintWriter(myIdFile); + w.println(myId); + w.close(); + } + + /** + * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg. + * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse + * the corresponding config options from the Hama XML configs and generate + * the appropriate ZooKeeper properties. + * @param conf Configuration to read from. + * @return Properties holding mappings representing ZooKeeper zoo.cfg file. + */ + public static Properties makeZKProps(Configuration conf) { + // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read + // it and grab its configuration properties. + ClassLoader cl = QuorumPeer.class.getClassLoader(); + InputStream inputStream = cl.getResourceAsStream(ZOOKEEPER_CONFIG_NAME); + if (inputStream != null) { + try { + return parseZooCfg(conf, inputStream); + } catch (IOException e) { + LOG.warn("Cannot read " + ZOOKEEPER_CONFIG_NAME + + ", loading from XML files", e); + } + } + + // Otherwise, use the configuration options from Hama's XML files. + Properties zkProperties = new Properties(); + + // Set the max session timeout from the provided client-side timeout + zkProperties.setProperty("maxSessionTimeout", + conf.get(Constants.ZOOKEEPER_SESSION_TIMEOUT, "1200000")); + + // Directly map all of the hama.zookeeper.property.KEY properties. + for (Entry entry : conf) { + String key = entry.getKey(); + if (key.startsWith(ZK_CFG_PROPERTY)) { + String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE); + String value = entry.getValue(); + // If the value has variables substitutions, need to do a get. + if (value.contains(VARIABLE_START)) { + value = conf.get(key); + } + zkProperties.put(zkKey, value); + } + } + + // If clientPort is not set, assign the default + if (zkProperties.getProperty(ZOOKEEPER_CLIENT_PORT) == null) { + zkProperties.put(ZOOKEEPER_CLIENT_PORT, DEFAULT_ZOOKEEPER_CLIENT_PORT); + } + + // Create the server.X properties. + int peerPort = conf.getInt("hama.zookeeper.peerport", 2888); + int leaderPort = conf.getInt("hama.zookeeper.leaderport", 3888); + + String[] serverHosts = conf.getStrings(ZOOKEEPER_QUORUM, "localhost"); + for (int i = 0; i < serverHosts.length; ++i) { + String serverHost = serverHosts[i]; + String address = serverHost + ":" + peerPort + ":" + leaderPort; + String key = "server." + i; + zkProperties.put(key, address); + } + + return zkProperties; + } + + /** + * Parse ZooKeeper's zoo.cfg, injecting Hama Configuration variables in. + * This method is used for testing so we can pass our own InputStream. + * @param conf Configuration to use for injecting variables. + * @param inputStream InputStream to read from. + * @return Properties parsed from config stream with variables substituted. + * @throws IOException if anything goes wrong parsing config + */ + public static Properties parseZooCfg(Configuration conf, + InputStream inputStream) throws IOException { + Properties properties = new Properties(); + try { + properties.load(inputStream); + } catch (IOException e) { + String msg = "fail to read properties from " + ZOOKEEPER_CONFIG_NAME; + LOG.fatal(msg); + throw new IOException(msg, e); + } + for (Entry entry : properties.entrySet()) { + String value = entry.getValue().toString().trim(); + String key = entry.getKey().toString().trim(); + StringBuilder newValue = new StringBuilder(); + int varStart = value.indexOf(VARIABLE_START); + int varEnd = 0; + while (varStart != -1) { + varEnd = value.indexOf(VARIABLE_END, varStart); + if (varEnd == -1) { + String msg = "variable at " + varStart + " has no end marker"; + LOG.fatal(msg); + throw new IOException(msg); + } + String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd); + + String substituteValue = System.getProperty(variable); + if (substituteValue == null) { + substituteValue = conf.get(variable); + } + if (substituteValue == null) { + String msg = "variable " + variable + " not set in system property " + + "or hama configs"; + LOG.fatal(msg); + throw new IOException(msg); + } + + newValue.append(substituteValue); + + varEnd += VARIABLE_END_LENGTH; + varStart = value.indexOf(VARIABLE_START, varEnd); + } + // Special case for 'hama.cluster.distributed' property being 'true' + if (key.startsWith("server.")) { + if (conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) && + value.startsWith("localhost")) { + String msg = "The server in zoo.cfg cannot be set to localhost " + + "in a fully-distributed setup because it won't be reachable. " + + "See \"Getting Started\" for more information."; + LOG.fatal(msg); + throw new IOException(msg); + } + } + newValue.append(value.substring(varEnd)); + properties.setProperty(key, newValue.toString()); + } + return properties; + } + + /** + * Return the ZK Quorum servers string given zk properties returned by + * makeZKProps + * @param properties + * @return Quorum servers String + */ + public static String getZKQuorumServersString(Properties properties) { + String clientPort = null; + List servers = new ArrayList(); + + // The clientPort option may come after the server.X hosts, so we need to + // grab everything and then create the final host:port comma separated list. + boolean anyValid = false; + for (Entry property : properties.entrySet()) { + String key = property.getKey().toString().trim(); + String value = property.getValue().toString().trim(); + if (key.equals("clientPort")) { + clientPort = value; + } + else if (key.startsWith("server.")) { + String host = value.substring(0, value.indexOf(':')); + servers.add(host); + try { + //noinspection ResultOfMethodCallIgnored + InetAddress.getByName(host); + anyValid = true; + } catch (UnknownHostException e) { + LOG.warn(StringUtils.stringifyException(e)); + } + } + } + + if (!anyValid) { + LOG.error("no valid quorum servers found in " + Constants.ZOOKEEPER_CONFIG_NAME); + return null; + } + + if (clientPort == null) { + LOG.error("no clientPort found in " + Constants.ZOOKEEPER_CONFIG_NAME); + return null; + } + + if (servers.isEmpty()) { + LOG.fatal("No server.X lines found in conf/zoo.cfg. Hama must have a " + + "ZooKeeper cluster configured for its operation."); + return null; + } + + StringBuilder hostPortBuilder = new StringBuilder(); + for (int i = 0; i < servers.size(); ++i) { + String host = servers.get(i); + if (i > 0) { + hostPortBuilder.append(','); + } + hostPortBuilder.append(host); + hostPortBuilder.append(':'); + hostPortBuilder.append(clientPort); + } + + return hostPortBuilder.toString(); + } + + /** + * Return the ZK Quorum servers string given the specified configuration. + * @param conf + * @return Quorum servers + */ + public static String getZKQuorumServersString(Configuration conf) { + return getZKQuorumServersString(makeZKProps(conf)); + } +} Index: core/src/main/java/org/apache/hama/zookeeper/ZKServerTool.java =================================================================== --- core/src/main/java/org/apache/hama/zookeeper/ZKServerTool.java (revision 0) +++ core/src/main/java/org/apache/hama/zookeeper/ZKServerTool.java (revision 0) @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.zookeeper; + +import java.util.Properties; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hama.HamaConfiguration; + +/** + * A tool class for Zookeeper use. + */ +public class ZKServerTool { + + /** + * Run the tool. + * + * @param args Command line arguments. First arg is path to zookeepers file. + */ + public static void main(String args[]) { + Configuration conf = new HamaConfiguration(); + // Note that we do not simply grab the property ZOOKEEPER_QUORUM from + // the HamaConfiguration because the user may be using a zoo.cfg file. + Properties zkProps = QuorumPeer.makeZKProps(conf); + for (Entry entry : zkProps.entrySet()) { + String key = entry.getKey().toString().trim(); + String value = entry.getValue().toString().trim(); + if (key.startsWith("server.")) { + String[] parts = value.split(":"); + String host = parts[0]; + System.out.println(host); + } + } + } +} Index: core/src/main/java/org/apache/hama/zookeeper/package.html =================================================================== --- core/src/main/java/org/apache/hama/zookeeper/package.html (revision 0) +++ core/src/main/java/org/apache/hama/zookeeper/package.html (revision 0) @@ -0,0 +1,23 @@ + + + + + +Tools to start the ZooKeeper server and quorum peers. + + Index: core/src/main/java/org/apache/hama/GroomServerRunner.java =================================================================== --- core/src/main/java/org/apache/hama/GroomServerRunner.java (revision 0) +++ core/src/main/java/org/apache/hama/GroomServerRunner.java (revision 0) @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hama.bsp.GroomServer; + +/** + * This class starts and runs the GroomServer. + */ +public class GroomServerRunner extends Configured implements Tool { + + public static final Log LOG = LogFactory.getLog(GroomServerRunner.class); + + @Override + public int run(String[] args) throws Exception { + StringUtils.startupShutdownMessage(GroomServer.class, args, LOG); + + if (args.length != 0) { + System.out.println("usage: GroomServerRunner"); + System.exit(-1); + } + + try { + Configuration conf = new HamaConfiguration(getConf()); + GroomServer groom = GroomServer.constructGroomServer(GroomServer.class, + conf); + GroomServer.startGroomServer(groom).join(); + } catch (Throwable e) { + LOG.fatal(StringUtils.stringifyException(e)); + return -1; + } + + return 0; + } + + public static void main(String[] args) throws Exception { + int exitCode = ToolRunner.run(new GroomServerRunner(), args); + System.exit(exitCode); + } + +} Index: core/src/main/java/org/apache/hama/bsp/BSPMaster.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPMaster.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPMaster.java (revision 0) @@ -0,0 +1,730 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.net.InetSocketAddress; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.http.HttpServer; +import org.apache.hama.ipc.JobSubmissionProtocol; +import org.apache.hama.ipc.MasterProtocol; +import org.apache.hama.ipc.GroomProtocol; + +/** + * BSPMaster is responsible to control all the groom servers and to manage bsp + * jobs. + */ +public class BSPMaster implements JobSubmissionProtocol, MasterProtocol, + GroomServerManager { + public static final Log LOG = LogFactory.getLog(BSPMaster.class); + + private HamaConfiguration conf; + + /** + * Constants for BSPMaster's status. + */ + public static enum State { + INITIALIZING, RUNNING + } + + private static final int FS_ACCESS_RETRY_PERIOD = 10000; + public static final long GROOMSERVER_EXPIRY_INTERVAL = 10 * 60 * 1000; + static long JOBINIT_SLEEP_INTERVAL = 2000; + + // States + State state = State.INITIALIZING; + + // Attributes + String masterIdentifier; + // private Server interServer; + private Server masterServer; + + // host and port + private String host; + private int port; + + // startTime + private long startTime; + + // HTTP server + private HttpServer infoServer; + private int infoPort; + + // Filesystem + static final String SUBDIR = "bspMaster"; + FileSystem fs = null; + Path systemDir = null; + // system directories are world-wide readable and owner readable + final static FsPermission SYSTEM_DIR_PERMISSION = FsPermission + .createImmutable((short) 0733); // rwx-wx-wx + // system files should have 700 permission + final static FsPermission SYSTEM_FILE_PERMISSION = FsPermission + .createImmutable((short) 0700); // rwx------ + + // Jobs' Meta Data + private Integer nextJobId = Integer.valueOf(1); + private int totalSubmissions = 0; // how many jobs has been submitted by + // clients + private int totalTasks = 0; // currnetly running tasks + private int totalTaskCapacity; // max tasks that groom server can run + + private Map jobs = new TreeMap(); + private TaskScheduler taskScheduler; + + // GroomServers cache + protected ConcurrentMap groomServers = new ConcurrentHashMap(); + + private Instructor instructor; + + private final List jobInProgressListeners = new CopyOnWriteArrayList(); + + private class ReportGroomStatusHandler implements DirectiveHandler { + + public void handle(Directive directive) throws DirectiveException { + // update GroomServerStatus held in the groomServers cache. + GroomServerStatus groomStatus = ((ReportGroomStatusDirective) directive) + .getStatus(); + // groomServers cache contains groom server status reported back + if (groomServers.containsKey(groomStatus)) { + GroomServerStatus tmpStatus = null; + for (GroomServerStatus old : groomServers.keySet()) { + if (old.equals(groomStatus)) { + tmpStatus = groomStatus; + updateGroomServersKey(old, tmpStatus); + break; + } + }// for + if (null != tmpStatus) { + List tlist = tmpStatus.getTaskReports(); + for (TaskStatus ts : tlist) { + JobInProgress jip = whichJob(ts.getJobId()); + TaskInProgress tip = jip.findTaskInProgress(((TaskAttemptID) ts + .getTaskId()).getTaskID()); + + if (ts.getRunState() == TaskStatus.State.SUCCEEDED) { + jip.completedTask(tip, ts); + } else if (ts.getRunState() == TaskStatus.State.RUNNING) { + // do nothing + } else if (ts.getRunState() == TaskStatus.State.FAILED) { + jip.status.setRunState(JobStatus.FAILED); + jip.failedTask(tip, ts); + } + if (jip.getStatus().getRunState() == JobStatus.SUCCEEDED) { + for (JobInProgressListener listener : jobInProgressListeners) { + try { + listener.jobRemoved(jip); + } catch (IOException ioe) { + LOG.error("Fail to alter scheduler a job is moved.", ioe); + } + } + } else if (jip.getStatus().getRunState() == JobStatus.RUNNING) { + jip.getStatus().setprogress(ts.getSuperstepCount()); + } else if (jip.getStatus().getRunState() == JobStatus.KILLED) { + GroomProtocol worker = findGroomServer(tmpStatus); + Directive d1 = new DispatchTasksDirective( + currentGroomServerPeers(), + new GroomServerAction[] { new KillTaskAction(ts.getTaskId()) }); + try { + worker.dispatch(d1); + } catch (IOException ioe) { + throw new DirectiveException("Error when dispatching kill task" + + " action.", ioe); + } + } + } + } else { + throw new RuntimeException("BSPMaster contains GroomServerSatus, " + + "but fail to retrieve it."); + } + } else { + throw new RuntimeException("GroomServer not found." + + groomStatus.getGroomName()); + } + } + } + + private class Instructor extends Thread { + private final BlockingQueue buffer = new LinkedBlockingQueue(); + private final ConcurrentMap, DirectiveHandler> handlers = new ConcurrentHashMap, DirectiveHandler>(); + + public void bind(Class instruction, + DirectiveHandler handler) { + handlers.putIfAbsent(instruction, handler); + } + + public void put(Directive directive) { + try { + buffer.put(directive); + } catch (InterruptedException ie) { + LOG.error("Fail to put directive into queue.", ie); + } + } + + public void run() { + while (true) { + try { + Directive directive = this.buffer.take(); + if (directive instanceof ReportGroomStatusDirective) { + ((DirectiveHandler) handlers.get(ReportGroomStatusDirective.class)) + .handle(directive); + } else { + throw new RuntimeException("Directive is not supported." + + directive); + } + } catch (InterruptedException ie) { + LOG.error("Unable to retrieve directive from the queue.", ie); + Thread.currentThread().interrupt(); + } catch (Exception e) { + LOG.error("Fail to execute directive command.", e); + } + } + } + } + + /** + * Start the BSPMaster process, listen on the indicated hostname/port + */ + public BSPMaster(HamaConfiguration conf) throws IOException, + InterruptedException { + this(conf, generateNewIdentifier()); + } + + BSPMaster(HamaConfiguration conf, String identifier) throws IOException, + InterruptedException { + this.conf = conf; + this.masterIdentifier = identifier; + + // Create the scheduler and init scheduler services + Class schedulerClass = conf.getClass( + "bsp.master.taskscheduler", SimpleTaskScheduler.class, + TaskScheduler.class); + this.taskScheduler = (TaskScheduler) ReflectionUtils.newInstance( + schedulerClass, conf); + + host = getAddress(conf).getHostName(); + port = getAddress(conf).getPort(); + LOG.info("RPC BSPMaster: host " + host + " port " + port); + + startTime = System.currentTimeMillis(); + this.masterServer = RPC.getServer(this, host, port, conf); + + infoPort = conf.getInt("bsp.http.infoserver.port", 40013); + + infoServer = new HttpServer("bspmaster", host, infoPort, true, conf); + infoServer.setAttribute("bsp.master", this); + + // starting webserver + infoServer.start(); + + while (!Thread.currentThread().isInterrupted()) { + try { + if (fs == null) { + fs = FileSystem.get(conf); + } + // clean up the system dir, which will only work if hdfs is out of + // safe mode + if (systemDir == null) { + systemDir = new Path(getSystemDir()); + } + + LOG.info("Cleaning up the system directory"); + LOG.info(systemDir); + fs.delete(systemDir, true); + if (FileSystem.mkdirs(fs, systemDir, new FsPermission( + SYSTEM_DIR_PERMISSION))) { + break; + } + LOG.error("Mkdirs failed to create " + systemDir); + LOG.info(SUBDIR); + + } catch (AccessControlException ace) { + LOG.warn("Failed to operate on bsp.system.dir (" + systemDir + + ") because of permissions."); + LOG.warn("Manually delete the bsp.system.dir (" + systemDir + + ") and then start the BSPMaster."); + LOG.warn("Bailing out ... "); + throw ace; + } catch (IOException ie) { + LOG.info("problem cleaning system directory: " + systemDir, ie); + } + Thread.sleep(FS_ACCESS_RETRY_PERIOD); + } + + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException(); + } + + deleteLocalFiles(SUBDIR); + } + + /** + * A GroomServer registers with its status to BSPMaster when startup, which + * will update GroomServers cache. + * + * @param status to be updated in cache. + * @return true if registering successfully; false if fail. + */ + @Override + public boolean register(GroomServerStatus status) throws IOException { + if (null == status) { + LOG.error("No groom server status."); + throw new NullPointerException("No groom server status."); + } + Throwable e = null; + try { + GroomProtocol wc = (GroomProtocol) RPC.waitForProxy( + GroomProtocol.class, GroomProtocol.versionID, + resolveWorkerAddress(status.getRpcServer()), this.conf); + if (null == wc) { + LOG.warn("Fail to create Worker client at host " + status.getPeerName()); + return false; + } + // TODO: need to check if peer name has changed + groomServers.putIfAbsent(status, wc); + } catch (UnsupportedOperationException u) { + e = u; + } catch (ClassCastException c) { + e = c; + } catch (NullPointerException n) { + e = n; + } catch (IllegalArgumentException i) { + e = i; + } catch (Exception ex) { + e = ex; + } + + if (null != e) { + LOG.error("Fail to register GroomServer " + status.getGroomName(), e); + return false; + } + + return true; + } + + private static InetSocketAddress resolveWorkerAddress(String data) { + return new InetSocketAddress(data.split(":")[0], Integer.parseInt(data + .split(":")[1])); + } + + private void updateGroomServersKey(GroomServerStatus old, + GroomServerStatus newKey) { + synchronized (groomServers) { + GroomProtocol worker = groomServers.remove(old); + groomServers.put(newKey, worker); + } + } + + @Override + public boolean report(Directive directive) throws IOException { + instructor.put(directive); + return true; + } + + private JobInProgress whichJob(BSPJobID id) { + for (JobInProgress job : taskScheduler + .getJobs(SimpleTaskScheduler.PROCESSING_QUEUE)) { + if (job.getJobID().equals(id)) { + return job; + } + } + return null; + } + + // ///////////////////////////////////////////////////////////// + // BSPMaster methods + // ///////////////////////////////////////////////////////////// + + // Get the job directory in system directory + Path getSystemDirectoryForJob(BSPJobID id) { + return new Path(getSystemDir(), id.toString()); + } + + String[] getLocalDirs() throws IOException { + return conf.getStrings("bsp.local.dir"); + } + + void deleteLocalFiles() throws IOException { + String[] localDirs = getLocalDirs(); + for (int i = 0; i < localDirs.length; i++) { + FileSystem.getLocal(conf).delete(new Path(localDirs[i]), true); + } + } + + void deleteLocalFiles(String subdir) throws IOException { + try { + String[] localDirs = getLocalDirs(); + for (int i = 0; i < localDirs.length; i++) { + FileSystem.getLocal(conf).delete(new Path(localDirs[i], subdir), true); + } + } catch (NullPointerException e) { + LOG.info(e); + } + } + + /** + * Constructs a local file name. Files are distributed among configured local + * directories. + */ + Path getLocalPath(String pathString) throws IOException { + return conf.getLocalPath("bsp.local.dir", pathString); + } + + public static BSPMaster startMaster(HamaConfiguration conf) + throws IOException, InterruptedException { + return startMaster(conf, generateNewIdentifier()); + } + + public static BSPMaster startMaster(HamaConfiguration conf, String identifier) + throws IOException, InterruptedException { + + BSPMaster result = new BSPMaster(conf, identifier); + result.taskScheduler.setGroomServerManager(result); + result.taskScheduler.start(); + + return result; + } + + public static InetSocketAddress getAddress(Configuration conf) { + String hamaMasterStr = conf.get("bsp.master.address", "localhost"); + int defaultPort = conf.getInt("bsp.master.port", 40000); + + return NetUtils.createSocketAddr(hamaMasterStr, defaultPort); + } + + /** + * BSPMaster identifier + * + * @return String BSPMaster identification number + */ + private static String generateNewIdentifier() { + return new SimpleDateFormat("yyyyMMddHHmm").format(new Date()); + } + + public void offerService() throws InterruptedException, IOException { + + this.masterServer.start(); + + synchronized (this) { + state = State.RUNNING; + } + + instructor = new Instructor(); + instructor.bind(ReportGroomStatusDirective.class, + new ReportGroomStatusHandler()); + instructor.start(); + + LOG.info("Starting RUNNING"); + + this.masterServer.join(); + + LOG.info("Stopped RPC Master server."); + } + + // ////////////////////////////////////////////////// + // InterServerProtocol + // ////////////////////////////////////////////////// + @Override + public long getProtocolVersion(String protocol, long clientVersion) + throws IOException { + if (protocol.equals(MasterProtocol.class.getName())) { + return MasterProtocol.versionID; + } else if (protocol.equals(JobSubmissionProtocol.class.getName())) { + return JobSubmissionProtocol.versionID; + } else { + throw new IOException("Unknown protocol to BSPMaster: " + protocol); + } + } + + // ////////////////////////////////////////////////// + // JobSubmissionProtocol + // ////////////////////////////////////////////////// + /** + * This method returns new job id. The returned job id increases sequentially. + */ + @Override + public BSPJobID getNewJobId() throws IOException { + int id; + synchronized (nextJobId) { + id = nextJobId; + nextJobId = Integer.valueOf(id + 1); + } + return new BSPJobID(this.masterIdentifier, id); + } + + @Override + public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException { + if (jobs.containsKey(jobID)) { + // job already running, don't start twice + LOG.info("The job (" + jobID + ") was already submitted"); + return jobs.get(jobID).getStatus(); + } + + JobInProgress job = new JobInProgress(jobID, new Path(jobFile), this, + this.conf); + return addJob(jobID, job); + } + + // ////////////////////////////////////////////////// + // GroomServerManager functions + // ////////////////////////////////////////////////// + + @Override + public ClusterStatus getClusterStatus(boolean detailed) { + Map groomPeersMap = null; + + // give the caller a snapshot of the cluster status + int numGroomServers = groomServers.size(); + if (detailed) { + groomPeersMap = new HashMap(); + for (Map.Entry entry : groomServers + .entrySet()) { + GroomServerStatus s = entry.getKey(); + groomPeersMap.put(s.getGroomName(), s.getPeerName()); + } + } + + // TODO currently we only have one task slot per groom server + this.totalTaskCapacity = numGroomServers; + + if (detailed) { + return new ClusterStatus(groomPeersMap, totalTasks, totalTaskCapacity, + state); + } else { + return new ClusterStatus(numGroomServers, totalTasks, totalTaskCapacity, + state); + } + } + + @Override + public GroomProtocol findGroomServer(GroomServerStatus status) { + return groomServers.get(status); + } + + @Override + public Collection findGroomServers() { + return groomServers.values(); + } + + @Override + public Collection groomServerStatusKeySet() { + return groomServers.keySet(); + } + + @Override + public void addJobInProgressListener(JobInProgressListener listener) { + jobInProgressListeners.add(listener); + } + + @Override + public void removeJobInProgressListener(JobInProgressListener listener) { + jobInProgressListeners.remove(listener); + } + + @Override + public Map currentGroomServerPeers() { + Map tmp = new HashMap(); + for (GroomServerStatus status : groomServers.keySet()) { + tmp.put(status.getGroomName(), status.getPeerName()); + } + return tmp; + } + + public String getBSPMasterName() { + return host + ":" + port; + } + + public long getStartTime() { + return startTime; + } + + public String getBSPMasterIdentifier() { + return masterIdentifier; + } + + public int getHttpPort() { + return infoPort; + } + + /** + * Adds a job to the bsp master. Make sure that the checks are inplace before + * adding a job. This is the core job submission logic + * + * @param jobId The id for the job submitted which needs to be added + */ + private synchronized JobStatus addJob(BSPJobID jobId, JobInProgress job) { + totalSubmissions++; + synchronized (jobs) { + jobs.put(job.getProfile().getJobID(), job); + for (JobInProgressListener listener : jobInProgressListeners) { + try { + listener.jobAdded(job); + } catch (IOException ioe) { + LOG.error("Fail to alter Scheduler a job is added.", ioe); + } + } + } + return job.getStatus(); + } + + @Override + public JobStatus[] jobsToComplete() throws IOException { + return getJobStatus(jobs.values(), true); + } + + @Override + public JobStatus[] getAllJobs() throws IOException { + LOG.debug("returns all jobs: " + jobs.size()); + return getJobStatus(jobs.values(), false); + } + + private synchronized JobStatus[] getJobStatus(Collection jips, + boolean toComplete) { + if (jips == null) { + return new JobStatus[] {}; + } + List jobStatusList = new ArrayList(); + for (JobInProgress jip : jips) { + JobStatus status = jip.getStatus(); + + status.setStartTime(jip.getStartTime()); + // Sets the user name + status.setUsername(jip.getProfile().getUser()); + status.setName(jip.getJobName()); + + if (toComplete) { + if (status.getRunState() == JobStatus.RUNNING + || status.getRunState() == JobStatus.PREP) { + jobStatusList.add(status); + } + } else { + jobStatusList.add(status); + } + } + + return jobStatusList.toArray(new JobStatus[jobStatusList.size()]); + } + + @Override + public synchronized String getFilesystemName() throws IOException { + if (fs == null) { + throw new IllegalStateException("FileSystem object not available yet"); + } + return fs.getUri().toString(); + } + + /** + * Return system directory to which BSP store control files. + */ + @Override + public String getSystemDir() { + Path sysDir = new Path(conf.get("bsp.system.dir", "/tmp/hadoop/bsp/system")); + return fs.makeQualified(sysDir).toString(); + } + + @Override + public JobProfile getJobProfile(BSPJobID jobid) throws IOException { + synchronized (this) { + JobInProgress job = jobs.get(jobid); + if (job != null) { + return job.getProfile(); + } + } + return null; + } + + @Override + public JobStatus getJobStatus(BSPJobID jobid) throws IOException { + synchronized (this) { + JobInProgress job = jobs.get(jobid); + if (job != null) { + return job.getStatus(); + } + } + return null; + } + + @Override + public void killJob(BSPJobID jobid) throws IOException { + JobInProgress job = jobs.get(jobid); + + if (null == job) { + LOG.info("killJob(): JobId " + jobid.toString() + " is not a valid job"); + return; + } + + killJob(job); + } + + private synchronized void killJob(JobInProgress job) { + LOG.info("Killing job " + job.getJobID()); + job.kill(); + } + + @Override + public boolean killTask(TaskAttemptID taskId, boolean shouldFail) + throws IOException { + return false; + } + + public static BSPMaster constructMaster( + Class masterClass, final Configuration conf) { + try { + Constructor c = masterClass + .getConstructor(Configuration.class); + return c.newInstance(conf); + } catch (Exception e) { + throw new RuntimeException("Failed construction of " + "Master: " + + masterClass.toString() + + ((e.getCause() != null) ? e.getCause().getMessage() : ""), e); + } + } + + public void shutdown() { + this.masterServer.stop(); + } + + public BSPMaster.State currentState() { + return this.state; + } +} Index: core/src/main/java/org/apache/hama/bsp/TaskScheduler.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskScheduler.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskScheduler.java (revision 0) @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; + +/** + * Used by a {@link BSPMaster} to schedule {@link Task}s on {@link GroomServer} + * s. + */ +abstract class TaskScheduler implements Configurable { + + protected Configuration conf; + protected GroomServerManager groomServerManager; + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration conf) { + this.conf = conf; + } + + public synchronized void setGroomServerManager( + GroomServerManager groomServerManager) { + this.groomServerManager = groomServerManager; + } + + /** + * Lifecycle method to allow the scheduler to start any work in separate + * threads. + * + * @throws IOException + */ + public void start() throws IOException { + // do nothing + } + + /** + * Lifecycle method to allow the scheduler to stop any work it is doing. + * + * @throws IOException + */ + public void terminate() throws IOException { + // do nothing + } + + // public abstract void addJob(JobInProgress job); + + /** + * Returns a collection of jobs in an order which is specific to the + * particular scheduler. + * + * @param Queue name. + * @return JobInProgress corresponded to the specified queue. + */ + public abstract Collection getJobs(String queue); +} Index: core/src/main/java/org/apache/hama/bsp/Messagable.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/Messagable.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/Messagable.java (revision 0) @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +/** + * A interface for BSP message class. + */ +public interface Messagable { + +} Index: core/src/main/java/org/apache/hama/bsp/TaskLog.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskLog.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskLog.java (revision 0) @@ -0,0 +1,343 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hama.HamaConfiguration; + +/** + * A simple logger to handle the task-specific user logs. + */ +public class TaskLog { + private static final Log LOG = LogFactory.getLog(TaskLog.class.getName()); + + private static final File LOG_DIR = new File( + System.getProperty("hama.log.dir"), "userlogs").getAbsoluteFile(); + + static { + if (!LOG_DIR.exists()) { + LOG_DIR.mkdirs(); + } + } + + public static File getTaskLogFile(TaskAttemptID taskid, LogName filter) { + return new File(new File(LOG_DIR, taskid.toString()), filter.toString()); + } + + /** + * The filter for userlogs. + */ + public static enum LogName { + /** Log on the stdout of the task. */ + STDOUT("stdout"), + + /** Log on the stderr of the task. */ + STDERR("stderr"), + + /** Log on the map-reduce system logs of the task. */ + SYSLOG("syslog"), + + /** The java profiler information. */ + PROFILE("profile.out"), + + /** Log the debug script's stdout */ + DEBUGOUT("debugout"); + + private String prefix; + + private LogName(String prefix) { + this.prefix = prefix; + } + + @Override + public String toString() { + return prefix; + } + } + + private static class TaskLogsPurgeFilter implements FileFilter { + long purgeTimeStamp; + + TaskLogsPurgeFilter(long purgeTimeStamp) { + this.purgeTimeStamp = purgeTimeStamp; + } + + public boolean accept(File file) { + LOG.debug("PurgeFilter - file: " + file + ", mtime: " + + file.lastModified() + ", purge: " + purgeTimeStamp); + return file.lastModified() < purgeTimeStamp; + } + } + + /** + * Purge old user logs. + * + * @throws IOException + */ + public static synchronized void cleanup(int logsRetainHours) + throws IOException { + // Purge logs of tasks on this tasktracker if their + // mtime has exceeded "mapred.task.log.retain" hours + long purgeTimeStamp = System.currentTimeMillis() + - (logsRetainHours * 60L * 60 * 1000); + File[] oldTaskLogs = LOG_DIR.listFiles(new TaskLogsPurgeFilter( + purgeTimeStamp)); + if (oldTaskLogs != null) { + for (int i = 0; i < oldTaskLogs.length; ++i) { + FileUtil.fullyDelete(oldTaskLogs[i]); + } + } + } + + static class Reader extends InputStream { + private long bytesRemaining; + private FileInputStream file; + + /** + * Read a log file from start to end positions. The offsets may be negative, + * in which case they are relative to the end of the file. For example, + * Reader(taskid, kind, 0, -1) is the entire file and Reader(taskid, kind, + * -4197, -1) is the last 4196 bytes. + * + * @param taskid the id of the task to read the log file for + * @param kind the kind of log to read + * @param start the offset to read from (negative is relative to tail) + * @param end the offset to read upto (negative is relative to tail) + * @throws IOException + */ + public Reader(TaskAttemptID taskid, LogName kind, long start, long end) + throws IOException { + // find the right log file + File filename = getTaskLogFile(taskid, kind); + // calculate the start and stop + long size = filename.length(); + if (start < 0) { + start += size + 1; + } + if (end < 0) { + end += size + 1; + } + start = Math.max(0, Math.min(start, size)); + end = Math.max(0, Math.min(end, size)); + bytesRemaining = end - start; + file = new FileInputStream(filename); + // skip upto start + long pos = 0; + while (pos < start) { + long result = file.skip(start - pos); + if (result < 0) { + bytesRemaining = 0; + break; + } + pos += result; + } + } + + @Override + public int read() throws IOException { + int result = -1; + if (bytesRemaining > 0) { + bytesRemaining -= 1; + result = file.read(); + } + return result; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + length = (int) Math.min(length, bytesRemaining); + int bytes = file.read(buffer, offset, length); + if (bytes > 0) { + bytesRemaining -= bytes; + } + return bytes; + } + + @Override + public int available() throws IOException { + return (int) Math.min(bytesRemaining, file.available()); + } + + @Override + public void close() throws IOException { + file.close(); + } + } + + private static final String bashCommand = "bash"; + private static final String tailCommand = "tail"; + + /** + * Get the desired maximum length of task's logs. + * + * @param conf the job to look in + * @return the number of bytes to cap the log files at + */ + public static long getTaskLogLength(HamaConfiguration conf) { + return conf.getLong("mapred.userlog.limit.kb", 100) * 1024; + } + + /** + * Wrap a command in a shell to capture stdout and stderr to files. If the + * tailLength is 0, the entire output will be saved. + * + * @param cmd The command and the arguments that should be run + * @param stdoutFilename The filename that stdout should be saved to + * @param stderrFilename The filename that stderr should be saved to + * @param tailLength The length of the tail to be saved. + * @return the modified command that should be run + */ + public static List captureOutAndError(List cmd, + File stdoutFilename, File stderrFilename, long tailLength) + throws IOException { + return captureOutAndError(null, cmd, stdoutFilename, stderrFilename, + tailLength); + } + + /** + * Wrap a command in a shell to capture stdout and stderr to files. Setup + * commands such as setting memory limit can be passed which will be executed + * before exec. If the tailLength is 0, the entire output will be saved. + * + * @param setup The setup commands for the execed process. + * @param cmd The command and the arguments that should be run + * @param stdoutFilename The filename that stdout should be saved to + * @param stderrFilename The filename that stderr should be saved to + * @param tailLength The length of the tail to be saved. + * @return the modified command that should be run + */ + public static List captureOutAndError(List setup, + List cmd, File stdoutFilename, File stderrFilename, + long tailLength) throws IOException { + String stdout = FileUtil.makeShellPath(stdoutFilename); + String stderr = FileUtil.makeShellPath(stderrFilename); + List result = new ArrayList(3); + result.add(bashCommand); + result.add("-c"); + StringBuffer mergedCmd = new StringBuffer(); + if (setup != null && setup.size() > 0) { + mergedCmd.append(addCommand(setup, false)); + mergedCmd.append(";"); + } + if (tailLength > 0) { + mergedCmd.append("("); + } else { + mergedCmd.append("exec "); + } + mergedCmd.append(addCommand(cmd, true)); + mergedCmd.append(" < /dev/null "); + if (tailLength > 0) { + mergedCmd.append(" | "); + mergedCmd.append(tailCommand); + mergedCmd.append(" -c "); + mergedCmd.append(tailLength); + mergedCmd.append(" >> "); + mergedCmd.append(stdout); + mergedCmd.append(" ; exit $PIPESTATUS ) 2>&1 | "); + mergedCmd.append(tailCommand); + mergedCmd.append(" -c "); + mergedCmd.append(tailLength); + mergedCmd.append(" >> "); + mergedCmd.append(stderr); + mergedCmd.append(" ; exit $PIPESTATUS"); + } else { + mergedCmd.append(" 1>> "); + mergedCmd.append(stdout); + mergedCmd.append(" 2>> "); + mergedCmd.append(stderr); + } + result.add(mergedCmd.toString()); + return result; + } + + /** + * Add quotes to each of the command strings and return as a single string + * + * @param cmd The command to be quoted + * @param isExecutable makes shell path if the first argument is executable + * @return returns The quoted string. + * @throws IOException + */ + public static String addCommand(List cmd, boolean isExecutable) + throws IOException { + StringBuffer command = new StringBuffer(); + for (String s : cmd) { + command.append('\''); + if (isExecutable) { + // the executable name needs to be expressed as a shell path for the + // shell to find it. + command.append(FileUtil.makeShellPath(new File(s))); + isExecutable = false; + } else { + command.append(s); + } + command.append('\''); + command.append(" "); + } + return command.toString(); + } + + /** + * Wrap a command in a shell to capture debug script's stdout and stderr to + * debugout. + * + * @param cmd The command and the arguments that should be run + * @param debugoutFilename The filename that stdout and stderr should be saved + * to. + * @return the modified command that should be run + * @throws IOException + */ + public static List captureDebugOut(List cmd, + File debugoutFilename) throws IOException { + String debugout = FileUtil.makeShellPath(debugoutFilename); + List result = new ArrayList(3); + result.add(bashCommand); + result.add("-c"); + StringBuffer mergedCmd = new StringBuffer(); + mergedCmd.append("exec "); + boolean isExecutable = true; + for (String s : cmd) { + if (isExecutable) { + // the executable name needs to be expressed as a shell path for the + // shell to find it. + mergedCmd.append(FileUtil.makeShellPath(new File(s))); + isExecutable = false; + } else { + mergedCmd.append(s); + } + mergedCmd.append(" "); + } + mergedCmd.append(" < /dev/null "); + mergedCmd.append(" >"); + mergedCmd.append(debugout); + mergedCmd.append(" 2>&1 "); + result.add(mergedCmd.toString()); + return result; + } + +} // TaskLog Index: core/src/main/java/org/apache/hama/bsp/BSPInterface.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPInterface.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPInterface.java (revision 0) @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configurable; +import org.apache.zookeeper.KeeperException; + +/** + * Interface BSP defines the basic operations needed to implement the BSP + * algorithm. + */ +public interface BSPInterface extends Configurable { + + /** + * A user defined function for programming in the BSP style. + * + * Applications can use the {@link org.apache.hama.bsp.BSPPeer} to handle the + * communication and synchronization between processors. + * + * @param bspPeer + * @throws IOException + * @throws KeeperException + * @throws InterruptedException + */ + public void bsp(BSPPeer bspPeer) throws IOException, KeeperException, + InterruptedException; +} Index: core/src/main/java/org/apache/hama/bsp/TaskAttemptContext.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskAttemptContext.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskAttemptContext.java (revision 0) @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.Progressable; + +/** + * The context for task attempts. + */ +public class TaskAttemptContext extends BSPJobContext implements Progressable { + private final TaskAttemptID taskId; + private String status = ""; + + public TaskAttemptContext(Configuration conf, TaskAttemptID taskId) { + super(conf, taskId.getJobID()); + this.taskId = taskId; + } + + /** + * Get the unique name for this task attempt. + */ + public TaskAttemptID getTaskAttemptID() { + return taskId; + } + + /** + * Set the current status of the task to the given string. + */ + public void setStatus(String msg) throws IOException { + status = msg; + } + + /** + * Get the last set status message. + * + * @return the current status message + */ + public String getStatus() { + return status; + } + + /** + * Report progress. The subtypes actually do work in this method. + */ + public void progress() { + } +} Index: core/src/main/java/org/apache/hama/bsp/TaskStatus.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskStatus.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskStatus.java (revision 0) @@ -0,0 +1,279 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; + +/** + * Describes the current status of a task. This is not intended to be a + * comprehensive piece of data. + */ +class TaskStatus implements Writable, Cloneable { + static final Log LOG = LogFactory.getLog(TaskStatus.class); + + // enumeration for reporting current phase of a task. + public static enum Phase { + STARTING, COMPUTE, BARRIER_SYNC, CLEANUP + } + + // what state is the task in? + public static enum State { + RUNNING, SUCCEEDED, FAILED, UNASSIGNED, KILLED, COMMIT_PENDING, FAILED_UNCLEAN, KILLED_UNCLEAN + } + + private BSPJobID jobId; + private TaskAttemptID taskId; + private float progress; + private volatile State runState; + private String stateString; + private String groomServer; + private long superstepCount; + + private long startTime; + private long finishTime; + + private volatile Phase phase = Phase.STARTING; + + /** + * + */ + public TaskStatus() { + jobId = new BSPJobID(); + taskId = new TaskAttemptID(); + this.superstepCount = 0; + } + + public TaskStatus(BSPJobID jobId, TaskAttemptID taskId, float progress, + State runState, String stateString, String groomServer, Phase phase) { + this.jobId = jobId; + this.taskId = taskId; + this.progress = progress; + this.runState = runState; + this.stateString = stateString; + this.groomServer = groomServer; + this.phase = phase; + this.superstepCount = 0; + } + + // ////////////////////////////////////////////////// + // Accessors and Modifiers + // ////////////////////////////////////////////////// + + public BSPJobID getJobId() { + return jobId; + } + + public TaskAttemptID getTaskId() { + return taskId; + } + + public float getProgress() { + return progress; + } + + public void setProgress(float progress) { + this.progress = progress; + } + + public State getRunState() { + return runState; + } + + public void setRunState(State state) { + this.runState = state; + } + + public String getStateString() { + return stateString; + } + + public void setStateString(String stateString) { + this.stateString = stateString; + } + + public String getGroomServer() { + return groomServer; + } + + public void setGroomServer(String groomServer) { + this.groomServer = groomServer; + } + + public long getFinishTime() { + return finishTime; + } + + void setFinishTime(long finishTime) { + this.finishTime = finishTime; + } + + /** + * Get start time of the task. + * + * @return 0 is start time is not set, else returns start time. + */ + public long getStartTime() { + return startTime; + } + + /** + * Set startTime of the task. + * + * @param startTime start time + */ + void setStartTime(long startTime) { + this.startTime = startTime; + } + + /** + * Get current phase of this task. + * + * @return . + */ + public Phase getPhase() { + return this.phase; + } + + /** + * Set current phase of this task. + * + * @param phase phase of this task + */ + void setPhase(Phase phase) { + this.phase = phase; + } + + /** + * Update the status of the task. + * + * This update is done by ping thread before sending the status. + * + * @param progress + * @param state + * @param counters + */ + synchronized void statusUpdate(float progress, String state) { + setProgress(progress); + setStateString(state); + } + + /** + * Update the status of the task. + * + * @param status updated status + */ + synchronized void statusUpdate(TaskStatus status) { + this.progress = status.getProgress(); + this.runState = status.getRunState(); + this.stateString = status.getStateString(); + + if (status.getStartTime() != 0) { + this.startTime = status.getStartTime(); + } + if (status.getFinishTime() != 0) { + this.finishTime = status.getFinishTime(); + } + + this.phase = status.getPhase(); + } + + /** + * Update specific fields of task status + * + * This update is done in BSPMaster when a cleanup attempt of task reports its + * status. Then update only specific fields, not all. + * + * @param runState + * @param progress + * @param state + * @param phase + * @param finishTime + */ + synchronized void statusUpdate(State runState, float progress, String state, + Phase phase, long finishTime) { + setRunState(runState); + setProgress(progress); + setStateString(state); + setPhase(phase); + if (finishTime != 0) { + this.finishTime = finishTime; + } + } + + /** + * @return The number of BSP super steps executed by the task. + */ + public long getSuperstepCount() { + return superstepCount; + } + + /** + * Increments the number of BSP super steps executed by the task. + */ + public void incrementSuperstepCount() { + superstepCount += 1; + } + + @Override + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException cnse) { + // Shouldn't happen since we do implement Clonable + throw new InternalError(cnse.toString()); + } + } + + // //////////////////////////////////////////// + // Writable + // //////////////////////////////////////////// + + @Override + public void readFields(DataInput in) throws IOException { + this.jobId.readFields(in); + this.taskId.readFields(in); + this.progress = in.readFloat(); + this.runState = WritableUtils.readEnum(in, State.class); + this.stateString = Text.readString(in); + this.phase = WritableUtils.readEnum(in, Phase.class); + this.startTime = in.readLong(); + this.finishTime = in.readLong(); + this.superstepCount = in.readLong(); + } + + @Override + public void write(DataOutput out) throws IOException { + jobId.write(out); + taskId.write(out); + out.writeFloat(progress); + WritableUtils.writeEnum(out, runState); + Text.writeString(out, stateString); + WritableUtils.writeEnum(out, phase); + out.writeLong(startTime); + out.writeLong(finishTime); + out.writeLong(superstepCount); + } +} Index: core/src/main/java/org/apache/hama/bsp/BSPTask.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPTask.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPTask.java (revision 0) @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hama.ipc.BSPPeerProtocol; +import org.apache.zookeeper.KeeperException; + +/** + * Base class for tasks. + */ +public class BSPTask extends Task { + + public static final Log LOG = LogFactory.getLog(BSPTask.class); + + private BSPJob conf; + + public BSPTask() { + } + + public BSPTask(BSPJobID jobId, String jobFile, TaskAttemptID taskid, int partition) { + this.jobId = jobId; + this.jobFile = jobFile; + this.taskId = taskid; + this.partition = partition; + } + + @Override + public BSPTaskRunner createRunner(GroomServer groom) { + return new BSPTaskRunner(this, groom, this.conf); + } + + @Override + public void run(BSPJob job, BSPPeer bspPeer, BSPPeerProtocol umbilical) + throws IOException { + + BSP bsp = (BSP) ReflectionUtils.newInstance(job.getConf().getClass( + "bsp.work.class", BSP.class), job.getConf()); + + try { + bsp.bsp(bspPeer); + } catch (IOException e) { + LOG.error("Exception during BSP execution!", e); + } catch (KeeperException e) { + LOG.error("Exception during BSP execution!", e); + } catch (InterruptedException e) { + LOG.error("Exception during BSP execution!", e); + } + + done(umbilical); + } + + public BSPJob getConf() { + return conf; + } + + public void setConf(BSPJob conf) { + this.conf = conf; + } + +} Index: core/src/main/java/org/apache/hama/bsp/GroomServer.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/GroomServer.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/GroomServer.java (revision 0) @@ -0,0 +1,920 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.reflect.Constructor; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FSError; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.net.DNS; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.DiskChecker; +import org.apache.hadoop.util.DiskChecker.DiskErrorException; +import org.apache.hadoop.util.RunJar; +import org.apache.hadoop.util.StringUtils; +import org.apache.hama.Constants; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.ipc.BSPPeerProtocol; +import org.apache.hama.ipc.MasterProtocol; +import org.apache.hama.ipc.GroomProtocol; +import org.apache.log4j.LogManager; + +/** + * A Groom Server (shortly referred to as groom) is a process that performs bsp + * tasks assigned by BSPMaster. Each groom contacts the BSPMaster, and it takes + * assigned tasks and reports its status by means of periodical piggybacks with + * BSPMaster. Each groom is designed to run with HDFS or other distributed + * storages. Basically, a groom server and a data node should be run on one + * physical node. + */ +public class GroomServer implements Runnable, GroomProtocol, BSPPeerProtocol { + public static final Log LOG = LogFactory.getLog(GroomServer.class); + static final String SUBDIR = "groomServer"; + + private volatile static int REPORT_INTERVAL = 1 * 1000; + + Configuration conf; + + // Constants + static enum State { + NORMAL, COMPUTE, SYNC, BARRIER, STALE, INTERRUPTED, DENIED + }; + + // Running States and its related things + volatile boolean initialized = false; + volatile boolean running = true; + volatile boolean shuttingDown = false; + boolean justInited = true; + GroomServerStatus status = null; + + // Attributes + String groomServerName; + String localHostname; + String groomHostName; + InetSocketAddress bspMasterAddr; + private Instructor instructor; + + // Filesystem + // private LocalDirAllocator localDirAllocator; + Path systemDirectory = null; + FileSystem systemFS = null; + + // Job + private int failures; + private int maxCurrentTasks = 1; + Map tasks = new HashMap(); + /** Map from taskId -> TaskInProgress. */ + Map runningTasks = null; + Map finishedTasks = null; + Map runningJobs = null; + + // new nexus between GroomServer and BSPMaster + // holds/ manage all tasks + // List tasksList = new + // CopyOnWriteArrayList(); + + private String rpcServer; + private Server workerServer; + MasterProtocol masterClient; + + InetSocketAddress taskReportAddress; + Server taskReportServer = null; + + private PeerNames allPeerNames = null; + + // private BlockingQueue tasksToCleanup = new + // LinkedBlockingQueue(); + + private class DispatchTasksHandler implements DirectiveHandler { + + public void handle(Directive directive) throws DirectiveException { + GroomServerAction[] actions = ((DispatchTasksDirective) directive) + .getActions(); + + allPeerNames = new PeerNames(((DispatchTasksDirective) directive) + .getGroomServerPeers().values()); + + if (LOG.isDebugEnabled()) { + LOG.debug("Got Response from BSPMaster with " + + ((actions != null) ? actions.length : 0) + " actions"); + } + + if (actions != null) { + for (GroomServerAction action : actions) { + if (action instanceof LaunchTaskAction) { + startNewTask((LaunchTaskAction) action); + } else { + + // TODO Use the cleanup thread + // tasksToCleanup.put(action); + + KillTaskAction killAction = (KillTaskAction) action; + if (tasks.containsKey(killAction.getTaskID())) { + TaskInProgress tip = tasks.get(killAction.getTaskID()); + tip.taskStatus.setRunState(TaskStatus.State.FAILED); + try { + tip.killAndCleanup(true); + } catch (IOException ioe) { + throw new DirectiveException("Error when killing a " + + "TaskInProgress.", ioe); + } + } + } + } + } + } + } + + private class Instructor extends Thread { + final BlockingQueue buffer = new LinkedBlockingQueue(); + final ConcurrentMap, DirectiveHandler> handlers = new ConcurrentHashMap, DirectiveHandler>(); + + public void bind(Class instruction, + DirectiveHandler handler) { + handlers.putIfAbsent(instruction, handler); + } + + public void put(Directive directive) { + try { + buffer.put(directive); + } catch (InterruptedException ie) { + LOG.error("Unable to put directive into queue.", ie); + Thread.currentThread().interrupt(); + } + } + + public void run() { + while (true) { + try { + Directive directive = buffer.take(); + if (directive instanceof DispatchTasksDirective) { + ((DirectiveHandler) handlers.get(DispatchTasksDirective.class)) + .handle(directive); + } else { + throw new RuntimeException("Directive is not supported." + + directive); + } + } catch (InterruptedException ie) { + LOG.error("Unable to retrieve directive from the queue.", ie); + Thread.currentThread().interrupt(); + } catch (Exception e) { + LOG.error("Fail to execute directive.", e); + } + } + } + } + + public GroomServer(Configuration conf) throws IOException { + LOG.info("groom start"); + this.conf = conf; + + bspMasterAddr = BSPMaster.getAddress(conf); + // FileSystem local = FileSystem.getLocal(conf); + // this.localDirAllocator = new LocalDirAllocator("bsp.local.dir"); + } + + public synchronized void initialize() throws IOException { + if (this.conf.get(Constants.PEER_HOST) != null) { + this.localHostname = conf.get(Constants.PEER_HOST); + } + + if (localHostname == null) { + this.localHostname = DNS.getDefaultHost( + conf.get("bsp.dns.interface", "default"), + conf.get("bsp.dns.nameserver", "default")); + } + // check local disk + checkLocalDirs(conf.getStrings("bsp.local.dir")); + deleteLocalFiles("groomserver"); + + // Clear out state tables + this.tasks.clear(); + this.runningJobs = new TreeMap(); + this.runningTasks = new LinkedHashMap(); + this.finishedTasks = new LinkedHashMap(); + this.conf.set(Constants.PEER_HOST, localHostname); + this.conf.set(Constants.GROOM_RPC_HOST, localHostname); + + int rpcPort = -1; + String rpcAddr = null; + if (false == this.initialized) { + rpcAddr = conf.get(Constants.GROOM_RPC_HOST, + Constants.DEFAULT_GROOM_RPC_HOST); + rpcPort = conf.getInt(Constants.GROOM_RPC_PORT, + Constants.DEFAULT_GROOM_RPC_PORT); + if (-1 == rpcPort || null == rpcAddr) + throw new IllegalArgumentException("Error rpc address " + rpcAddr + + " port" + rpcPort); + this.workerServer = RPC.getServer(this, rpcAddr, rpcPort, conf); + this.workerServer.start(); + this.rpcServer = rpcAddr + ":" + rpcPort; + + LOG.info("Worker rpc server --> " + rpcServer); + } + + @SuppressWarnings("deprecation") + String address = NetUtils.getServerAddress(conf, + "bsp.groom.report.bindAddress", "bsp.groom.report.port", + "bsp.groom.report.address"); + InetSocketAddress socAddr = NetUtils.createSocketAddr(address); + String bindAddress = socAddr.getHostName(); + int tmpPort = socAddr.getPort(); + + // RPC initialization + // TODO numHandlers should be a .. + this.taskReportServer = RPC.getServer(this, bindAddress, tmpPort, 10, + false, this.conf); + + this.taskReportServer.start(); + + // get the assigned address + this.taskReportAddress = taskReportServer.getListenerAddress(); + this.conf.set("bsp.groom.report.address", taskReportAddress.getHostName() + + ":" + taskReportAddress.getPort()); + LOG.info("GroomServer up at: " + this.taskReportAddress); + + this.groomHostName = rpcAddr; + this.groomServerName = "groomd_" + this.rpcServer.replace(':', '_'); + LOG.info("Starting groom: " + this.rpcServer); + + DistributedCache.purgeCache(this.conf); + + // establish the communication link to bsp master + this.masterClient = (MasterProtocol) RPC.waitForProxy(MasterProtocol.class, + MasterProtocol.versionID, bspMasterAddr, conf); + + // enroll in bsp master + if (-1 == rpcPort || null == rpcAddr) + throw new IllegalArgumentException("Error rpc address " + rpcAddr + + " port" + rpcPort); + if (!this.masterClient.register(new GroomServerStatus(groomServerName, + getBspPeerName(), cloneAndResetRunningTaskStatuses(), failures, + maxCurrentTasks, this.rpcServer))) { + LOG.error("There is a problem in establishing communication" + + " link with BSPMaster"); + throw new IOException("There is a problem in establishing" + + " communication link with BSPMaster."); + } + + this.instructor = new Instructor(); + this.instructor.bind(DispatchTasksDirective.class, + new DispatchTasksHandler()); + instructor.start(); + this.running = true; + this.initialized = true; + } + + /** Return the port at which the tasktracker bound to */ + public synchronized InetSocketAddress getTaskTrackerReportAddress() { + return taskReportAddress; + } + + @Override + public void dispatch(Directive directive) throws IOException { + if (!instructor.isAlive()) + throw new IOException(); + + instructor.put(directive); + } + + private static void checkLocalDirs(String[] localDirs) + throws DiskErrorException { + boolean writable = false; + + LOG.info(localDirs); + + if (localDirs != null) { + for (int i = 0; i < localDirs.length; i++) { + try { + LOG.info(localDirs[i]); + DiskChecker.checkDir(new File(localDirs[i])); + writable = true; + } catch (DiskErrorException e) { + LOG.warn("BSP Processor local " + e.getMessage()); + } + } + } + + if (!writable) + throw new DiskErrorException("all local directories are not writable"); + } + + public String[] getLocalDirs() { + return conf.getStrings("bsp.local.dir"); + } + + public void deleteLocalFiles() throws IOException { + String[] localDirs = getLocalDirs(); + for (int i = 0; i < localDirs.length; i++) { + FileSystem.getLocal(this.conf).delete(new Path(localDirs[i]), true); + } + } + + public void deleteLocalFiles(String subdir) throws IOException { + try { + String[] localDirs = getLocalDirs(); + for (int i = 0; i < localDirs.length; i++) { + FileSystem.getLocal(this.conf).delete(new Path(localDirs[i], subdir), + true); + } + } catch (NullPointerException e) { + LOG.info(e); + } + } + + public void cleanupStorage() throws IOException { + deleteLocalFiles(); + } + + private void startCleanupThreads() throws IOException { + + } + + public State offerService() throws Exception { + while (running && !shuttingDown) { + try { + + // Reports to a BSPMaster + for (Map.Entry e : runningTasks + .entrySet()) { + Thread.sleep(REPORT_INTERVAL); + TaskInProgress tip = e.getValue(); + TaskStatus taskStatus = tip.getStatus(); + + if (taskStatus.getRunState() == TaskStatus.State.RUNNING) { + taskStatus.setProgress(taskStatus.getSuperstepCount()); + + if (!tip.runner.isAlive()) { + if (taskStatus.getRunState() != TaskStatus.State.FAILED) { + taskStatus.setRunState(TaskStatus.State.SUCCEEDED); + } + taskStatus.setPhase(TaskStatus.Phase.CLEANUP); + } + } + + doReport(taskStatus); + } + + Thread.sleep(REPORT_INTERVAL); + } catch (InterruptedException ie) { + } + + try { + if (justInited) { + String dir = masterClient.getSystemDir(); + if (dir == null) { + LOG.error("Fail to get system directory."); + throw new IOException("Fail to get system directory."); + } + systemDirectory = new Path(dir); + systemFS = systemDirectory.getFileSystem(conf); + } + justInited = false; + } catch (DiskErrorException de) { + String msg = "Exiting groom server for disk error:\n" + + StringUtils.stringifyException(de); + LOG.error(msg); + + return State.STALE; + } catch (RemoteException re) { + return State.DENIED; + } catch (Exception except) { + String msg = "Caught exception: " + + StringUtils.stringifyException(except); + LOG.error(msg); + } + } + return State.NORMAL; + } + + private void startNewTask(LaunchTaskAction action) { + Task t = action.getTask(); + BSPJob jobConf = null; + try { + jobConf = new BSPJob(t.getJobID(), t.getJobFile()); + } catch (IOException e1) { + LOG.error(e1); + } + + TaskInProgress tip = new TaskInProgress(t, jobConf, this.groomServerName); + + synchronized (this) { + tasks.put(t.getTaskID(), tip); + runningTasks.put(t.getTaskID(), tip); + } + + try { + localizeJob(tip); + } catch (Throwable e) { + String msg = ("Error initializing " + tip.getTask().getTaskID() + ":\n" + StringUtils + .stringifyException(e)); + LOG.warn(msg); + } + } + + /** + * Update and report refresh status back to BSPMaster. + */ + public void doReport(TaskStatus taskStatus) { + GroomServerStatus groomStatus = new GroomServerStatus(groomServerName, + getBspPeerName(), updateTaskStatus(taskStatus), failures, + maxCurrentTasks, rpcServer); + try { + boolean ret = masterClient.report(new ReportGroomStatusDirective( + groomStatus)); + if (!ret) { + LOG.warn("Fail to renew BSPMaster's GroomServerStatus. " + + " groom name: " + groomStatus.getGroomName() + " peer name:" + + groomStatus.getPeerName() + " rpc server:" + rpcServer); + } + } catch (IOException ioe) { + LOG.error("Fail to communicate with BSPMaster for reporting.", ioe); + } + } + + public List updateTaskStatus(TaskStatus taskStatus) { + List tlist = new ArrayList(); + synchronized (runningTasks) { + + if (taskStatus.getRunState() == TaskStatus.State.SUCCEEDED + || taskStatus.getRunState() == TaskStatus.State.FAILED) { + synchronized (finishedTasks) { + TaskInProgress tip = runningTasks.remove(taskStatus.getTaskId()); + tlist.add((TaskStatus) taskStatus.clone()); + finishedTasks.put(taskStatus.getTaskId(), tip); + } + } else if (taskStatus.getRunState() == TaskStatus.State.RUNNING) { + tlist.add((TaskStatus) taskStatus.clone()); + } + + } + return tlist; + } + + private void localizeJob(TaskInProgress tip) throws IOException { + Task task = tip.getTask(); + conf.addResource(task.getJobFile()); + BSPJob defaultJobConf = new BSPJob((HamaConfiguration) conf); + Path localJobFile = defaultJobConf.getLocalPath(SUBDIR + "/" + + task.getTaskID() + "/" + "job.xml"); + + RunningJob rjob = addTaskToJob(task.getJobID(), localJobFile, tip); + BSPJob jobConf = null; + + synchronized (rjob) { + if (!rjob.localized) { + Path localJarFile = defaultJobConf.getLocalPath(SUBDIR + "/" + + task.getTaskID() + "/" + "job.jar"); + systemFS.copyToLocalFile(new Path(task.getJobFile()), localJobFile); + + HamaConfiguration conf = new HamaConfiguration(); + conf.addResource(localJobFile); + jobConf = new BSPJob(conf, task.getJobID().toString()); + + Path jarFile = new Path(jobConf.getJar()); + jobConf.setJar(localJarFile.toString()); + + if (jarFile != null) { + systemFS.copyToLocalFile(jarFile, localJarFile); + + // also unjar the job.jar files in workdir + File workDir = new File( + new File(localJobFile.toString()).getParent(), "work"); + if (!workDir.mkdirs()) { + if (!workDir.isDirectory()) { + throw new IOException("Mkdirs failed to create " + + workDir.toString()); + } + } + RunJar.unJar(new File(localJarFile.toString()), workDir); + } + rjob.localized = true; + } + } + + launchTaskForJob(tip, jobConf); + } + + private void launchTaskForJob(TaskInProgress tip, BSPJob jobConf) { + try { + tip.setJobConf(jobConf); + tip.launchTask(); + } catch (Throwable ie) { + tip.taskStatus.setRunState(TaskStatus.State.FAILED); + String error = StringUtils.stringifyException(ie); + LOG.info(error); + } + } + + private RunningJob addTaskToJob(BSPJobID jobId, Path localJobFile, + TaskInProgress tip) { + synchronized (runningJobs) { + RunningJob rJob = null; + if (!runningJobs.containsKey(jobId)) { + rJob = new RunningJob(jobId, localJobFile); + rJob.localized = false; + rJob.tasks = new HashSet(); + rJob.jobFile = localJobFile; + runningJobs.put(jobId, rJob); + } else { + rJob = runningJobs.get(jobId); + } + rJob.tasks.add(tip); + return rJob; + } + } + + /** + * The datastructure for initializing a job + */ + static class RunningJob { + private BSPJobID jobid; + private Path jobFile; + // keep this for later use + Set tasks; + boolean localized; + boolean keepJobFiles; + + RunningJob(BSPJobID jobid, Path jobFile) { + this.jobid = jobid; + localized = false; + tasks = new HashSet(); + this.jobFile = jobFile; + keepJobFiles = false; + } + + Path getJobFile() { + return jobFile; + } + + BSPJobID getJobId() { + return jobid; + } + } + + private synchronized List cloneAndResetRunningTaskStatuses() { + List result = new ArrayList(runningTasks.size()); + for (TaskInProgress tip : runningTasks.values()) { + TaskStatus status = tip.getStatus(); + result.add((TaskStatus) status.clone()); + } + return result; + } + + public void run() { + try { + initialize(); + startCleanupThreads(); + boolean denied = false; + while (running && !shuttingDown && !denied) { + + boolean staleState = false; + try { + while (running && !staleState && !shuttingDown && !denied) { + try { + State osState = offerService(); + if (osState == State.STALE) { + staleState = true; + } else if (osState == State.DENIED) { + denied = true; + } + } catch (Exception e) { + if (!shuttingDown) { + LOG.info("Lost connection to BSP Master [" + bspMasterAddr + + "]. Retrying...", e); + try { + Thread.sleep(5000); + } catch (InterruptedException ie) { + } + } + } + } + } finally { + // close(); + } + if (shuttingDown) { + return; + } + LOG.warn("Reinitializing local state"); + initialize(); + } + } catch (IOException ioe) { + LOG.error("Got fatal exception while reinitializing GroomServer: " + + StringUtils.stringifyException(ioe)); + return; + } + } + + public synchronized void shutdown() throws IOException { + shuttingDown = true; + close(); + } + + public synchronized void close() throws IOException { + this.running = false; + this.initialized = false; + cleanupStorage(); + this.workerServer.stop(); + RPC.stopProxy(masterClient); + + if (taskReportServer != null) { + taskReportServer.stop(); + taskReportServer = null; + } + } + + public static Thread startGroomServer(final GroomServer hrs) { + return startGroomServer(hrs, "regionserver" + hrs.groomServerName); + } + + public static Thread startGroomServer(final GroomServer hrs, final String name) { + Thread t = new Thread(hrs); + t.setName(name); + t.start(); + return t; + } + + // ///////////////////////////////////////////////////// + // TaskInProgress maintains all the info for a Task that + // lives at this GroomServer. It maintains the Task object, + // its TaskStatus, and the BSPTaskRunner. + // ///////////////////////////////////////////////////// + class TaskInProgress { + Task task; + BSPJob jobConf; + BSPJob localJobConf; + BSPTaskRunner runner; + volatile boolean done = false; + volatile boolean wasKilled = false; + private TaskStatus taskStatus; + + public TaskInProgress(Task task, BSPJob jobConf, String groomServer) { + this.task = task; + this.jobConf = jobConf; + this.localJobConf = null; + this.taskStatus = new TaskStatus(task.getJobID(), task.getTaskID(), 0, + TaskStatus.State.UNASSIGNED, "running", groomServer, + TaskStatus.Phase.STARTING); + } + + private void localizeTask(Task task) throws IOException { + Path localJobFile = this.jobConf.getLocalPath(SUBDIR + "/" + + task.getTaskID() + "/job.xml"); + Path localJarFile = this.jobConf.getLocalPath(SUBDIR + "/" + + task.getTaskID() + "/job.jar"); + + String jobFile = task.getJobFile(); + systemFS.copyToLocalFile(new Path(jobFile), localJobFile); + task.setJobFile(localJobFile.toString()); + + localJobConf = new BSPJob(task.getJobID(), localJobFile.toString()); + localJobConf.set("bsp.task.id", task.getTaskID().toString()); + String jarFile = localJobConf.getJar(); + + if (jarFile != null) { + systemFS.copyToLocalFile(new Path(jarFile), localJarFile); + localJobConf.setJar(localJarFile.toString()); + } + + LOG.debug("localizeTask : " + localJobConf.getJar()); + LOG.debug("localizeTask : " + localJobFile.toString()); + + task.setConf(localJobConf); + } + + public synchronized void setJobConf(BSPJob jobConf) { + this.jobConf = jobConf; + } + + public synchronized BSPJob getJobConf() { + return localJobConf; + } + + public void launchTask() throws IOException { + localizeTask(task); + taskStatus.setRunState(TaskStatus.State.RUNNING); + this.runner = task.createRunner(GroomServer.this); + this.runner.start(); + } + + /** + * This task has run on too long, and should be killed. + */ + public synchronized void killAndCleanup(boolean wasFailure) + throws IOException { + runner.kill(); + } + + /** + */ + public Task getTask() { + return task; + } + + /** + */ + public synchronized TaskStatus getStatus() { + return taskStatus; + } + + /** + */ + public TaskStatus.State getRunState() { + return taskStatus.getRunState(); + } + + public boolean wasKilled() { + return wasKilled; + } + + @Override + public boolean equals(Object obj) { + return (obj instanceof TaskInProgress) + && task.getTaskID().equals( + ((TaskInProgress) obj).getTask().getTaskID()); + } + + @Override + public int hashCode() { + return task.getTaskID().hashCode(); + } + } + + public boolean isRunning() { + return running; + } + + public static GroomServer constructGroomServer( + Class groomServerClass, final Configuration conf2) { + try { + Constructor c = groomServerClass + .getConstructor(Configuration.class); + return c.newInstance(conf2); + } catch (Exception e) { + throw new RuntimeException("Failed construction of " + "Master: " + + groomServerClass.toString(), e); + } + } + + @Override + public long getProtocolVersion(String protocol, long clientVersion) + throws IOException { + if (protocol.equals(GroomProtocol.class.getName())) { + return GroomProtocol.versionID; + } else if (protocol.equals(BSPPeerProtocol.class.getName())) { + return BSPPeerProtocol.versionID; + } else { + throw new IOException("Unknown protocol to GroomServer: " + protocol); + } + } + + /** + * @return bsp peer information in the form of "address:port". + */ + public String getBspPeerName() { + // TODO Later, peers list should be returned. + return this.groomHostName + ":" + Constants.DEFAULT_PEER_PORT; + } + + /** + * The main() for child processes. + */ + public static class Child { + + public static void main(String[] args) throws Throwable { + LOG.debug("Child starting"); + + HamaConfiguration defaultConf = new HamaConfiguration(); + // report address + String host = args[0]; + int port = Integer.parseInt(args[1]); + InetSocketAddress address = new InetSocketAddress(host, port); + TaskAttemptID taskid = TaskAttemptID.forName(args[2]); + + // ////////////////// + BSPPeerProtocol umbilical = (BSPPeerProtocol) RPC.getProxy( + BSPPeerProtocol.class, BSPPeerProtocol.versionID, address, + defaultConf); + + Task task = umbilical.getTask(taskid); + + defaultConf.addResource(new Path(task.getJobFile())); + BSPJob job = new BSPJob(task.getJobID(), task.getJobFile()); + + defaultConf.set(Constants.PEER_HOST, args[3]); + defaultConf.setInt(Constants.PEER_PORT, Constants.DEFAULT_PEER_PORT); + + BSPPeer bspPeer = new BSPPeer(defaultConf, taskid, umbilical); + bspPeer.setJobConf(job); + bspPeer.setAllPeerNames(umbilical.getAllPeerNames().getAllPeerNames()); + + TaskStatus taskStatus = new TaskStatus(task.getJobID(), task.getTaskID(), + 0, TaskStatus.State.RUNNING, "running", host, + TaskStatus.Phase.STARTING); + + bspPeer.setCurrentTaskStatus(taskStatus); + + try { + // use job-specified working directory + FileSystem.get(job.getConf()).setWorkingDirectory( + job.getWorkingDirectory()); + + task.run(job, bspPeer, umbilical); // run the task + + } catch (FSError e) { + LOG.fatal("FSError from child", e); + umbilical.fsError(taskid, e.getMessage()); + } catch (Throwable throwable) { + LOG.warn("Error running child", throwable); + // Report back any failures, for diagnostic purposes + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + throwable.printStackTrace(new PrintStream(baos)); + } finally { + bspPeer.close(); // close peer. + + RPC.stopProxy(umbilical); + // Shutting down log4j of the child-vm... + // This assumes that on return from Task.run() + // there is no more logging done. + LogManager.shutdown(); + } + } + } + + @Override + public Task getTask(TaskAttemptID taskid) throws IOException { + TaskInProgress tip = tasks.get(taskid); + if (tip != null) { + return tip.getTask(); + } else { + return null; + } + } + + public void incrementSuperstepCount(TaskAttemptID taskid) throws IOException { + TaskInProgress tip = tasks.get(taskid); + tip.getStatus().incrementSuperstepCount(); + } + + @Override + public boolean ping(TaskAttemptID taskid) throws IOException { + // TODO Auto-generated method stub + return false; + } + + @Override + public void done(TaskAttemptID taskid, boolean shouldBePromoted) + throws IOException { + // TODO Auto-generated method stub + } + + @Override + public void fsError(TaskAttemptID taskId, String message) throws IOException { + // TODO Auto-generated method stub + } + + @Override + public PeerNames getAllPeerNames() { + return allPeerNames; + } +} \ No newline at end of file Index: core/src/main/java/org/apache/hama/bsp/Directive.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/Directive.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/Directive.java (revision 0) @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Writable; + +/** + * A generic directive from the {@link org.apache.hama.bsp.BSPMaster} to the + * {@link org.apache.hama.bsp.GroomServer} to take some 'action'. + */ +public class Directive implements Writable{ + + protected long timestamp; + protected Directive.Type type; + + public static enum Type { + Request(1), Response(2); + int t; + + Type(int t) { + this.t = t; + } + + public int value() { + return this.t; + } + }; + + public Directive(){} + + public Directive(Directive.Type type) { + this.timestamp = System.currentTimeMillis(); + this.type = type; + } + + public long getTimestamp() { + return this.timestamp; + } + + public Directive.Type getType() { + return this.type; + } + + /** + * Command for BSPMaster or GroomServer to execute. + public abstract void execute() throws Exception; + */ + + public void write(DataOutput out) throws IOException { + out.writeLong(this.timestamp); + out.writeInt(this.type.value()); + } + + public void readFields(DataInput in) throws IOException { + this.timestamp = in.readLong(); + int t = in.readInt(); + if (Directive.Type.Request.value() == t) { + this.type = Directive.Type.Request; + }else{ + this.type = Directive.Type.Response; + } + } +} Index: core/src/main/java/org/apache/hama/bsp/JobStatus.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/JobStatus.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/JobStatus.java (revision 0) @@ -0,0 +1,279 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableFactories; +import org.apache.hadoop.io.WritableFactory; + +/** + * Describes the current status of a job. + */ +public class JobStatus implements Writable, Cloneable { + public static final Log LOG = LogFactory.getLog(JobStatus.class); + + static { + WritableFactories.setFactory(JobStatus.class, new WritableFactory() { + public Writable newInstance() { + return new JobStatus(); + } + }); + } + + public static enum State { + RUNNING(1), SUCCEEDED(2), FAILED(3), PREP(4), KILLED(5); + int s; + + State(int s) { + this.s = s; + } + + public int value() { + return this.s; + } + + @Override + public String toString() { + String name = null; + switch (this) { + case RUNNING: + name = "RUNNING"; + break; + case SUCCEEDED: + name = "SUCCEEDED"; + break; + case FAILED: + name = "FAILED"; + break; + case PREP: + name = "SETUP"; + break; + case KILLED: + name = "KILLED"; + break; + } + + return name; + } + + } + + public static final int RUNNING = 1; + public static final int SUCCEEDED = 2; + public static final int FAILED = 3; + public static final int PREP = 4; + public static final int KILLED = 5; + + private BSPJobID jobid; + private long progress; + private long cleanupProgress; + private long setupProgress; + private volatile State state;// runState in enum + private int runState; + private long startTime; + private String schedulingInfo = "NA"; + private String user; + private long superstepCount; + private String name; + + private long finishTime; + + public JobStatus() { + } + + public JobStatus(BSPJobID jobid, String user, long progress, int runState) { + this(jobid, user, progress, 0, runState); + } + + public JobStatus(BSPJobID jobid, String user, long progress, + long cleanupProgress, int runState) { + this(jobid, user, 0, progress, cleanupProgress, runState); + } + + public JobStatus(BSPJobID jobid, String user, long setupProgress, + long progress, long cleanupProgress, int runState) { + this(jobid, user, 0, progress, cleanupProgress, runState, 0); + } + + public JobStatus(BSPJobID jobid, String user, long setupProgress, + long progress, long cleanupProgress, int runState, long superstepCount) { + this.jobid = jobid; + this.setupProgress = setupProgress; + this.progress = progress; + this.cleanupProgress = cleanupProgress; + this.runState = runState; + this.state = State.values()[runState - 1]; + this.superstepCount = superstepCount; + this.user = user; + } + + public BSPJobID getJobID() { + return jobid; + } + + public synchronized long progress() { + return progress; + } + + synchronized void setprogress(long p) { + this.progress = p; + } + + public synchronized long cleanupProgress() { + return cleanupProgress; + } + + synchronized void setCleanupProgress(int p) { + this.cleanupProgress = p; + } + + public synchronized long setupProgress() { + return setupProgress; + } + + synchronized void setSetupProgress(long p) { + this.setupProgress = p; + } + + public JobStatus.State getState() { + return this.state; + } + + public void setState(JobStatus.State state) { + this.state = state; + } + + public synchronized int getRunState() { + return runState; + } + + public synchronized void setRunState(int state) { + this.runState = state; + } + + public synchronized long getSuperstepCount() { + return superstepCount; + } + + public synchronized void setSuperstepCount(long superstepCount) { + this.superstepCount = superstepCount; + } + + public synchronized void setStartTime(long startTime) { + this.startTime = startTime; + } + + public synchronized long getStartTime() { + return startTime; + } + + public synchronized void setFinishTime(long finishTime) { + this.finishTime = finishTime; + } + + /** + * Get the finish time of the job. + */ + public synchronized long getFinishTime() { + return finishTime; + } + + /** + * @param user The username of the job + */ + public synchronized void setUsername(String user) { + this.user = user; + } + + /** + * @return the username of the job + */ + public synchronized String getUsername() { + return user; + } + + @Override + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException cnse) { + throw new InternalError(cnse.toString()); + } + } + + public synchronized String getSchedulingInfo() { + return schedulingInfo; + } + + public synchronized void setSchedulingInfo(String schedulingInfo) { + this.schedulingInfo = schedulingInfo; + } + + public synchronized boolean isJobComplete() { + return (runState == JobStatus.SUCCEEDED || runState == JobStatus.FAILED || runState == JobStatus.KILLED); + } + + public synchronized void write(DataOutput out) throws IOException { + jobid.write(out); + out.writeLong(setupProgress); + out.writeLong(progress); + out.writeLong(cleanupProgress); + out.writeInt(runState); + out.writeLong(startTime); + out.writeLong(finishTime); + Text.writeString(out, user); + Text.writeString(out, schedulingInfo); + out.writeLong(superstepCount); + } + + public synchronized void readFields(DataInput in) throws IOException { + this.jobid = new BSPJobID(); + jobid.readFields(in); + this.setupProgress = in.readLong(); + this.progress = in.readLong(); + this.cleanupProgress = in.readLong(); + this.runState = in.readInt(); + this.startTime = in.readLong(); + this.finishTime = in.readLong(); + this.user = Text.readString(in); + this.schedulingInfo = Text.readString(in); + this.superstepCount = in.readLong(); + } + + /** + * @param name the name to set + */ + public synchronized void setName(String name) { + this.name = name; + } + + /** + * @return the name + */ + public synchronized String getName() { + return name; + } + +} Index: core/src/main/java/org/apache/hama/bsp/TaskInProgress.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskInProgress.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskInProgress.java (revision 0) @@ -0,0 +1,288 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobStatus; + +/** + *TaskInProgress maintains all the info needed for a Task in the lifetime of + * its owning Job. + */ +class TaskInProgress { + public static final Log LOG = LogFactory.getLog(TaskInProgress.class); + + private Configuration conf; + + // Constants + static final int MAX_TASK_EXECS = 1; + int maxTaskAttempts = 4; + private boolean failed = false; + private static final int NUM_ATTEMPTS_PER_RESTART = 1000; + + // Job Meta + private String jobFile = null; + private int partition; + private BSPMaster bspMaster; + private TaskID id; + private JobInProgress job; + private int completes = 0; + + // Status + // private double progress = 0; + // private String state = ""; + private long startTime = 0; + + // The 'next' usable taskid of this tip + int nextTaskId = 0; + + // The taskid that took this TIP to SUCCESS + private TaskAttemptID successfulTaskId; + + // The first taskid of this tip + private TaskAttemptID firstTaskId; + + // Map from task Id -> GroomServer Id, contains tasks that are + // currently runnings + private TreeMap activeTasks = new TreeMap(); + // All attempt Ids of this TIP + // private TreeSet tasks = new TreeSet(); + /** + * Map from taskId -> TaskStatus + */ + private TreeMap taskStatuses = new TreeMap(); + + private BSPJobID jobId; + + /** + * Constructor for new nexus between BSPMaster and GroomServer. + * + * @param jobId is identification of JobInProgress. + * @param jobFile the path of job file + * @param partition which partition this TaskInProgress owns. + */ + public TaskInProgress(BSPJobID jobId, String jobFile, int partition) { + this.jobId = jobId; + this.jobFile = jobFile; + this.partition = partition; + + this.id = new TaskID(jobId, partition); + } + + public TaskInProgress(BSPJobID jobId, String jobFile, BSPMaster master, + Configuration conf, JobInProgress job, int partition) { + this.jobId = jobId; + this.jobFile = jobFile; + this.setBspMaster(master); + this.job = job; + this.setConf(conf); + this.partition = partition; + + this.id = new TaskID(jobId, partition); + } + + /** + * Return a Task that can be sent to a GroomServer for execution. + */ + public Task getTaskToRun(GroomServerStatus status) throws IOException { + Task t = null; + + TaskAttemptID taskid = null; + if (nextTaskId < (MAX_TASK_EXECS + maxTaskAttempts)) { + int attemptId = job.getNumRestarts() * NUM_ATTEMPTS_PER_RESTART + + nextTaskId; + taskid = new TaskAttemptID(id, attemptId); + ++nextTaskId; + } else { + LOG.warn("Exceeded limit of " + (MAX_TASK_EXECS + maxTaskAttempts) + + " attempts for the tip '" + getTIPId() + "'"); + return null; + } + + t = new BSPTask(jobId, jobFile, taskid, partition); + activeTasks.put(taskid, status.getGroomName()); + + return t; + } + + // ////////////////////////////////// + // Accessors + // ////////////////////////////////// + /** + * Return the start time + */ + public long getStartTime() { + return startTime; + } + + /** + * Return the parent job + */ + public JobInProgress getJob() { + return job; + } + + public TaskID getTIPId() { + return id; + } + + public TaskID getTaskId() { + return this.id; + } + + public TreeMap getTasks() { + return activeTasks; + } + + /** + * Is the Task associated with taskid is the first attempt of the tip? + * + * @param taskId + * @return Returns true if the Task is the first attempt of the tip + */ + public boolean isFirstAttempt(TaskAttemptID taskId) { + return firstTaskId == null ? false : firstTaskId.equals(taskId); + } + + /** + * Is this tip currently running any tasks? + * + * @return true if any tasks are running + */ + public boolean isRunning() { + return !activeTasks.isEmpty(); + } + + /** + * Is this tip complete? + * + * @return true if the tip is complete, else false + */ + public synchronized boolean isComplete() { + return (completes > 0); + } + + /** + * Is the given taskid the one that took this tip to completion? + * + * @param taskid taskid of attempt to check for completion + * @return true if taskid is complete, else false + */ + public boolean isComplete(TaskAttemptID taskid) { + return (completes > 0 && taskid.equals(getSuccessfulTaskid())); + } + + private TreeSet tasksReportedClosed = new TreeSet(); + + public boolean shouldCloseForClosedJob(TaskAttemptID taskid) { + TaskStatus ts = (TaskStatus) taskStatuses.get(taskid); + if ((ts != null) && (!tasksReportedClosed.contains(taskid)) + && (job.getStatus().getRunState() != JobStatus.RUNNING)) { + tasksReportedClosed.add(taskid); + return true; + } else { + return false; + } + } + + public void completed(TaskAttemptID taskid) { + LOG.info("Task '" + taskid.getTaskID().toString() + "' has completed."); + + TaskStatus status = (TaskStatus) taskStatuses.get(taskid); + status.setRunState(TaskStatus.State.SUCCEEDED); + activeTasks.remove(taskid); + + // Note the successful taskid + setSuccessfulTaskid(taskid); + + // + // Now that the TIP is complete, the other speculative + // subtasks will be closed when the owning groom server + // reports in and calls shouldClose() on this object. + // + + this.completes++; + } + + public void terminated(TaskAttemptID taskid) { + LOG.info("Task '" + taskid.getTaskID().toString() + "' has failed."); + + TaskStatus status = (TaskStatus) taskStatuses.get(taskid); + status.setRunState(TaskStatus.State.FAILED); + activeTasks.remove(taskid); + } + + private void setSuccessfulTaskid(TaskAttemptID taskid) { + this.successfulTaskId = taskid; + } + + private TaskAttemptID getSuccessfulTaskid() { + return successfulTaskId; + } + + public void updateStatus(TaskStatus status) { + taskStatuses.put(status.getTaskId(), status); + } + + public TaskStatus getTaskStatus(TaskAttemptID taskId) { + return this.taskStatuses.get(taskId); + } + + public void kill() { + this.failed = true; + } + + public boolean isFailed() { + return failed; + } + + /** + * @param conf the conf to set + */ + public void setConf(Configuration conf) { + this.conf = conf; + } + + /** + * @return the conf + */ + public Configuration getConf() { + return conf; + } + + /** + * @param bspMaster the bspMaster to set + */ + public void setBspMaster(BSPMaster bspMaster) { + this.bspMaster = bspMaster; + } + + /** + * @return the bspMaster + */ + public BSPMaster getBspMaster() { + return bspMaster; + } + +} Index: core/src/main/java/org/apache/hama/bsp/DoubleMessage.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/DoubleMessage.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/DoubleMessage.java (revision 0) @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A message that consists of a string tag and a double data. + */ +public class DoubleMessage extends BSPMessage { + + private String tag; + private double data; + + public DoubleMessage() { + super(); + } + + public DoubleMessage(String tag, double data) { + super(); + this.data = data; + this.tag = tag; + } + + @Override + public String getTag() { + return tag; + } + + @Override + public Double getData() { + return data; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeUTF(tag); + out.writeDouble(data); + } + + @Override + public void readFields(DataInput in) throws IOException { + tag = in.readUTF(); + data = in.readDouble(); + } + +} Index: core/src/main/java/org/apache/hama/bsp/QueueManager.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/QueueManager.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/QueueManager.java (revision 0) @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; + +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.conf.Configuration; + +/** + * A BSPJob Queue Manager. + */ +public class QueueManager{ + + private ConcurrentMap> queues = + new ConcurrentHashMap>(); + + public QueueManager(Configuration conf){ } + + /** + * Initialize a job. + * @param job required initialzied. + */ + public void initJob(JobInProgress job){ + try{ + //job.updateStatus(); + job.initTasks(); + }catch(IOException ioe){ + ioe.printStackTrace(); + } + } + + /** + * Add a job to the specified queue. + * @param name of the queue. + * @param job to be added. + */ + public void addJob(String name, JobInProgress job){ + Queue queue = queues.get(name); + if(null != queue) queue.addJob(job); + } + + /** + * Remove a job from the head of a designated queue. + * @param name from which a job is removed. + * @param job to be removed from the queue. + */ + public void removeJob(String name, JobInProgress job){ + Queue queue = queues.get(name); + if(null != queue) queue.removeJob(job); + } + + /** + * Move a job from a queue to another. + * @param from a queue a job is to be removed. + * @param to a queue a job is to be added. + */ + public void moveJob(String from, String to, JobInProgress job){ + synchronized(queues){ + removeJob(from, job); + addJob(to, job); + } + } + + /** + * Create a FCFS queue with the name provided. + * @param name of the queue. + */ + public void createFCFSQueue(String name){ + queues.putIfAbsent(name, new FCFSQueue(name)); + } + + /** + * Find Queue according to the name specified. + * @param name of the queue. + * @return queue of JobInProgress + */ + public Queue findQueue(String name){ + return queues.get(name); + } + +} Index: core/src/main/java/org/apache/hama/bsp/ReportGroomStatusDirective.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/ReportGroomStatusDirective.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/ReportGroomStatusDirective.java (revision 0) @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Writable; + +/** + * Reports status of GroomServer. + */ +public class ReportGroomStatusDirective extends Directive implements Writable { + public static final Log LOG = LogFactory.getLog(ReportGroomStatusDirective.class); + + private GroomServerStatus status; + + public ReportGroomStatusDirective(){ super(); } + + public ReportGroomStatusDirective(GroomServerStatus status) { + super(Directive.Type.Response); + this.status = status; + } + + public GroomServerStatus getStatus() { + return this.status; + } + + public void write(DataOutput out) throws IOException { + super.write(out); + this.status.write(out); + } + + public void readFields(DataInput in) throws IOException { + super.readFields(in); + this.status = new GroomServerStatus(); + this.status.readFields(in); + } +} Index: core/src/main/java/org/apache/hama/bsp/TaskLogAppender.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskLogAppender.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskLogAppender.java (revision 0) @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.util.LinkedList; +import java.util.Queue; + +import org.apache.log4j.FileAppender; +import org.apache.log4j.spi.LoggingEvent; + +/** + * A simple log4j-appender for the task child's BSP system logs. + */ +public class TaskLogAppender extends FileAppender { + private String taskId; // taskId should be managed as String rather than + // TaskID object + // so that log4j can configure it from the configuration(log4j.properties). + private int maxEvents; + private Queue tail = null; + + @Override + public void activateOptions() { + synchronized (this) { + if (maxEvents > 0) { + tail = new LinkedList(); + } + setFile(TaskLog.getTaskLogFile(TaskAttemptID.forName(taskId), + TaskLog.LogName.SYSLOG).toString()); + setAppend(true); + super.activateOptions(); + } + } + + @Override + public void append(LoggingEvent event) { + synchronized (this) { + if (tail == null) { + super.append(event); + } else { + if (tail.size() >= maxEvents) { + tail.remove(); + } + tail.add(event); + } + } + } + + @Override + public synchronized void close() { + if (tail != null) { + for (LoggingEvent event : tail) { + super.append(event); + } + } + super.close(); + } + + /** + * Getter/Setter methods for log4j. + */ + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + private static final int EVENT_SIZE = 100; + + public long getTotalLogFileSize() { + return maxEvents * EVENT_SIZE; + } + + public void setTotalLogFileSize(long logSize) { + maxEvents = (int) logSize / EVENT_SIZE; + } + +} Index: core/src/main/java/org/apache/hama/bsp/BSPJobContext.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPJobContext.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPJobContext.java (revision 0) @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hama.HamaConfiguration; + +/** + * A read-only view of the bsp job that is provided to the tasks while they are + * running. + */ +public class BSPJobContext { + // Put all of the attribute names in here so that BSPJob and JobContext are + // consistent. + protected static final String WORK_CLASS_ATTR = "bsp.work.class"; + protected static final String INPUT_FORMAT_CLASS_ATTR = "bsp.inputformat.class"; + protected static final String OUTPUT_FORMAT_CLASS_ATTR = "bsp.outputformat.class"; + protected static final String WORKING_DIR = "bsp.working.dir"; + + protected final Configuration conf; + private final BSPJobID jobId; + + public BSPJobContext(Configuration conf, BSPJobID jobId) { + this.conf = conf; + this.jobId = jobId; + } + + public BSPJobContext(Path config, BSPJobID jobId) throws IOException { + this.conf = new HamaConfiguration(); + this.jobId = jobId; + this.conf.addResource(config); + } + + public BSPJobID getJobID() { + return jobId; + } + + public Path getWorkingDirectory() throws IOException { + String name = conf.get(WORKING_DIR); + + if (name != null) { + return new Path(name); + } else { + try { + Path dir = FileSystem.get(conf).getWorkingDirectory(); + conf.set(WORKING_DIR, dir.toString()); + return dir; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public String getJobName() { + return conf.get("bsp.job.name", ""); + } + + public String getJar() { + return conf.get("bsp.jar"); + } + + /** + * Constructs a local file name. Files are distributed among configured + * local directories. + */ + public Path getLocalPath(String pathString) throws IOException { + return conf.getLocalPath("bsp.local.dir", pathString); + } + + public String getUser() { + return conf.get("user.name"); + } + + public void writeXml(OutputStream out) throws IOException { + conf.writeXml(out); + } + + public Configuration getConf() { + return this.conf; + } + + public String get(String name) { + return conf.get(name); + } + + public int getInt(String name, int defaultValue) { + return conf.getInt(name, defaultValue); + } +} Index: core/src/main/java/org/apache/hama/bsp/BooleanMessage.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BooleanMessage.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BooleanMessage.java (revision 0) @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A message that consists of a string tag and a boolean value. + */ +public class BooleanMessage extends BSPMessage { + + String tag; + boolean data; + + public BooleanMessage() { + super(); + } + + public BooleanMessage(String tag, boolean data) { + super(); + this.tag = tag; + this.data = data; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeUTF(tag); + out.writeBoolean(data); + } + + @Override + public void readFields(DataInput in) throws IOException { + tag = in.readUTF(); + data = in.readBoolean(); + } + + @Override + public String getTag() { + return tag; + } + + @Override + public Boolean getData() { + return data; + } +} \ No newline at end of file Index: core/src/main/java/org/apache/hama/bsp/ClusterStatus.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/ClusterStatus.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/ClusterStatus.java (revision 0) @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; + +/** + * Status information on the current state of the BSP cluster. + * + *

ClusterStatus provides clients with information such as: + *

    + *
  1. + * Size of the cluster. + *
  2. + *
  3. + * Name of the grooms. + *
  4. + *
  5. + * Task capacity of the cluster. + *
  6. + *
  7. + * The number of currently running bsp tasks. + *
  8. + *
  9. + * State of the BSPMaster. + *
  10. + *

+ * + *

Clients can query for the latest ClusterStatus, via + * {@link BSPJobClient#getClusterStatus(boolean)}.

+ * + * @see BSPMaster + */ +public class ClusterStatus implements Writable { + + private int numActiveGrooms; + private Map activeGrooms = new HashMap(); + private int tasks; + private int maxTasks; + private BSPMaster.State state; + + /** + * + */ + public ClusterStatus() {} + + public ClusterStatus(int grooms, int tasks, int maxTasks, BSPMaster.State state) { + this.numActiveGrooms = grooms; + this.tasks = tasks; + this.maxTasks = maxTasks; + this.state = state; + } + + public ClusterStatus(Map activeGrooms, int tasks, int maxTasks, + BSPMaster.State state) { + this(activeGrooms.size(), tasks, maxTasks, state); + this.activeGrooms = activeGrooms; + } + + /** + * Get the number of groom servers in the cluster. + * + * @return the number of groom servers in the cluster. + */ + public int getGroomServers() { + return numActiveGrooms; + } + + /** + * Get the names of groom servers, and their peers, in the cluster. + * + * @return the active groom servers in the cluster. + */ + public Map getActiveGroomNames() { + return activeGrooms; + } + + /** + * Get the number of currently running tasks in the cluster. + * + * @return the number of currently running tasks in the cluster. + */ + public int getTasks() { + return tasks; + } + + /** + * Get the maximum capacity for running tasks in the cluster. + * + * @return the maximum capacity for running tasks in the cluster. + */ + public int getMaxTasks() { + return maxTasks; + } + + /** + * Get the current state of the BSPMaster, + * as {@link BSPMaster.State} + * + * @return the current state of the BSPMaster. + */ + public BSPMaster.State getBSPMasterState() { + return state; + } + + ////////////////////////////////////////////// + // Writable + ////////////////////////////////////////////// + @Override + public void write(DataOutput out) throws IOException { + if (activeGrooms.isEmpty()) { + out.writeInt(numActiveGrooms); + out.writeBoolean(false); + } else { + out.writeInt(activeGrooms.size()); + out.writeBoolean(true); + + String[] groomNames = activeGrooms.keySet().toArray(new String[0]); + List peerNames = new ArrayList(); + + for (String groomName : groomNames) { + peerNames.add(activeGrooms.get(groomName)); + } + + WritableUtils.writeCompressedStringArray(out, groomNames); + WritableUtils.writeCompressedStringArray(out, peerNames.toArray(new String[0])); + } + out.writeInt(tasks); + out.writeInt(maxTasks); + WritableUtils.writeEnum(out, state); + } + + @Override + public void readFields(DataInput in) throws IOException { + numActiveGrooms = in.readInt(); + boolean groomListFollows = in.readBoolean(); + + if (groomListFollows) { + String[] groomNames = WritableUtils.readCompressedStringArray(in); + String[] peerNames = WritableUtils.readCompressedStringArray(in); + activeGrooms = new HashMap(groomNames.length); + + for (int i = 0; i < groomNames.length; i++) { + activeGrooms.put(groomNames[i], peerNames[i]); + } + } + + tasks = in.readInt(); + maxTasks = in.readInt(); + state = WritableUtils.readEnum(in, BSPMaster.State.class); + } +} Index: core/src/main/java/org/apache/hama/bsp/FCFSQueue.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/FCFSQueue.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/FCFSQueue.java (revision 0) @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.util.Collection; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +class FCFSQueue implements Queue { + + public static final Log LOG = LogFactory.getLog(FCFSQueue.class); + private final String name; + private BlockingQueue queue = new LinkedBlockingQueue(); + + public FCFSQueue(String name) { + this.name = name; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public void addJob(JobInProgress job) { + try { + queue.put(job); + } catch (InterruptedException ie) { + LOG.error("Fail to add a job to the " + this.name + " queue.", ie); + } + } + + @Override + public void removeJob(JobInProgress job) { + queue.remove(job); + } + + @Override + public JobInProgress removeJob() { + try { + return queue.take(); + } catch (InterruptedException ie) { + LOG.error("Fail to remove a job from the " + this.name + " queue.", ie); + } + return null; + } + + @Override + public Collection jobs() { + return queue; + } + +} Index: core/src/main/java/org/apache/hama/bsp/JobInProgress.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/JobInProgress.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/JobInProgress.java (revision 0) @@ -0,0 +1,360 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; + +/** + * JobInProgress maintains all the info for keeping a Job on the straight and + * narrow. It keeps its JobProfile and its latest JobStatus, plus a set of + * tables for doing bookkeeping of its Tasks.ss + */ +class JobInProgress { + + /** + * Used when the a kill is issued to a job which is initializing. + */ + static class KillInterruptedException extends InterruptedException { + private static final long serialVersionUID = 1L; + + public KillInterruptedException(String msg) { + super(msg); + } + } + + static final Log LOG = LogFactory.getLog(JobInProgress.class); + boolean tasksInited = false; + boolean jobInited = false; + + Configuration conf; + JobProfile profile; + JobStatus status; + Path jobFile = null; + Path localJobFile = null; + Path localJarFile = null; + private LocalFileSystem localFs; + // Indicates how many times the job got restarted + private int restartCount; + + long startTime; + long launchTime; + long finishTime; + + private String jobName; + + // private LocalFileSystem localFs; + private BSPJobID jobId; + final BSPMaster master; + TaskInProgress tasks[] = new TaskInProgress[0]; + private long superstepCounter; + + int numBSPTasks = 0; + int clusterSize; + + public JobInProgress(BSPJobID jobId, Path jobFile, BSPMaster master, + Configuration conf) throws IOException { + this.conf = conf; + this.jobId = jobId; + this.localFs = FileSystem.getLocal(conf); + this.jobFile = jobFile; + this.master = master; + this.status = new JobStatus(jobId, null, 0L, 0L, + JobStatus.State.PREP.value()); + this.startTime = System.currentTimeMillis(); + this.superstepCounter = 0; + this.restartCount = 0; + + this.localJobFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId + + ".xml"); + this.localJarFile = master.getLocalPath(BSPMaster.SUBDIR + "/" + jobId + + ".jar"); + + Path jobDir = master.getSystemDirectoryForJob(jobId); + FileSystem fs = jobDir.getFileSystem(conf); + fs.copyToLocalFile(jobFile, localJobFile); + BSPJob job = new BSPJob(jobId, localJobFile.toString()); + this.numBSPTasks = job.getNumBspTask(); + + this.profile = new JobProfile(job.getUser(), jobId, jobFile.toString(), + job.getJobName()); + + this.setJobName(job.getJobName()); + + status.setUsername(job.getUser()); + status.setStartTime(startTime); + + String jarFile = job.getJar(); + if (jarFile != null) { + fs.copyToLocalFile(new Path(jarFile), localJarFile); + } + + } + + public JobProfile getProfile() { + return profile; + } + + public JobStatus getStatus() { + return status; + } + + public synchronized long getLaunchTime() { + return launchTime; + } + + public long getStartTime() { + return startTime; + } + + public long getFinishTime() { + return finishTime; + } + + /** + * @return the number of desired tasks. + */ + public int desiredBSPTasks() { + return numBSPTasks; + } + + /** + * @return The JobID of this JobInProgress. + */ + public BSPJobID getJobID() { + return jobId; + } + + public synchronized TaskInProgress findTaskInProgress(TaskID id) { + if (areTasksInited()) { + for (TaskInProgress tip : tasks) { + if (tip.getTaskId().equals(id)) { + return tip; + } + } + } + return null; + } + + public synchronized boolean areTasksInited() { + return this.tasksInited; + } + + public String toString() { + return "jobName:" + profile.getJobName() + "\n" + "submit user:" + + profile.getUser() + "\n" + "JobId:" + jobId + "\n" + "JobFile:" + + jobFile + "\n"; + } + + // /////////////////////////////////////////////////// + // Create/manage tasks + // /////////////////////////////////////////////////// + + public synchronized void initTasks() throws IOException { + if (tasksInited) { + return; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("numBSPTasks: " + numBSPTasks); + } + + // adjust number of map tasks to actual number of splits + this.tasks = new TaskInProgress[numBSPTasks]; + for (int i = 0; i < numBSPTasks; i++) { + tasks[i] = new TaskInProgress(getJobID(), this.jobFile.toString(), + this.master, this.conf, this, i); + } + + // Update job status + this.status = new JobStatus(this.status.getJobID(), this.profile.getUser(), + 0L, 0L, JobStatus.RUNNING); + + tasksInited = true; + LOG.debug("Job is initialized."); + } + + public synchronized Task obtainNewTask(GroomServerStatus status, + int clusterSize) { + this.clusterSize = clusterSize; + + if (this.status.getRunState() != JobStatus.RUNNING) { + LOG.info("Cannot create task split for " + profile.getJobID()); + return null; + } + + Task result = null; + try { + for (int i = 0; i < tasks.length; i++) { + if (!tasks[i].isRunning() && !tasks[i].isComplete()) { + result = tasks[i].getTaskToRun(status); + break; + } + } + + } catch (IOException e) { + LOG.error("Exception while obtaining new task!", e); + } + return result; + } + + public synchronized void completedTask(TaskInProgress tip, TaskStatus status) { + TaskAttemptID taskid = status.getTaskId(); + updateTaskStatus(tip, status); + LOG.info("Taskid '" + taskid + "' has finished successfully."); + tip.completed(taskid); + + // + // If all tasks are complete, then the job is done! + // + + boolean allDone = true; + for (TaskInProgress taskInProgress : tasks) { + if (!taskInProgress.isComplete()) { + allDone = false; + break; + } + } + + if (allDone) { + this.status = new JobStatus(this.status.getJobID(), + this.profile.getUser(), superstepCounter, superstepCounter, + superstepCounter, JobStatus.SUCCEEDED, superstepCounter); + this.finishTime = System.currentTimeMillis(); + this.status.setFinishTime(this.finishTime); + + LOG.debug("Job successfully done."); + + garbageCollect(); + } + } + + public void failedTask(TaskInProgress tip, TaskStatus status) { + TaskAttemptID taskid = status.getTaskId(); + updateTaskStatus(tip, status); + LOG.info("Taskid '" + taskid + "' has failed."); + tip.terminated(taskid); + + // + // If all tasks are complete, then the job is done! + // + + boolean allDone = true; + for (TaskInProgress taskInProgress : tasks) { + if (!taskInProgress.isFailed()) { + allDone = false; + break; + } + } + + if (allDone) { + this.status = new JobStatus(this.status.getJobID(), + this.profile.getUser(), superstepCounter, superstepCounter, + superstepCounter, JobStatus.FAILED, superstepCounter); + this.finishTime = System.currentTimeMillis(); + this.status.setFinishTime(this.finishTime); + + LOG.debug("Job failed."); + + garbageCollect(); + } + } + + public synchronized void updateTaskStatus(TaskInProgress tip, + TaskStatus taskStatus) { + tip.updateStatus(taskStatus); // update tip + + if (superstepCounter < taskStatus.getSuperstepCount()) { + superstepCounter = taskStatus.getSuperstepCount(); + // TODO Later, we have to update JobInProgress status here + + } + } + + public synchronized void kill() { + if (status.getRunState() != JobStatus.KILLED) { + this.status = new JobStatus(status.getJobID(), this.profile.getUser(), + 0L, 0L, 0L, JobStatus.KILLED); + this.finishTime = System.currentTimeMillis(); + this.status.setFinishTime(this.finishTime); + // + // kill all TIPs. + // + for (int i = 0; i < tasks.length; i++) { + tasks[i].kill(); + } + + garbageCollect(); + } + + } + + /** + * The job is dead. We're now GC'ing it, getting rid of the job from all + * tables. Be sure to remove all of this job's tasks from the various tables. + */ + synchronized void garbageCollect() { + try { + // Definitely remove the local-disk copy of the job file + if (localJobFile != null) { + localFs.delete(localJobFile, true); + localJobFile = null; + } + if (localJarFile != null) { + localFs.delete(localJarFile, true); + localJarFile = null; + } + + // JobClient always creates a new directory with job files + // so we remove that directory to cleanup + FileSystem fs = FileSystem.get(conf); + fs.delete(new Path(profile.getJobFile()).getParent(), true); + + } catch (IOException e) { + LOG.info("Error cleaning up " + profile.getJobID() + ": " + e); + } + } + + /** + * Get the number of times the job has restarted + */ + int getNumRestarts() { + return restartCount; + } + + /** + * @param jobName the jobName to set + */ + public void setJobName(String jobName) { + this.jobName = jobName; + } + + /** + * @return the jobName + */ + public String getJobName() { + return jobName; + } + +} Index: core/src/main/java/org/apache/hama/bsp/BSPMessage.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPMessage.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPMessage.java (revision 0) @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import org.apache.hadoop.io.Writable; + +/** + * BSPMessage consists of the tag and the arbitrary amount of data to be + * communicated. + */ +public abstract class BSPMessage implements Messagable, Writable { + + /** + * BSP messages are typically identified with tags. This allows to get the tag + * of data. + * + * @return tag of data of BSP message + */ + public abstract Object getTag(); + + /** + * @return data of BSP message + */ + public abstract Object getData(); + +} Index: core/src/main/java/org/apache/hama/bsp/ID.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/ID.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/ID.java (revision 0) @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.WritableComparable; + +/** + * A general identifier, which internally stores the id as an integer. This is + * the super class of {@link BSPJobID}, {@link TaskID} and {@link TaskAttemptID} + * . + */ +public abstract class ID implements WritableComparable { + protected static final char SEPARATOR = '_'; + protected int id; + + public ID(int id) { + this.id = id; + } + + protected ID() { + } + + public int getId() { + return id; + } + + @Override + public String toString() { + return String.valueOf(id); + } + + @Override + public int hashCode() { + return Integer.valueOf(id).hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null) + return false; + if (o.getClass() == this.getClass()) { + ID that = (ID) o; + return this.id == that.id; + } else + return false; + } + + public int compareTo(ID that) { + return this.id - that.id; + } + + public void readFields(DataInput in) throws IOException { + this.id = in.readInt(); + } + + public void write(DataOutput out) throws IOException { + out.writeInt(id); + } +} Index: core/src/main/java/org/apache/hama/bsp/IntegerDoubleMessage.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/IntegerDoubleMessage.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/IntegerDoubleMessage.java (revision 0) @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A message that consists of a int tag and a double data. + */ +public class IntegerDoubleMessage extends BSPMessage { + + int tag; + double data; + + public IntegerDoubleMessage() { + super(); + } + + public IntegerDoubleMessage(int tag, double data) { + super(); + this.tag = tag; + this.data = data; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(tag); + out.writeDouble(data); + } + + @Override + public void readFields(DataInput in) throws IOException { + tag = in.readInt(); + data = in.readDouble(); + } + + @Override + public Integer getTag() { + return tag; + } + + @Override + public Double getData() { + return data; + } + +} Index: core/src/main/java/org/apache/hama/bsp/BSPPeer.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPPeer.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPPeer.java (revision 0) @@ -0,0 +1,423 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.RPC.Server; +import org.apache.hama.Constants; +import org.apache.hama.ipc.BSPPeerProtocol; +import org.apache.hama.util.Bytes; +import org.apache.hama.zookeeper.QuorumPeer; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; + +/** + * This class represents a BSP peer. + */ +public class BSPPeer implements Watcher, BSPPeerInterface { + + public static final Log LOG = LogFactory.getLog(BSPPeer.class); + + private Configuration conf; + private BSPJob jobConf; + + private Server server = null; + private ZooKeeper zk = null; + private volatile Integer mutex = 0; + + private final String bspRoot; + private final String quorumServers; + + private final Map peers = new ConcurrentHashMap(); + private final Map> outgoingQueues = new ConcurrentHashMap>(); + private ConcurrentLinkedQueue localQueue = new ConcurrentLinkedQueue(); + private ConcurrentLinkedQueue localQueueForNextIteration = new ConcurrentLinkedQueue(); + private final Map peerSocketCache = new ConcurrentHashMap(); + + private SortedSet allPeerNames = new TreeSet(); + private InetSocketAddress peerAddress; + private TaskStatus currentTaskStatus; + + private TaskAttemptID taskid; + private BSPPeerProtocol umbilical; + + /** + * Protected default constructor for LocalBSPRunner. + */ + protected BSPPeer() { + bspRoot = null; + quorumServers = null; + } + + /** + * Constructor + * + * @param umbilical + * @param taskid + */ + public BSPPeer(Configuration conf, TaskAttemptID taskid, + BSPPeerProtocol umbilical) throws IOException { + this.conf = conf; + this.taskid = taskid; + this.umbilical = umbilical; + + String bindAddress = conf.get(Constants.PEER_HOST, + Constants.DEFAULT_PEER_HOST); + int bindPort = conf + .getInt(Constants.PEER_PORT, Constants.DEFAULT_PEER_PORT); + bspRoot = conf.get(Constants.ZOOKEEPER_ROOT, + Constants.DEFAULT_ZOOKEEPER_ROOT); + quorumServers = QuorumPeer.getZKQuorumServersString(conf); + LOG.debug("Quorum " + quorumServers); + + // TODO: may require to dynamic reflect the underlying + // network e.g. ip address, port. + peerAddress = new InetSocketAddress(bindAddress, bindPort); + reinitialize(); + } + + public void reinitialize() { + try { + LOG.debug("reinitialize(): " + getPeerName()); + server = RPC.getServer(this, peerAddress.getHostName(), + peerAddress.getPort(), conf); + server.start(); + LOG.info(" BSPPeer address:" + peerAddress.getHostName() + " port:" + + peerAddress.getPort()); + } catch (IOException e) { + LOG.error("Exception during reinitialization!", e); + } + + try { + zk = new ZooKeeper(quorumServers, conf.getInt( + Constants.ZOOKEEPER_SESSION_TIMEOUT, 1200000), this); + } catch (IOException e) { + LOG.error("Exception during reinitialization!", e); + } + + Stat s = null; + if (zk != null) { + try { + s = zk.exists(Constants.DEFAULT_ZOOKEEPER_ROOT, false); + } catch (Exception e) { + LOG.error(s, e); + } + + if (s == null) { + try { + zk.create(Constants.DEFAULT_ZOOKEEPER_ROOT, new byte[0], + Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } catch (KeeperException e) { + LOG.error(e); + } catch (InterruptedException e) { + LOG.error(e); + } + } + } + + } + + @Override + public BSPMessage getCurrentMessage() throws IOException { + return localQueue.poll(); + } + + /* + * (non-Javadoc) + * @see org.apache.hama.bsp.BSPPeerInterface#send(java.net.InetSocketAddress, + * org.apache.hadoop.io.Writable, org.apache.hadoop.io.Writable) + */ + @Override + public void send(String peerName, BSPMessage msg) throws IOException { + if (peerName.equals(getPeerName())) { + LOG.debug("Local send bytes (" + msg.getData().toString() + ")"); + localQueueForNextIteration.add(msg); + } else { + LOG.debug("Send bytes (" + msg.getData().toString() + ") to " + peerName); + InetSocketAddress targetPeerAddress = null; + // Get socket for target peer. + if (peerSocketCache.containsKey(peerName)) { + targetPeerAddress = peerSocketCache.get(peerName); + } else { + targetPeerAddress = getAddress(peerName); + peerSocketCache.put(peerName, targetPeerAddress); + } + ConcurrentLinkedQueue queue = outgoingQueues + .get(targetPeerAddress); + if (queue == null) { + queue = new ConcurrentLinkedQueue(); + } + queue.add(msg); + outgoingQueues.put(targetPeerAddress, queue); + } + } + + /* + * (non-Javadoc) + * @see org.apache.hama.bsp.BSPPeerInterface#sync() + */ + @Override + public void sync() throws IOException, KeeperException, InterruptedException { + enterBarrier(); + long startTime = System.currentTimeMillis(); + Iterator>> it = this.outgoingQueues + .entrySet().iterator(); + + while (it.hasNext()) { + Entry> entry = it + .next(); + + BSPPeerInterface peer = peers.get(entry.getKey()); + if (peer == null) { + peer = getBSPPeerConnection(entry.getKey()); + } + Iterable messages = entry.getValue(); + BSPMessageBundle bundle = new BSPMessageBundle(); + for (BSPMessage message : messages) { + bundle.addMessage(message); + } + peer.put(bundle); + } + + if ((System.currentTimeMillis() - startTime) < 200) { + Thread.sleep(200); + } + + leaveBarrier(); + currentTaskStatus.incrementSuperstepCount(); + umbilical.incrementSuperstepCount(taskid); + + startTime = System.currentTimeMillis(); + // Clear outgoing queues. + clearOutgoingQueues(); + + // Add non-processed messages from this iteration for the next's queue. + while (!localQueue.isEmpty()) { + BSPMessage message = localQueue.poll(); + localQueueForNextIteration.add(message); + } + // Switch local queues. + localQueue = localQueueForNextIteration; + localQueueForNextIteration = new ConcurrentLinkedQueue(); + + // TODO: This is a quite temporary solution of HAMA-387. + // If zk.getChildren() response is slower than 200 milliseconds, + // BSP system will be hanged. + + // We have to consider another way to avoid this problem. + if ((System.currentTimeMillis() - startTime) < 200) { + Thread.sleep(200); // at least wait + } + } + + protected boolean enterBarrier() throws KeeperException, InterruptedException { + LOG.debug("[" + getPeerName() + "] enter the enterbarrier: " + + this.getSuperstepCount()); + zk.create(bspRoot + "/" + getPeerName(), + Bytes.toBytes(this.getSuperstepCount()), Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL); + + while (true) { + synchronized (mutex) { + List list = zk.getChildren(bspRoot, true); + + if (list.size() < jobConf.getNumBspTask()) { + mutex.wait(); + } else { + return true; + } + } + } + } + + protected boolean leaveBarrier() throws KeeperException, InterruptedException { + zk.delete(bspRoot + "/" + getPeerName(), 0); + while (true) { + synchronized (mutex) { + List list = zk.getChildren(bspRoot, true); + + if (list.size() > 0) { + mutex.wait(); + } else { + return true; + } + } + } + } + + @Override + public void process(WatchedEvent event) { + synchronized (mutex) { + mutex.notify(); + } + } + + public void clear() { + this.localQueue.clear(); + this.outgoingQueues.clear(); + } + + @Override + public void close() throws IOException { + server.stop(); + } + + @Override + public void put(BSPMessage msg) throws IOException { + this.localQueueForNextIteration.add(msg); + } + + @Override + public void put(BSPMessageBundle messages) throws IOException { + for (BSPMessage message : messages.getMessages()) { + this.localQueueForNextIteration.add(message); + } + } + + @Override + public long getProtocolVersion(String arg0, long arg1) throws IOException { + return BSPPeerInterface.versionID; + } + + protected BSPPeerInterface getBSPPeerConnection(InetSocketAddress addr) { + BSPPeerInterface peer; + synchronized (this.peers) { + peer = peers.get(addr); + + if (peer == null) { + try { + peer = (BSPPeerInterface) RPC.getProxy(BSPPeerInterface.class, + BSPPeerInterface.versionID, addr, this.conf); + } catch (IOException e) { + + } + this.peers.put(addr, peer); + } + } + + return peer; + } + + /** + * @return the string as host:port of this Peer + */ + public String getPeerName() { + return peerAddress.getHostName() + ":" + peerAddress.getPort(); + } + + private InetSocketAddress getAddress(String peerName) { + String[] peerAddrParts = peerName.split(":"); + return new InetSocketAddress(peerAddrParts[0], + Integer.parseInt(peerAddrParts[1])); + } + + @Override + public String[] getAllPeerNames() { + return allPeerNames.toArray(new String[0]); + } + + /** + * To be invoked by the Groom Server with a list of peers received from an + * heartbeat response (BSPMaster). + * + * @param allPeers + */ + void setAllPeerNames(Collection allPeerNames) { + this.allPeerNames = new TreeSet(allPeerNames); + } + + /** + * @return the number of messages + */ + public int getNumCurrentMessages() { + return localQueue.size(); + } + + /** + * Sets the current status + * + * @param currentTaskStatus + */ + public void setCurrentTaskStatus(TaskStatus currentTaskStatus) { + this.currentTaskStatus = currentTaskStatus; + } + + /** + * @return the count of current super-step + */ + public long getSuperstepCount() { + return currentTaskStatus.getSuperstepCount(); + } + + /** + * Sets the job configuration + * + * @param jobConf + */ + public void setJobConf(BSPJob jobConf) { + this.jobConf = jobConf; + } + + /** + * @return the size of local queue + */ + public int getLocalQueueSize() { + return localQueue.size(); + } + + /** + * @return the size of outgoing queue + */ + public int getOutgoingQueueSize() { + return outgoingQueues.size(); + } + + /** + * Clears local queue + */ + public void clearLocalQueue() { + this.localQueue.clear(); + } + + /** + * Clears outgoing queues + */ + public void clearOutgoingQueues() { + this.outgoingQueues.clear(); + } +} Index: core/src/main/java/org/apache/hama/bsp/BSPTaskRunner.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPTaskRunner.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPTaskRunner.java (revision 0) @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Base class that runs a task in a separate process. + */ +public class BSPTaskRunner extends TaskRunner { + + public static final Log LOG = LogFactory.getLog(BSPTaskRunner.class); + + public BSPTaskRunner(BSPTask bspTask, GroomServer groom, BSPJob conf) { + super(bspTask, groom, conf); + } + +} Index: core/src/main/java/org/apache/hama/bsp/IntegerMessage.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/IntegerMessage.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/IntegerMessage.java (revision 0) @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A message that consists of a string tag and a int data. + */ +public class IntegerMessage extends BSPMessage { + + String tag; + int data; + + public IntegerMessage() { + super(); + } + + public IntegerMessage(String tag, int data) { + super(); + this.tag = tag; + this.data = data; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeUTF(tag); + out.writeInt(data); + } + + @Override + public void readFields(DataInput in) throws IOException { + tag = in.readUTF(); + data = in.readInt(); + } + + @Override + public String getTag() { + return tag; + } + + @Override + public Integer getData() { + return data; + } + +} \ No newline at end of file Index: core/src/main/java/org/apache/hama/bsp/Task.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/Task.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/Task.java (revision 0) @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.LocalDirAllocator; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hama.ipc.BSPPeerProtocol; + +/** + * Base class for tasks. + */ +public abstract class Task implements Writable { + public static final Log LOG = LogFactory.getLog(Task.class); + // ////////////////////////////////////////// + // Fields + // ////////////////////////////////////////// + + protected BSPJobID jobId; + protected String jobFile; + protected TaskAttemptID taskId; + protected int partition; + + protected LocalDirAllocator lDirAlloc; + + public Task() { + jobId = new BSPJobID(); + taskId = new TaskAttemptID(); + } + + public Task(BSPJobID jobId, String jobFile, TaskAttemptID taskId, + int partition) { + this.jobId = jobId; + this.jobFile = jobFile; + this.taskId = taskId; + this.partition = partition; + } + + // ////////////////////////////////////////// + // Accessors + // ////////////////////////////////////////// + public void setJobFile(String jobFile) { + this.jobFile = jobFile; + } + + public String getJobFile() { + return jobFile; + } + + public TaskAttemptID getTaskAttemptId() { + return this.taskId; + } + + public TaskAttemptID getTaskID() { + return taskId; + } + + /** + * Get the job name for this task. + * + * @return the job name + */ + public BSPJobID getJobID() { + return jobId; + } + + /** + * Get the index of this task within the job. + * + * @return the integer part of the task id + */ + public int getPartition() { + return partition; + } + + @Override + public String toString() { + return taskId.toString(); + } + + // ////////////////////////////////////////// + // Writable + // ////////////////////////////////////////// + @Override + public void write(DataOutput out) throws IOException { + jobId.write(out); + Text.writeString(out, jobFile); + taskId.write(out); + out.writeInt(partition); + } + + @Override + public void readFields(DataInput in) throws IOException { + jobId.readFields(in); + jobFile = Text.readString(in); + taskId.readFields(in); + partition = in.readInt(); + } + + /** + * Run this task as a part of the named job. This method is executed in the + * child process. + * + * @param bspPeer for communications + * @param umbilical for communications with GroomServer + */ + public abstract void run(BSPJob job, BSPPeer bspPeer, BSPPeerProtocol umbilical) + throws IOException; + + public abstract BSPTaskRunner createRunner(GroomServer groom); + + public void done(BSPPeerProtocol umbilical) throws IOException { + umbilical.done(getTaskID(), true); + } + + public abstract BSPJob getConf(); + public abstract void setConf(BSPJob localJobConf); + +} Index: core/src/main/java/org/apache/hama/bsp/DispatchTasksDirective.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/DispatchTasksDirective.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/DispatchTasksDirective.java (revision 0) @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; + +/** + * Handles the tasks dispatching between the BSPMaster and the GroomServers. + */ +public final class DispatchTasksDirective extends Directive implements Writable { + + public static final Log LOG = LogFactory.getLog(DispatchTasksDirective.class); + + private Map groomServerPeers; + private GroomServerAction[] actions; + + public DispatchTasksDirective() { + super(); + } + + public DispatchTasksDirective(Map groomServerPeers, + GroomServerAction[] actions) { + super(Directive.Type.Request); + this.groomServerPeers = groomServerPeers; + this.actions = actions; + } + + public Map getGroomServerPeers() { + return this.groomServerPeers; + } + + public GroomServerAction[] getActions() { + return this.actions; + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + if (this.actions == null) { + WritableUtils.writeVInt(out, 0); + } else { + WritableUtils.writeVInt(out, actions.length); + for (GroomServerAction action : this.actions) { + WritableUtils.writeEnum(out, action.getActionType()); + action.write(out); + } + } + String[] groomServerNames = groomServerPeers.keySet() + .toArray(new String[0]); + WritableUtils.writeCompressedStringArray(out, groomServerNames); + + List groomServerAddresses = new ArrayList( + groomServerNames.length); + for (String groomName : groomServerNames) { + groomServerAddresses.add(groomServerPeers.get(groomName)); + } + WritableUtils.writeCompressedStringArray(out, groomServerAddresses + .toArray(new String[0])); + + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + int length = WritableUtils.readVInt(in); + if (length > 0) { + this.actions = new GroomServerAction[length]; + for (int i = 0; i < length; ++i) { + GroomServerAction.ActionType actionType = WritableUtils.readEnum(in, + GroomServerAction.ActionType.class); + actions[i] = GroomServerAction.createAction(actionType); + actions[i].readFields(in); + } + } else { + this.actions = null; + } + String[] groomServerNames = WritableUtils.readCompressedStringArray(in); + String[] groomServerAddresses = WritableUtils.readCompressedStringArray(in); + groomServerPeers = new HashMap(groomServerNames.length); + + for (int i = 0; i < groomServerNames.length; i++) { + groomServerPeers.put(groomServerNames[i], groomServerAddresses[i]); + } + } +} Index: core/src/main/java/org/apache/hama/bsp/GroomServerManager.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/GroomServerManager.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/GroomServerManager.java (revision 0) @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.util.Collection; +import java.util.Map; + +import org.apache.hama.ipc.GroomProtocol; + +/** + * Manages information about the {@link GroomServer}s in the cluster + * environment. This interface is not intended to be implemented by users. + */ +interface GroomServerManager { + + /** + * Get the current status of the cluster + * @param detailed if true then report groom names as well + * @return summary of the state of the cluster + */ + ClusterStatus getClusterStatus(boolean detailed); + + /** + * Find WorkerProtocol with corresponded groom server status + * + * @param groomId The identification value of GroomServer + * @return GroomServerStatus + */ + GroomProtocol findGroomServer(GroomServerStatus status); + + /** + * Find the collection of groom servers. + * + * @return Collection of groom servers list. + */ + Collection findGroomServers(); + + /** + * Collection of GroomServerStatus as the key set. + * + * @return Collection of GroomServerStatus. + */ + Collection groomServerStatusKeySet(); + + /** + * Registers a JobInProgressListener to GroomServerManager. Therefore, + * adding a JobInProgress will trigger the jobAdded function. + * @param the JobInProgressListener listener to be added. + */ + void addJobInProgressListener(JobInProgressListener listener); + + /** + * Unregisters a JobInProgressListener to GroomServerManager. Therefore, + * the remove of a JobInProgress will trigger the jobRemoved action. + * @param the JobInProgressListener to be removed. + */ + void removeJobInProgressListener(JobInProgressListener listener); + + /** + * Current GroomServer Peers. + * @return GroomName and PeerName(host:port) in pair. + */ + Map currentGroomServerPeers(); +} Index: core/src/main/java/org/apache/hama/bsp/GroomServerAction.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/GroomServerAction.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/GroomServerAction.java (revision 0) @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; + +/** + * A generic directive from the {@link org.apache.hama.bsp.BSPMaster} + * to the {@link org.apache.hama.bsp.GroomServer} to take some 'action'. + */ +abstract class GroomServerAction implements Writable { + + /** + * Ennumeration of various 'actions' that the {@link BSPMaster} + * directs the {@link GroomServer} to perform periodically. + * + */ + public static enum ActionType { + /** Launch a new task. */ + LAUNCH_TASK, + + /** Kill a task. */ + KILL_TASK, + + /** Kill any tasks of this job and cleanup. */ + KILL_JOB, + + /** Reinitialize the groomserver. */ + REINIT_GROOM, + + /** Ask a task to save its output. */ + COMMIT_TASK + }; + + /** + * A factory-method to create objects of given {@link ActionType}. + * @param actionType the {@link ActionType} of object to create. + * @return an object of {@link ActionType}. + */ + public static GroomServerAction createAction(ActionType actionType) { + GroomServerAction action = null; + + switch (actionType) { + case LAUNCH_TASK: + { + action = new LaunchTaskAction(); + } + break; + case KILL_TASK: + { + action = new KillTaskAction(); + } + break; + case KILL_JOB: + { + action = new KillJobAction(); + } + break; + case REINIT_GROOM: + { + action = new ReinitGroomAction(); + } + break; + case COMMIT_TASK: + { + action = new CommitTaskAction(); + } + break; + } + + return action; + } + + private ActionType actionType; + + protected GroomServerAction(ActionType actionType) { + this.actionType = actionType; + } + + /** + * Return the {@link ActionType}. + * @return the {@link ActionType}. + */ + ActionType getActionType() { + return actionType; + } + + public void write(DataOutput out) throws IOException { + WritableUtils.writeEnum(out, actionType); + } + + public void readFields(DataInput in) throws IOException { + actionType = WritableUtils.readEnum(in, ActionType.class); + } +} Index: core/src/main/java/org/apache/hama/bsp/ByteMessage.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/ByteMessage.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/ByteMessage.java (revision 0) @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A message that consists of a byte tag and a byte data. + */ +public class ByteMessage extends BSPMessage { + + private byte[] tag; + private byte[] data; + + public ByteMessage() { + super(); + } + + public ByteMessage(byte[] tag, byte[] data) { + super(); + this.tag = tag; + this.data = data; + } + + @Override + public byte[] getTag() { + return this.tag; + } + + @Override + public byte[] getData() { + return this.data; + } + + @Override + public void readFields(DataInput in) throws IOException { + this.tag = new byte[in.readInt()]; + in.readFully(tag, 0, this.tag.length); + this.data = new byte[in.readInt()]; + in.readFully(data, 0, this.data.length); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(tag.length); + out.write(tag); + out.writeInt(data.length); + out.write(data); + } + +} Index: core/src/main/java/org/apache/hama/bsp/CommitTaskAction.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/CommitTaskAction.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/CommitTaskAction.java (revision 0) @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} + * to the {@link org.apache.hama.bsp.GroomServer} to commit the output + * of the task. + */ +class CommitTaskAction extends GroomServerAction { + private TaskAttemptID taskId; + + public CommitTaskAction() { + super(ActionType.COMMIT_TASK); + taskId = new TaskAttemptID(); + } + + public CommitTaskAction(TaskAttemptID taskId) { + super(ActionType.COMMIT_TASK); + this.taskId = taskId; + } + + public TaskAttemptID getTaskID() { + return taskId; + } + + public void write(DataOutput out) throws IOException { + taskId.write(out); + } + + public void readFields(DataInput in) throws IOException { + taskId.readFields(in); + } +} Index: core/src/main/java/org/apache/hama/bsp/RunningJob.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/RunningJob.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/RunningJob.java (revision 0) @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; + +/** + * RunningJob is the user-interface to query for details on a + * running BSP job. + * + *

+ * Clients can get hold of RunningJob via the {@link BSPJobClient} + * and then query the running-job for details such as name, configuration, + * progress etc. + *

+ * + * @see BSPJobClient + */ +public interface RunningJob { + /** + * Get the job identifier. + * + * @return the job identifier. + */ + public BSPJobID getID(); + + /** + * Get the name of the job. + * + * @return the name of the job. + */ + public String getJobName(); + + /** + * Get the path of the submitted job configuration. + * + * @return the path of the submitted job configuration. + */ + public String getJobFile(); + + /** + * Get the progress of the job's tasks, as a float between 0.0 and 1.0. + * When all bsp tasks have completed, the function returns 1.0. + * + * @return the progress of the job's tasks. + * @throws IOException + */ + public long progress() throws IOException; + + /** + * Check if the job is finished or not. This is a non-blocking call. + * + * @return true if the job is complete, else false. + * @throws IOException + */ + public boolean isComplete() throws IOException; + + /** + * Check if the job completed successfully. + * + * @return true if the job succeeded, else false. + * @throws IOException + */ + public boolean isSuccessful() throws IOException; + + /** + * Blocks until the job is complete. + * + * @throws IOException + */ + public void waitForCompletion() throws IOException; + + /** + * Returns the current state of the Job. {@link JobStatus} + * + * @throws IOException + */ + public int getJobState() throws IOException; + + /** + * Kill the running job. Blocks until all job tasks have been killed as well. + * If the job is no longer running, it simply returns. + * + * @throws IOException + */ + public void killJob() throws IOException; + + /** + * Kill indicated task attempt. + * + * @param taskId the id of the task to be terminated. + * @param shouldFail if true the task is failed and added to failed tasks + * list, otherwise it is just killed, w/o affecting job failure + * status. + * @throws IOException + */ + public void killTask(TaskAttemptID taskId, boolean shouldFail) + throws IOException; + + public long getSuperstepCount() throws IOException; +} Index: core/src/main/java/org/apache/hama/bsp/Schedulable.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/Schedulable.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/Schedulable.java (revision 0) @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; + +/** + * This is the class that schedules commands to GroomServer(s) + */ +public interface Schedulable{ + + /** + * Schedule job to designated GroomServer(s) immediately. + * @param job to be scheduled. + * @param statuses of GroomServer(s). + * @throws IOException + */ + void schedule(JobInProgress job, GroomServerStatus... statuses) + throws IOException; +} Index: core/src/main/java/org/apache/hama/bsp/SimpleTaskScheduler.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/SimpleTaskScheduler.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/SimpleTaskScheduler.java (revision 0) @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hama.ipc.GroomProtocol; + +/** + * A simple task scheduler. + */ +class SimpleTaskScheduler extends TaskScheduler { + + private static final Log LOG = LogFactory.getLog(SimpleTaskScheduler.class); + + public static final String WAIT_QUEUE = "waitQueue"; + public static final String PROCESSING_QUEUE = "processingQueue"; + public static final String FINISHED_QUEUE = "finishedQueue"; + + private QueueManager queueManager; + private volatile boolean initialized; + private JobListener jobListener; + private JobProcessor jobProcessor; + + private class JobListener extends JobInProgressListener { + @Override + public void jobAdded(JobInProgress job) throws IOException { + queueManager.initJob(job); // init task + queueManager.addJob(WAIT_QUEUE, job); + } + + @Override + public void jobRemoved(JobInProgress job) throws IOException { + // queueManager.removeJob(WAIT_QUEUE, job); + queueManager.moveJob(PROCESSING_QUEUE, FINISHED_QUEUE, job); + } + } + + private class JobProcessor extends Thread implements Schedulable { + JobProcessor() { + super("JobProcess"); + } + + /** + * Main logic scheduling task to GroomServer(s). Also, it will move + * JobInProgress from Wait Queue to Processing Queue. + */ + public void run() { + if (false == initialized) { + throw new IllegalStateException("SimpleTaskScheduler initialization" + + " is not yet finished!"); + } + while (initialized) { + Queue queue = queueManager.findQueue(WAIT_QUEUE); + if (null == queue) { + LOG.error(WAIT_QUEUE + " does not exist."); + throw new NullPointerException(WAIT_QUEUE + " does not exist."); + } + // move a job from the wait queue to the processing queue + JobInProgress j = queue.removeJob(); + queueManager.addJob(PROCESSING_QUEUE, j); + // schedule + Collection glist = groomServerManager + .groomServerStatusKeySet(); + schedule(j, (GroomServerStatus[]) glist + .toArray(new GroomServerStatus[glist.size()])); + } + } + + /** + * Schedule job to designated GroomServer(s) immediately. + * + * @param Targeted GroomServer(s). + * @param Job to be scheduled. + */ + @Override + public void schedule(JobInProgress job, GroomServerStatus... statuses) { + ClusterStatus clusterStatus = groomServerManager.getClusterStatus(false); + final int numGroomServers = clusterStatus.getGroomServers(); + final ScheduledExecutorService sched = Executors + .newScheduledThreadPool(statuses.length + 5); + for (GroomServerStatus status : statuses) { + sched + .schedule(new TaskWorker(status, numGroomServers, job), 0, SECONDS); + }// for + } + } + + private class TaskWorker implements Runnable { + private final GroomServerStatus stus; + private final int groomNum; + private final JobInProgress jip; + + TaskWorker(final GroomServerStatus stus, final int num, + final JobInProgress jip) { + this.stus = stus; + this.groomNum = num; + this.jip = jip; + if (null == this.stus) + throw new NullPointerException("Target groom server is not " + + "specified."); + if (-1 == this.groomNum) + throw new IllegalArgumentException("Groom number is not specified."); + if (null == this.jip) + throw new NullPointerException("No job is specified."); + } + + public void run() { + // obtain tasks + Task t = jip.obtainNewTask(this.stus, groomNum); + + // assembly into actions + // List tasks = new ArrayList(); + if (jip.getStatus().getRunState() == JobStatus.RUNNING) { + GroomProtocol worker = groomServerManager.findGroomServer(this.stus); + try { + // dispatch() to the groom server + Directive d1 = new DispatchTasksDirective(groomServerManager + .currentGroomServerPeers(), new GroomServerAction[] { + new LaunchTaskAction(t)}); + worker.dispatch(d1); + } catch (IOException ioe) { + LOG.error("Fail to dispatch tasks to GroomServer " + + this.stus.getGroomName(), ioe); + } + } else { + LOG.warn("Currently master only shcedules job in running state. " + + "This may be refined in the future. JobId:" + jip.getJobID()); + } + } + } + + public SimpleTaskScheduler() { + this.jobListener = new JobListener(); + this.jobProcessor = new JobProcessor(); + } + + @Override + public void start() { + this.queueManager = new QueueManager(getConf()); // TODO: need factory? + this.queueManager.createFCFSQueue(WAIT_QUEUE); + this.queueManager.createFCFSQueue(PROCESSING_QUEUE); + this.queueManager.createFCFSQueue(FINISHED_QUEUE); + groomServerManager.addJobInProgressListener(this.jobListener); + this.initialized = true; + this.jobProcessor.start(); + } + + @Override + public void terminate() { + this.initialized = false; + if (null != this.jobListener) + groomServerManager.removeJobInProgressListener(this.jobListener); + } + + @Override + public Collection getJobs(String queue) { + return (queueManager.findQueue(queue)).jobs(); + // return jobQueue; + } +} Index: core/src/main/java/org/apache/hama/bsp/DirectiveHandler.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/DirectiveHandler.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/DirectiveHandler.java (revision 0) @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +/** + * A DirectiveHandler interface. + */ +public interface DirectiveHandler{ + + /** + * Handle directives on demand. + * @param directive to be handled. + */ + void handle(Directive directive) throws DirectiveException; +} Index: core/src/main/java/org/apache/hama/bsp/KillTaskAction.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/KillTaskAction.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/KillTaskAction.java (revision 0) @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} + * to the {@link org.apache.hama.bsp.GroomServer} to kill a task. + */ +class KillTaskAction extends GroomServerAction { + TaskAttemptID taskId; + + public KillTaskAction() { + super(ActionType.KILL_TASK); + taskId = new TaskAttemptID(); + } + + public KillTaskAction(TaskAttemptID killTaskId) { + super(ActionType.KILL_TASK); + this.taskId = killTaskId; + } + + public TaskAttemptID getTaskID() { + return taskId; + } + + @Override + public void write(DataOutput out) throws IOException { + taskId.write(out); + } + + @Override + public void readFields(DataInput in) throws IOException { + taskId.readFields(in); + } +} Index: core/src/main/java/org/apache/hama/bsp/GroomServerStatus.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/GroomServerStatus.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/GroomServerStatus.java (revision 0) @@ -0,0 +1,227 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableFactories; +import org.apache.hadoop.io.WritableFactory; + +/** + * A GroomServerStatus is a BSP primitive. Keeps info on a BSPMaster. The + * BSPMaster maintains a set of the most recent GroomServerStatus objects for + * each unique GroomServer it knows about. + */ +public class GroomServerStatus implements Writable { + public static final Log LOG = LogFactory.getLog(GroomServerStatus.class); + + static { + WritableFactories.setFactory(GroomServerStatus.class, + new WritableFactory() { + public Writable newInstance() { + return new GroomServerStatus(); + } + }); + } + + String groomName; + String peerName; + String rpcServer; + int failures; + List taskReports; + + volatile long lastSeen; + private int maxTasks; + + public GroomServerStatus() { + // taskReports = new ArrayList(); + taskReports = new CopyOnWriteArrayList(); + } + + public GroomServerStatus(String groomName, String peerName, + List taskReports, int failures, int maxTasks) { + this(groomName, peerName, taskReports, failures, maxTasks, ""); + } + + public GroomServerStatus(String groomName, String peerName, + List taskReports, int failures, int maxTasks, String rpc) { + this.groomName = groomName; + this.peerName = peerName; + this.taskReports = new ArrayList(taskReports); + this.failures = failures; + this.maxTasks = maxTasks; + this.rpcServer = rpc; + } + + public String getGroomName() { + return groomName; + } + + /** + * The host (and port) from where the groom server can be reached. + * + * @return The groom server address in the form of "hostname:port" + */ + public String getPeerName() { + return peerName; + } + + public String getRpcServer() { + return rpcServer; + } + + /** + * Get the current tasks at the GroomServer. Tasks are tracked by a + * {@link TaskStatus} object. + * + * @return a list of {@link TaskStatus} representing the current tasks at the + * GroomServer. + */ + public List getTaskReports() { + return taskReports; + } + + public int getFailures() { + return failures; + } + + public long getLastSeen() { + return lastSeen; + } + + public void setLastSeen(long lastSeen) { + this.lastSeen = lastSeen; + } + + public int getMaxTasks() { + return maxTasks; + } + + /** + * Return the current MapTask count + */ + public int countTasks() { + int taskCount = 0; + for (Iterator it = taskReports.iterator(); it.hasNext();) { + TaskStatus ts = it.next(); + TaskStatus.State state = ts.getRunState(); + if (state == TaskStatus.State.RUNNING + || state == TaskStatus.State.UNASSIGNED) { + taskCount++; + } + } + + return taskCount; + } + + /** + * For BSPMaster to distinguish between different GroomServers, because + * BSPMaster stores using GroomServerStatus as key. + */ + @Override + public int hashCode() { + int result = 17; + result = 37 * result + groomName.hashCode(); + result = 37 * result + peerName.hashCode(); + result = 37 * result + rpcServer.hashCode(); + /* + * result = 37*result + (int)failures; result = 37*result + + * taskReports.hashCode(); result = 37*result + + * (int)(lastSeen^(lastSeen>>>32)); result = 37*result + (int)maxTasks; + */ + return result; + } + + @Override + public boolean equals(Object o) { + if (o == this) + return true; + if (null == o) + return false; + if (getClass() != o.getClass()) + return false; + + GroomServerStatus s = (GroomServerStatus) o; + if (!s.groomName.equals(groomName)) + return false; + if (!s.peerName.equals(peerName)) + return false; + if (!s.rpcServer.equals(rpcServer)) + return false; + /* + * if(s.failures != failures) return false; if(null == s.taskReports){ + * if(null != s.taskReports) return false; }else + * if(!s.taskReports.equals(taskReports)){ return false; } if(s.lastSeen != + * lastSeen) return false; if(s.maxTasks != maxTasks) return false; + */ + return true; + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput) + */ + @Override + public void readFields(DataInput in) throws IOException { + this.groomName = Text.readString(in); + this.peerName = Text.readString(in); + this.rpcServer = Text.readString(in); + this.failures = in.readInt(); + this.maxTasks = in.readInt(); + taskReports.clear(); + int numTasks = in.readInt(); + + TaskStatus status; + for (int i = 0; i < numTasks; i++) { + status = new TaskStatus(); + status.readFields(in); + taskReports.add(status); + } + } + + /* + * (non-Javadoc) + * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput) + */ + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, groomName); + Text.writeString(out, peerName); + Text.writeString(out, rpcServer); + out.writeInt(failures); + out.writeInt(maxTasks); + out.writeInt(taskReports.size()); + for (TaskStatus taskStatus : taskReports) { + taskStatus.write(out); + } + } + + public Iterator taskReports() { + return taskReports.iterator(); + } +} Index: core/src/main/java/org/apache/hama/bsp/BSPJobClient.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPJobClient.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPJobClient.java (revision 0) @@ -0,0 +1,679 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; +import java.util.Random; + +import javax.security.auth.login.LoginException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UnixUserGroupInformation; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.ipc.JobSubmissionProtocol; + +/** + * BSPJobClient is the primary interface for the user-job to interact with the + * BSPMaster. + * + * BSPJobClient provides facilities to submit jobs, track their progress, access + * component-tasks' reports/logs, get the BSP cluster status information etc. + */ +public class BSPJobClient extends Configured implements Tool { + private static final Log LOG = LogFactory.getLog(BSPJobClient.class); + + public static enum TaskStatusFilter { + NONE, KILLED, FAILED, SUCCEEDED, ALL + } + + private static final long MAX_JOBPROFILE_AGE = 1000 * 2; + + class NetworkedJob implements RunningJob { + JobProfile profile; + JobStatus status; + long statustime; + + public NetworkedJob(JobStatus job) throws IOException { + this.status = job; + this.profile = jobSubmitClient.getJobProfile(job.getJobID()); + this.statustime = System.currentTimeMillis(); + } + + /** + * Some methods rely on having a recent job profile object. Refresh it, if + * necessary + */ + synchronized void ensureFreshStatus() throws IOException { + if (System.currentTimeMillis() - statustime > MAX_JOBPROFILE_AGE) { + updateStatus(); + } + } + + /** + * Some methods need to update status immediately. So, refresh immediately + * + * @throws IOException + */ + synchronized void updateStatus() throws IOException { + this.status = jobSubmitClient.getJobStatus(profile.getJobID()); + this.statustime = System.currentTimeMillis(); + } + + /* + * (non-Javadoc) + * @see org.apache.hama.bsp.RunningJob#getID() + */ + @Override + public BSPJobID getID() { + return profile.getJobID(); + } + + /* + * (non-Javadoc) + * @see org.apache.hama.bsp.RunningJob#getJobName() + */ + @Override + public String getJobName() { + return profile.getJobName(); + } + + /* + * (non-Javadoc) + * @see org.apache.hama.bsp.RunningJob#getJobFile() + */ + @Override + public String getJobFile() { + return profile.getJobFile(); + } + + @Override + public long progress() throws IOException { + ensureFreshStatus(); + return status.progress(); + } + + /** + * Returns immediately whether the whole job is done yet or not. + */ + public synchronized boolean isComplete() throws IOException { + updateStatus(); + return (status.getRunState() == JobStatus.SUCCEEDED + || status.getRunState() == JobStatus.FAILED || status.getRunState() == JobStatus.KILLED); + } + + /** + * True if job completed successfully. + */ + public synchronized boolean isSuccessful() throws IOException { + updateStatus(); + return status.getRunState() == JobStatus.SUCCEEDED; + } + + public synchronized long getSuperstepCount() throws IOException { + ensureFreshStatus(); + return status.getSuperstepCount(); + } + + /** + * Blocks until the job is finished + */ + public void waitForCompletion() throws IOException { + while (!isComplete()) { + try { + Thread.sleep(5000); + } catch (InterruptedException ie) { + } + } + } + + /** + * Tells the service to get the state of the current job. + */ + public synchronized int getJobState() throws IOException { + updateStatus(); + return status.getRunState(); + } + + /** + * Tells the service to terminate the current job. + */ + public synchronized void killJob() throws IOException { + jobSubmitClient.killJob(getID()); + } + + @Override + public void killTask(TaskAttemptID taskId, boolean shouldFail) + throws IOException { + jobSubmitClient.killTask(taskId, shouldFail); + } + } + + private JobSubmissionProtocol jobSubmitClient = null; + private Path sysDir = null; + private FileSystem fs = null; + + // job files are world-wide readable and owner writable + final private static FsPermission JOB_FILE_PERMISSION = FsPermission + .createImmutable((short) 0644); // rw-r--r-- + + // job submission directory is world readable/writable/executable + final static FsPermission JOB_DIR_PERMISSION = FsPermission + .createImmutable((short) 0777); // rwx-rwx-rwx + + public BSPJobClient(Configuration conf) throws IOException { + setConf(conf); + init(conf); + } + + public BSPJobClient() { + } + + public void init(Configuration conf) throws IOException { + String masterAdress = conf.get("bsp.master.address"); + if (masterAdress != null && !masterAdress.equals("local")) { + this.jobSubmitClient = (JobSubmissionProtocol) RPC.getProxy( + JobSubmissionProtocol.class, JobSubmissionProtocol.versionID, + BSPMaster.getAddress(conf), conf, + NetUtils.getSocketFactory(conf, JobSubmissionProtocol.class)); + } else { + LOG.debug("Using local BSP runner."); + this.jobSubmitClient = new LocalBSPRunner(conf); + } + } + + /** + * Close the JobClient. + */ + public synchronized void close() throws IOException { + RPC.stopProxy(jobSubmitClient); + } + + /** + * Get a filesystem handle. We need this to prepare jobs for submission to the + * BSP system. + * + * @return the filesystem handle. + */ + public synchronized FileSystem getFs() throws IOException { + if (this.fs == null) { + Path sysDir = getSystemDir(); + this.fs = sysDir.getFileSystem(getConf()); + } + return fs; + } + + /** + * Gets the jobs that are submitted. + * + * @return array of {@link JobStatus} for the submitted jobs. + * @throws IOException + */ + public JobStatus[] getAllJobs() throws IOException { + return jobSubmitClient.getAllJobs(); + } + + /** + * Gets the jobs that are not completed and not failed. + * + * @return array of {@link JobStatus} for the running/to-be-run jobs. + * @throws IOException + */ + public JobStatus[] jobsToComplete() throws IOException { + return jobSubmitClient.jobsToComplete(); + } + + private UnixUserGroupInformation getUGI(Configuration conf) + throws IOException { + UnixUserGroupInformation ugi = null; + try { + ugi = UnixUserGroupInformation.login(conf, true); + } catch (LoginException e) { + throw (IOException) (new IOException( + "Failed to get the current user's information.").initCause(e)); + } + return ugi; + } + + /** + * Submit a job to the BSP system. This returns a handle to the + * {@link RunningJob} which can be used to track the running-job. + * + * @param job the job configuration. + * @return a handle to the {@link RunningJob} which can be used to track the + * running-job. + * @throws FileNotFoundException + * @throws IOException + */ + public RunningJob submitJob(BSPJob job) throws FileNotFoundException, + IOException { + return submitJobInternal(job); + } + + static Random r = new Random(); + + public RunningJob submitJobInternal(BSPJob job) throws IOException { + BSPJobID jobId = jobSubmitClient.getNewJobId(); + + Path submitJobDir = new Path(getSystemDir(), "submit_" + + Integer.toString(Math.abs(r.nextInt()), 36)); + Path submitJarFile = new Path(submitJobDir, "job.jar"); + Path submitJobFile = new Path(submitJobDir, "job.xml"); + LOG.debug("BSPJobClient.submitJobDir: " + submitJobDir); + + /* + * set this user's id in job configuration, so later job files can be + * accessed using this user's id + */ + UnixUserGroupInformation ugi = getUGI(job.getConf()); + + ClusterStatus clusterStatus = getClusterStatus(true); + + // check the number of BSP tasks + int tasks = job.getNumBspTask(); + int maxTasks = clusterStatus.getMaxTasks(); + + if (tasks <= 0 || tasks > maxTasks) { + LOG.warn("The number of tasks you've entered was invalid. Using default value of " + + maxTasks + "!"); + job.setNumBspTask(maxTasks); + } + + // Create a number of filenames in the BSPMaster's fs namespace + FileSystem fs = getFs(); + fs.delete(submitJobDir, true); + submitJobDir = fs.makeQualified(submitJobDir); + submitJobDir = new Path(submitJobDir.toUri().getPath()); + FsPermission bspSysPerms = new FsPermission(JOB_DIR_PERMISSION); + FileSystem.mkdirs(fs, submitJobDir, bspSysPerms); + fs.mkdirs(submitJobDir); + short replication = (short) job.getInt("bsp.submit.replication", 10); + + String originalJarPath = job.getJar(); + + if (originalJarPath != null) { // copy jar to BSPMaster's fs + // use jar name if job is not named. + if ("".equals(job.getJobName())) { + job.setJobName(new Path(originalJarPath).getName()); + } + job.setJar(submitJarFile.toString()); + fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile); + + fs.setReplication(submitJarFile, replication); + fs.setPermission(submitJarFile, new FsPermission(JOB_FILE_PERMISSION)); + } else { + LOG.warn("No job jar file set. User classes may not be found. " + + "See BSPJob#setJar(String) or check Your jar file."); + } + + // Set the user's name and working directory + job.setUser(ugi.getUserName()); + if (ugi.getGroupNames().length > 0) { + job.set("group.name", ugi.getGroupNames()[0]); + } + if (job.getWorkingDirectory() == null) { + job.setWorkingDirectory(fs.getWorkingDirectory()); + } + + // Write job file to BSPMaster's fs + FSDataOutputStream out = FileSystem.create(fs, submitJobFile, + new FsPermission(JOB_FILE_PERMISSION)); + + try { + job.writeXml(out); + } finally { + out.close(); + } + + // + // Now, actually submit the job (using the submit name) + // + JobStatus status = jobSubmitClient.submitJob(jobId, + submitJobFile.toString()); + if (status != null) { + return new NetworkedJob(status); + } else { + throw new IOException("Could not launch job"); + } + } + + /** + * Monitor a job and print status in real-time as progress is made and tasks + * fail. + * + * @param job + * @param info + * @return true, if job is successful + * @throws IOException + * @throws InterruptedException + */ + public boolean monitorAndPrintJob(BSPJob job, RunningJob info) + throws IOException, InterruptedException { + + String lastReport = null; + LOG.info("Running job: " + info.getID()); + + while (!job.isComplete()) { + Thread.sleep(3000); + long step = job.progress(); + String report = "Current supersteps number: " + step; + + if (!report.equals(lastReport)) { + LOG.info(report); + lastReport = report; + } + } + + LOG.info("The total number of supersteps: " + info.getSuperstepCount()); + return job.isSuccessful(); + } + + /** + * Grab the bspmaster system directory path where job-specific files are to be + * placed. + * + * @return the system directory where job-specific files are to be placed. + */ + public Path getSystemDir() { + if (sysDir == null) { + sysDir = new Path(jobSubmitClient.getSystemDir()); + } + return sysDir; + } + + public static void runJob(BSPJob job) throws FileNotFoundException, + IOException { + BSPJobClient jc = new BSPJobClient(job.getConf()); + + if (job.getNumBspTask() == 0 + || job.getNumBspTask() > jc.getClusterStatus(false).getMaxTasks()) { + job.setNumBspTask(jc.getClusterStatus(false).getMaxTasks()); + } + + RunningJob running = jc.submitJobInternal(job); + BSPJobID jobId = running.getID(); + LOG.info("Running job: " + jobId.toString()); + + while (true) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + + if (running.isComplete()) { + break; + } + + running = jc.getJob(jobId); + } + + LOG.info("Job complete: " + jobId); + LOG.info("The total number of supersteps: " + running.getSuperstepCount()); + + // TODO if error found, kill job + // running.killJob(); + jc.close(); + } + + /** + * Get an RunningJob object to track an ongoing job. Returns null if the id + * does not correspond to any known job. + * + * @throws IOException + */ + private RunningJob getJob(BSPJobID jobId) throws IOException { + JobStatus status = jobSubmitClient.getJobStatus(jobId); + if (status != null) { + return new NetworkedJob(status); + } else { + return null; + } + } + + /** + * Get status information about the BSP cluster + * + * @param detailed if true then get a detailed status including the + * groomserver names + * + * @return the status information about the BSP cluster as an object of + * {@link ClusterStatus}. + * + * @throws IOException + */ + public ClusterStatus getClusterStatus(boolean detailed) throws IOException { + return jobSubmitClient.getClusterStatus(detailed); + } + + @Override + public int run(String[] args) throws Exception { + int exitCode = -1; + if (args.length < 1) { + displayUsage(""); + return exitCode; + } + + // process arguments + String cmd = args[0]; + boolean listJobs = false; + boolean listAllJobs = false; + boolean listActiveGrooms = false; + boolean killJob = false; + boolean submitJob = false; + boolean getStatus = false; + String submitJobFile = null; + String jobid = null; + + HamaConfiguration conf = new HamaConfiguration(getConf()); + init(conf); + + if ("-list".equals(cmd)) { + if (args.length != 1 && !(args.length == 2 && "all".equals(args[1]))) { + displayUsage(cmd); + return exitCode; + } + if (args.length == 2 && "all".equals(args[1])) { + listAllJobs = true; + } else { + listJobs = true; + } + } else if ("-list-active-grooms".equals(cmd)) { + if (args.length != 1) { + displayUsage(cmd); + return exitCode; + } + listActiveGrooms = true; + } else if ("-submit".equals(cmd)) { + if (args.length == 1) { + displayUsage(cmd); + return exitCode; + } + + submitJob = true; + submitJobFile = args[1]; + } else if ("-kill".equals(cmd)) { + if (args.length == 1) { + displayUsage(cmd); + return exitCode; + } + killJob = true; + jobid = args[1]; + + } else if ("-status".equals(cmd)) { + if (args.length != 2) { + displayUsage(cmd); + return exitCode; + } + jobid = args[1]; + getStatus = true; + + // TODO Later, below functions should be implemented + // with the Fault Tolerant mechanism. + } else if ("-list-attempt-ids".equals(cmd)) { + System.out.println("This function is not implemented yet."); + return exitCode; + } else if ("-kill-task".equals(cmd)) { + System.out.println("This function is not implemented yet."); + return exitCode; + } else if ("-fail-task".equals(cmd)) { + System.out.println("This function is not implemented yet."); + return exitCode; + } + + BSPJobClient jc = new BSPJobClient(new HamaConfiguration()); + if (listJobs) { + listJobs(); + exitCode = 0; + } else if (listAllJobs) { + listAllJobs(); + exitCode = 0; + } else if (listActiveGrooms) { + listActiveGrooms(); + exitCode = 0; + } else if (submitJob) { + HamaConfiguration tConf = new HamaConfiguration(new Path(submitJobFile)); + RunningJob job = jc.submitJob(new BSPJob(tConf)); + System.out.println("Created job " + job.getID().toString()); + } else if (killJob) { + RunningJob job = jc.getJob(BSPJobID.forName(jobid)); + if (job == null) { + System.out.println("Could not find job " + jobid); + } else { + job.killJob(); + System.out.println("Killed job " + jobid); + } + exitCode = 0; + } else if (getStatus) { + RunningJob job = jc.getJob(BSPJobID.forName(jobid)); + if (job == null) { + System.out.println("Could not find job " + jobid); + } else { + JobStatus jobStatus = jobSubmitClient.getJobStatus(job.getID()); + System.out.println("Job name: " + job.getJobName()); + System.out.printf("States are:\n\tRunning : 1\tSucceded : 2" + + "\tFailed : 3\tPrep : 4\n"); + System.out.printf("%s\t%d\t%d\t%s\n", jobStatus.getJobID(), + jobStatus.getRunState(), jobStatus.getStartTime(), + jobStatus.getUsername()); + + exitCode = 0; + } + } + + return 0; + } + + /** + * Display usage of the command-line tool and terminate execution + */ + private void displayUsage(String cmd) { + String prefix = "Usage: hama job "; + String taskStates = "running, completed"; + if ("-submit".equals(cmd)) { + System.err.println(prefix + "[" + cmd + " ]"); + } else if ("-status".equals(cmd) || "-kill".equals(cmd)) { + System.err.println(prefix + "[" + cmd + " ]"); + } else if ("-list".equals(cmd)) { + System.err.println(prefix + "[" + cmd + " [all]]"); + } else if ("-kill-task".equals(cmd) || "-fail-task".equals(cmd)) { + System.err.println(prefix + "[" + cmd + " ]"); + } else if ("-list-active-grooms".equals(cmd)) { + System.err.println(prefix + "[" + cmd + "]"); + } else if ("-list-attempt-ids".equals(cmd)) { + System.err.println(prefix + "[" + cmd + " ]. " + + "Valid values for are " + taskStates); + } else { + System.err.printf(prefix + " \n"); + System.err.printf("\t[-submit ]\n"); + System.err.printf("\t[-status ]\n"); + System.err.printf("\t[-kill ]\n"); + System.err.printf("\t[-list [all]]\n"); + System.err.printf("\t[-list-active-grooms]\n"); + System.err.println("\t[-list-attempt-ids " + "]\n"); + System.err.printf("\t[-kill-task ]\n"); + System.err.printf("\t[-fail-task ]\n\n"); + } + } + + /** + * Dump a list of currently running jobs + * + * @throws IOException + */ + private void listJobs() throws IOException { + JobStatus[] jobs = jobsToComplete(); + if (jobs == null) + jobs = new JobStatus[0]; + + System.out.printf("%d jobs currently running\n", jobs.length); + displayJobList(jobs); + } + + /** + * Dump a list of all jobs submitted. + * + * @throws IOException + */ + private void listAllJobs() throws IOException { + JobStatus[] jobs = getAllJobs(); + if (jobs == null) + jobs = new JobStatus[0]; + System.out.printf("%d jobs submitted\n", jobs.length); + System.out.printf("States are:\n\tRunning : 1\tSucceded : 2" + + "\tFailed : 3\tPrep : 4\n"); + displayJobList(jobs); + } + + void displayJobList(JobStatus[] jobs) { + System.out.printf("JobId\tState\tStartTime\tUserName\n"); + for (JobStatus job : jobs) { + System.out.printf("%s\t%d\t%d\t%s\n", job.getJobID(), job.getRunState(), + job.getStartTime(), job.getUsername()); + } + } + + /** + * Display the list of active groom servers + */ + private void listActiveGrooms() throws IOException { + ClusterStatus c = jobSubmitClient.getClusterStatus(true); + Map grooms = c.getActiveGroomNames(); + for (String groomName : grooms.keySet()) { + System.out.println(groomName); + } + } + + /** + */ + public static void main(String[] args) throws Exception { + int res = ToolRunner.run(new BSPJobClient(), args); + System.exit(res); + } +} Index: core/src/main/java/org/apache/hama/bsp/TaskAttemptID.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskAttemptID.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskAttemptID.java (revision 0) @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * TaskAttemptID is a unique identifier for a task attempt. + */ +public class TaskAttemptID extends ID { + protected static final String ATTEMPT = "attempt"; + private TaskID taskId; + + public TaskAttemptID(TaskID taskId, int id) { + super(id); + if (taskId == null) { + throw new IllegalArgumentException("taskId cannot be null"); + } + this.taskId = taskId; + } + + public TaskAttemptID(String jtIdentifier, int jobId, int taskId, int id) { + this(new TaskID(jtIdentifier, jobId, taskId), id); + } + + public TaskAttemptID() { + taskId = new TaskID(); + } + + public BSPJobID getJobID() { + return taskId.getJobID(); + } + + public TaskID getTaskID() { + return taskId; + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) + return false; + + TaskAttemptID that = (TaskAttemptID) o; + return this.taskId.equals(that.taskId); + } + + protected StringBuilder appendTo(StringBuilder builder) { + return taskId.appendTo(builder).append(SEPARATOR).append(id); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + taskId.readFields(in); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + taskId.write(out); + } + + @Override + public int hashCode() { + return taskId.hashCode() * 5 + id; + } + + @Override + public int compareTo(ID o) { + TaskAttemptID that = (TaskAttemptID) o; + int tipComp = this.taskId.compareTo(that.taskId); + if (tipComp == 0) { + return this.id - that.id; + } else + return tipComp; + } + + @Override + public String toString() { + return appendTo(new StringBuilder(ATTEMPT)).toString(); + } + + public static TaskAttemptID forName(String str) + throws IllegalArgumentException { + if (str == null) + return null; + try { + String[] parts = str.split(Character.toString(SEPARATOR)); + if (parts.length == 5) { + if (parts[0].equals(ATTEMPT)) { + return new TaskAttemptID(parts[1], Integer.parseInt(parts[2]), + Integer.parseInt(parts[3]), Integer.parseInt(parts[4])); + } + } + } catch (Exception ex) { + // fall below + } + throw new IllegalArgumentException("TaskAttemptId string : " + str + + " is not properly formed"); + } +} Index: core/src/main/java/org/apache/hama/bsp/package.html =================================================================== --- core/src/main/java/org/apache/hama/bsp/package.html (revision 0) +++ core/src/main/java/org/apache/hama/bsp/package.html (revision 0) @@ -0,0 +1,23 @@ + + + + + +BSP computing framework. + + Index: core/src/main/java/org/apache/hama/bsp/BSPJob.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPJob.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPJob.java (revision 0) @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Enumeration; + +import org.apache.hadoop.fs.Path; +import org.apache.hama.HamaConfiguration; + +/** + * A BSP job configuration. + * + * BSPJob is the primary interface for a user to describe a BSP job to the Hama + * BSP framework for execution. + */ +public class BSPJob extends BSPJobContext { + public static enum JobState { + DEFINE, RUNNING + }; + + private JobState state = JobState.DEFINE; + private BSPJobClient jobClient; + private RunningJob info; + + public BSPJob() throws IOException { + this(new HamaConfiguration()); + } + + public BSPJob(HamaConfiguration conf) throws IOException { + super(conf, null); + jobClient = new BSPJobClient(conf); + } + + public BSPJob(HamaConfiguration conf, String jobName) throws IOException { + this(conf); + setJobName(jobName); + } + + public BSPJob(BSPJobID jobID, String jobFile) throws IOException { + super(new Path(jobFile), jobID); + } + + public BSPJob(HamaConfiguration conf, Class exampleClass) + throws IOException { + this(conf); + setJarByClass(exampleClass); + } + + public BSPJob(HamaConfiguration conf, int numPeer) { + super(conf, null); + this.setNumBspTask(numPeer); + } + + private void ensureState(JobState state) throws IllegalStateException { + if (state != this.state) { + throw new IllegalStateException("Job in state " + this.state + + " instead of " + state); + } + } + + // ///////////////////////////////////// + // Setter for Job Submission + // ///////////////////////////////////// + public void setWorkingDirectory(Path dir) throws IOException { + ensureState(JobState.DEFINE); + dir = new Path(getWorkingDirectory(), dir); + conf.set(WORKING_DIR, dir.toString()); + } + + /** + * Set the BSP algorithm class for the job. + * + * @param cls + * @throws IllegalStateException + */ + public void setBspClass(Class cls) + throws IllegalStateException { + ensureState(JobState.DEFINE); + conf.setClass(WORK_CLASS_ATTR, cls, BSP.class); + } + + @SuppressWarnings("unchecked") + public Class getBspClass() { + return (Class) conf.getClass(WORK_CLASS_ATTR, BSP.class); + } + + public void setJar(String jar) { + conf.set("bsp.jar", jar); + } + + public void setJarByClass(Class cls) { + String jar = findContainingJar(cls); + if (jar != null) { + conf.set("bsp.jar", jar); + } + } + + private static String findContainingJar(Class my_class) { + ClassLoader loader = my_class.getClassLoader(); + String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; + try { + for (Enumeration itr = loader.getResources(class_file); itr + .hasMoreElements();) { + + URL url = itr.nextElement(); + if ("jar".equals(url.getProtocol())) { + String toReturn = url.getPath(); + if (toReturn.startsWith("file:")) { + toReturn = toReturn.substring("file:".length()); + } + toReturn = URLDecoder.decode(toReturn, "UTF-8"); + return toReturn.replaceAll("!.*$", ""); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + } + + public void setJobName(String name) throws IllegalStateException { + ensureState(JobState.DEFINE); + conf.set("bsp.job.name", name); + } + + public void setInputPath(HamaConfiguration conf, Path iNPUTPATH) { + + } + + public void setUser(String user) { + conf.set("user.name", user); + } + + // ///////////////////////////////////// + // Methods for Job Control + // ///////////////////////////////////// + public long progress() throws IOException { + ensureState(JobState.RUNNING); + return info.progress(); + } + + public boolean isComplete() throws IOException { + ensureState(JobState.RUNNING); + return info.isComplete(); + } + + public boolean isSuccessful() throws IOException { + ensureState(JobState.RUNNING); + return info.isSuccessful(); + } + + public void killJob() throws IOException { + ensureState(JobState.RUNNING); + info.killJob(); + } + + public void killTask(TaskAttemptID taskId) throws IOException { + ensureState(JobState.RUNNING); + info.killTask(taskId, false); + } + + public void failTask(TaskAttemptID taskId) throws IOException { + ensureState(JobState.RUNNING); + info.killTask(taskId, true); + } + + public void submit() throws IOException, InterruptedException { + ensureState(JobState.DEFINE); + info = jobClient.submitJobInternal(this); + state = JobState.RUNNING; + } + + public boolean waitForCompletion(boolean verbose) throws IOException, + InterruptedException, ClassNotFoundException { + if (state == JobState.DEFINE) { + submit(); + } + if (verbose) { + jobClient.monitorAndPrintJob(this, info); + } else { + info.waitForCompletion(); + } + return isSuccessful(); + } + + public void set(String name, String value) { + conf.set(name, value); + } + + public void setNumBspTask(int tasks) { + conf.setInt("bsp.peers.num", tasks); + } + + public int getNumBspTask() { + return conf.getInt("bsp.peers.num", 0); + } +} Index: core/src/main/java/org/apache/hama/bsp/PeerNames.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/PeerNames.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/PeerNames.java (revision 0) @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; + +/** + * + */ +public class PeerNames implements Writable { + Collection allPeers; + + public PeerNames() { + this.allPeers = new ArrayList(); + } + + public PeerNames(Collection allPeers) { + this.allPeers = allPeers; + } + + public Collection getAllPeerNames() { + return allPeers; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(allPeers.size()); + for (String peerName : allPeers) { + Text.writeString(out, peerName); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + int peersNum = in.readInt(); + for (int i = 0; i < peersNum; i++) { + allPeers.add(Text.readString(in)); + } + } + +} Index: core/src/main/java/org/apache/hama/bsp/TaskID.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskID.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskID.java (revision 0) @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.text.NumberFormat; + +/** + * TaskID represents the immutable and unique identifier for a BSP Task. + */ +public class TaskID extends ID { + protected static final String TASK = "task"; + protected static final NumberFormat idFormat = NumberFormat.getInstance(); + static { + idFormat.setGroupingUsed(false); + idFormat.setMinimumIntegerDigits(6); + } + + private BSPJobID jobId; + + public TaskID(BSPJobID jobId, int id) { + super(id); + if (jobId == null) { + throw new IllegalArgumentException("jobId cannot be null"); + } + this.jobId = jobId; + } + + public TaskID(String jtIdentifier, int jobId, int id) { + this(new BSPJobID(jtIdentifier, jobId), id); + } + + public TaskID() { + jobId = new BSPJobID(); + } + + /** Returns the {@link BSPJobID} object that this tip belongs to */ + public BSPJobID getJobID() { + return jobId; + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) + return false; + + TaskID that = (TaskID) o; + return this.jobId.equals(that.jobId); + } + + @Override + public int compareTo(ID o) { + TaskID that = (TaskID) o; + int jobComp = this.jobId.compareTo(that.jobId); + if (jobComp == 0) { + return this.id - that.id; + } else { + return jobComp; + } + } + + @Override + public String toString() { + return appendTo(new StringBuilder(TASK)).toString(); + } + + protected StringBuilder appendTo(StringBuilder builder) { + return jobId.appendTo(builder).append(SEPARATOR) + .append(idFormat.format(id)); + } + + @Override + public int hashCode() { + return jobId.hashCode() * 524287 + id; + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + jobId.readFields(in); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + jobId.write(out); + } + + public static TaskID forName(String str) throws IllegalArgumentException { + if (str == null) + return null; + try { + String[] parts = str.split("_"); + if (parts.length == 5) { + if (parts[0].equals(TASK)) { + return new TaskID(parts[1], Integer.parseInt(parts[2]), Integer + .parseInt(parts[4])); + } + } + } catch (Exception ex) { + } + throw new IllegalArgumentException("TaskId string : " + str + + " is not properly formed"); + } +} Index: core/src/main/java/org/apache/hama/bsp/BSPPeerInterface.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPPeerInterface.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPPeerInterface.java (revision 0) @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.hama.Constants; +import org.apache.hama.ipc.HamaRPCProtocolVersion; +import org.apache.zookeeper.KeeperException; + +/** + * BSP communication interface. + */ +public interface BSPPeerInterface extends HamaRPCProtocolVersion, Closeable, + Constants { + + /** + * Send a data with a tag to another BSPSlave corresponding to hostname. + * Messages sent by this method are not guaranteed to be received in a sent + * order. + * + * @param peerName + * @param msg + * @throws IOException + */ + public void send(String peerName, BSPMessage msg) throws IOException; + + /** + * Puts a message to local queue. + * + * @param msg + * @throws IOException + */ + public void put(BSPMessage msg) throws IOException; + + /** + * Puts a bundle of messages to local queue. + * + * @param messages + * @throws IOException + */ + public void put(BSPMessageBundle messages) throws IOException; + + /** + * @return A message from the peer's received messages queue (a FIFO). + * @throws IOException + */ + public BSPMessage getCurrentMessage() throws IOException; + + /** + * @return The number of messages in the peer's received messages queue. + */ + public int getNumCurrentMessages(); + + /** + * Barrier Synchronization. + * + * Sends all the messages in the outgoing message queues to the corresponding + * remote peers. + * + * @throws InterruptedException + * @throws KeeperException + */ + public void sync() throws IOException, KeeperException, InterruptedException; + + /** + * @return the count of current super-step + */ + public long getSuperstepCount(); + + /** + * @return The name of this peer in the format "hostname:port". + */ + public String getPeerName(); + + /** + * @return The names of all the peers executing tasks from the same job + * (including this peer). + */ + public String[] getAllPeerNames(); + + /** + * Clears all queues entries. + */ + public void clear(); +} Index: core/src/main/java/org/apache/hama/bsp/JobChangeEvent.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/JobChangeEvent.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/JobChangeEvent.java (revision 0) @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +/** + * {@link JobChangeEvent} is used to capture state changes in a job. A job can + * change its state w.r.t priority, progress, run-state etc. + */ +abstract class JobChangeEvent { + private JobInProgress jip; + + JobChangeEvent(JobInProgress jip) { + this.jip = jip; + } + + /** + * Get the job object for which the change is reported + */ + JobInProgress getJobInProgress() { + return jip; + } +} Index: core/src/main/java/org/apache/hama/bsp/BSPMessageBundle.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPMessageBundle.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPMessageBundle.java (revision 0) @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * BSPMessageBundle stores a group of BSPMessages so that they can be sent in + * batch rather than individually. + * + */ +public class BSPMessageBundle implements Writable { + + public static final Log LOG = LogFactory.getLog(BSPMessageBundle.class); + + private HashMap> messages = new HashMap>(); + private HashMap> classCache = new HashMap>(); + + public BSPMessageBundle() { + } + + /** + * Add message to this bundle. + * + * @param message BSPMessage to add. + */ + public void addMessage(BSPMessage message) { + String className = message.getClass().getName(); + if (!messages.containsKey(className)) { + // use linked list because we're just iterating over them + LinkedList list = new LinkedList(); + list.add(message); + messages.put(className, list); + } else { + messages.get(className).add(message); + } + } + + public List getMessages() { + // here we use an arraylist, because we know the size and outside may need + // random access + List mergeList = new ArrayList(messages.size()); + for (LinkedList c : messages.values()) { + mergeList.addAll(c); + } + return mergeList; + } + + public void write(DataOutput out) throws IOException { + // writes the k/v mapping size + out.writeInt(messages.size()); + if (messages.size() > 0) { + for (Entry> entry : messages.entrySet()) { + out.writeUTF(entry.getKey()); + LinkedList messageList = entry.getValue(); + out.writeInt(messageList.size()); + for (BSPMessage msg : messageList) { + msg.write(out); + } + } + } + } + + @SuppressWarnings("unchecked") + public void readFields(DataInput in) throws IOException { + if (messages == null) { + messages = new HashMap>(); + } + int numMessages = in.readInt(); + if (numMessages > 0) { + for (int entries = 0; entries < numMessages; entries++) { + String className = in.readUTF(); + int size = in.readInt(); + LinkedList msgList = new LinkedList(); + messages.put(className, msgList); + + Class clazz = null; + if ((clazz = classCache.get(className)) == null) { + try { + clazz = (Class) Class.forName(className); + classCache.put(className, clazz); + } catch (ClassNotFoundException e) { + LOG.error("Class was not found.",e); + } + } + + for (int i = 0; i < size; i++) { + BSPMessage msg = ReflectionUtils.newInstance(clazz, null); + msg.readFields(in); + msgList.add(msg); + } + + } + } + } + +} Index: core/src/main/java/org/apache/hama/bsp/BSP.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSP.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSP.java (revision 0) @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +/** + * This class provides an abstract implementation of the BSP interface. + */ +public abstract class BSP implements BSPInterface { +} Index: core/src/main/java/org/apache/hama/bsp/KillJobAction.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/KillJobAction.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/KillJobAction.java (revision 0) @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Text; + +/** + * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the + * {@link org.apache.hama.bsp.GroomServer} to kill the task of a job and cleanup + * resources. + */ +class KillJobAction extends GroomServerAction { + String jobId; + + public KillJobAction() { + super(ActionType.KILL_JOB); + jobId = new String(); + } + + public KillJobAction(String killJobId) { + super(ActionType.KILL_JOB); + this.jobId = killJobId; + } + + public String getJobID() { + return jobId; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, jobId); + } + + @Override + public void readFields(DataInput in) throws IOException { + jobId = Text.readString(in); + } + +} Index: core/src/main/java/org/apache/hama/bsp/Queue.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/Queue.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/Queue.java (revision 0) @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.util.Collection; + +/** + * Job Queue interface. + * + * @param + */ +public interface Queue{ + + /** + * The queue name. + * @return the name of current queue. + */ + String getName(); + + /** + * Add a job to a queue. + * @param job to be added to the queue. + */ + void addJob(T job); + + /** + * Remove a job from the queue. + * @param job to be removed from the queue. + */ + void removeJob(T job); + + /** + * Get a job + * @return job that is removed from the queue. + */ + T removeJob(); + + /** + * Return all data stored in this queue. + * @return Collection of jobs. + */ + public Collection jobs(); + +} Index: core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java (revision 0) @@ -0,0 +1,356 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hama.bsp.BSPMaster.State; +import org.apache.hama.ipc.JobSubmissionProtocol; +import org.apache.zookeeper.KeeperException; + +/** + * A multithreaded local BSP runner that can be used for debugging BSP's. It + * uses the working directory "/user/hama/bsp/" and starts runners based on the + * number of the machines core. + * + */ +public class LocalBSPRunner implements JobSubmissionProtocol { + public static final Log LOG = LogFactory.getLog(LocalBSPRunner.class); + + private static final String IDENTIFIER = "localrunner"; + private static String WORKING_DIR = "/user/hama/bsp/"; + protected static volatile ThreadPoolExecutor threadPool; + protected static int threadPoolSize; + protected static final LinkedList> futureList = new LinkedList>(); + protected static CyclicBarrier barrier; + + static { + threadPoolSize = Runtime.getRuntime().availableProcessors(); + barrier = new CyclicBarrier(threadPoolSize); + threadPool = (ThreadPoolExecutor) Executors + .newFixedThreadPool(threadPoolSize); + } + + protected HashMap localGrooms = new HashMap(); + protected String jobFile; + protected String jobName; + + protected JobStatus currentJobStatus; + + protected Configuration conf; + protected FileSystem fs; + + public LocalBSPRunner(Configuration conf) throws IOException { + super(); + this.conf = conf; + this.fs = FileSystem.get(conf); + String path = conf.get("bsp.local.dir"); + if (path != null && !path.isEmpty()) + WORKING_DIR = path; + + threadPoolSize = conf.getInt("bsp.local.tasks.maximum", 20); + threadPool = (ThreadPoolExecutor) Executors + .newFixedThreadPool(threadPoolSize); + barrier = new CyclicBarrier(threadPoolSize); + + for (int i = 0; i < threadPoolSize; i++) { + String name = IDENTIFIER + " " + i; + localGrooms.put(name, new LocalGroom(name)); + } + + } + + @Override + public long getProtocolVersion(String protocol, long clientVersion) + throws IOException { + return 3; + } + + @Override + public BSPJobID getNewJobId() throws IOException { + return new BSPJobID(IDENTIFIER, 1); + } + + @Override + public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException { + this.jobFile = jobFile; + BSPJob job = new BSPJob(jobID, jobFile); + job.setNumBspTask(threadPoolSize); + this.jobName = job.getJobName(); + currentJobStatus = new JobStatus(jobID, System.getProperty("user.name"), 0, + JobStatus.RUNNING); + for (int i = 0; i < threadPoolSize; i++) { + String name = IDENTIFIER + " " + i; + LocalGroom localGroom = new LocalGroom(name); + localGrooms.put(name, localGroom); + futureList.add(threadPool.submit(new BSPRunner(conf, job, ReflectionUtils + .newInstance(job.getBspClass(), conf), localGroom))); + } + new Thread(new ThreadObserver(currentJobStatus)).start(); + return currentJobStatus; + } + + @Override + public ClusterStatus getClusterStatus(boolean detailed) throws IOException { + Map map = new HashMap(); + for (Entry entry : localGrooms.entrySet()) { + map.put(entry.getKey(), entry.getValue().getPeerName()); + } + return new ClusterStatus(map, threadPoolSize, threadPoolSize, State.RUNNING); + } + + @Override + public JobProfile getJobProfile(BSPJobID jobid) throws IOException { + return new JobProfile(System.getProperty("user.name"), jobid, jobFile, + jobName); + } + + @Override + public JobStatus getJobStatus(BSPJobID jobid) throws IOException { + if (currentJobStatus == null) { + currentJobStatus = new JobStatus(jobid, System.getProperty("user.name"), + 0L, JobStatus.RUNNING); + } + return currentJobStatus; + } + + @Override + public String getFilesystemName() throws IOException { + return fs.getUri().toString(); + } + + @Override + public JobStatus[] jobsToComplete() throws IOException { + return null; + } + + @Override + public JobStatus[] getAllJobs() throws IOException { + return null; + } + + @Override + public String getSystemDir() { + return WORKING_DIR; + } + + @Override + public void killJob(BSPJobID jobid) throws IOException { + return; + } + + @Override + public boolean killTask(TaskAttemptID taskId, boolean shouldFail) + throws IOException { + return false; + } + + // this class will spawn a new thread and executes the BSP + class BSPRunner implements Callable { + + Configuration conf; + BSPJob job; + BSP bsp; + LocalGroom groom; + + public BSPRunner(Configuration conf, BSPJob job, BSP bsp, LocalGroom groom) { + super(); + this.conf = conf; + this.job = job; + this.bsp = bsp; + this.groom = groom; + } + + public void run() { + bsp.setConf(conf); + try { + bsp.bsp(groom); + } catch (Exception e) { + LOG.error("Exception during BSP execution!", e); + } + } + + @Override + public BSP call() throws Exception { + run(); + return bsp; + } + } + + // this thread observes the status of the runners. + class ThreadObserver implements Runnable { + + JobStatus status; + + public ThreadObserver(JobStatus currentJobStatus) { + this.status = currentJobStatus; + } + + @Override + public void run() { + boolean success = true; + for (Future future : futureList) { + try { + future.get(); + } catch (InterruptedException e) { + LOG.error("Exception during BSP execution!", e); + success = false; + } catch (ExecutionException e) { + LOG.error("Exception during BSP execution!", e); + success = false; + } + } + if (success) { + currentJobStatus.setState(JobStatus.State.SUCCEEDED); + currentJobStatus.setRunState(JobStatus.SUCCEEDED); + } else { + currentJobStatus.setState(JobStatus.State.FAILED); + currentJobStatus.setRunState(JobStatus.FAILED); + } + threadPool.shutdownNow(); + } + + } + + class LocalGroom extends BSPPeer { + private long superStepCount = 0; + private final ConcurrentLinkedQueue localMessageQueue = new ConcurrentLinkedQueue(); + // outgoing queue + private final Map> outgoingQueues = new ConcurrentHashMap>(); + private final String peerName; + + public LocalGroom(String peerName) throws IOException { + this.peerName = peerName; + } + + @Override + public void send(String peerName, BSPMessage msg) throws IOException { + if (this.peerName.equals(peerName)) { + put(msg); + } else { + // put this into a outgoing queue + if (outgoingQueues.get(peerName) == null) { + outgoingQueues.put(peerName, new ConcurrentLinkedQueue()); + } + outgoingQueues.get(peerName).add(msg); + } + } + + @Override + public void put(BSPMessage msg) throws IOException { + localMessageQueue.add(msg); + } + + @Override + public BSPMessage getCurrentMessage() throws IOException { + return localMessageQueue.poll(); + } + + @Override + public int getNumCurrentMessages() { + return localMessageQueue.size(); + } + + @Override + public void sync() throws IOException, KeeperException, + InterruptedException { + // wait until all threads reach this barrier + barrierSync(); + // send the messages + for (Entry> entry : outgoingQueues + .entrySet()) { + String peerName = entry.getKey(); + for (BSPMessage msg : entry.getValue()) + localGrooms.get(peerName).put(msg); + } + // clear the local outgoing queue + outgoingQueues.clear(); + // sync again to avoid data inconsistency + barrierSync(); + incrementSuperSteps(); + } + + private void barrierSync() throws InterruptedException { + try { + barrier.await(); + } catch (BrokenBarrierException e) { + throw new InterruptedException("Barrier has been broken!" + e); + } + } + + private void incrementSuperSteps() { + currentJobStatus.setprogress(superStepCount++); + currentJobStatus.setSuperstepCount(currentJobStatus.progress()); + } + + @Override + public long getSuperstepCount() { + return superStepCount; + } + + @Override + public String getPeerName() { + return peerName; + } + + @Override + public String[] getAllPeerNames() { + return localGrooms.keySet().toArray( + new String[localGrooms.keySet().size()]); + } + + @Override + public void clear() { + localMessageQueue.clear(); + } + + @Override + public long getProtocolVersion(String protocol, long clientVersion) + throws IOException { + return 3; + } + + @Override + public void close() throws IOException { + + } + + @Override + public void put(BSPMessageBundle messages) throws IOException { + } + + } +} Index: core/src/main/java/org/apache/hama/bsp/JobProfile.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/JobProfile.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/JobProfile.java (revision 0) @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableFactories; +import org.apache.hadoop.io.WritableFactory; + +/** + * A JobProfile tracks job's status. + */ +public class JobProfile implements Writable { + + static { // register actor + WritableFactories.setFactory(JobProfile.class, new WritableFactory() { + public Writable newInstance() { + return new JobProfile(); + } + }); + } + + String user; + final BSPJobID jobid; + String jobFile; + String name; + + /** + * Construct an empty {@link JobProfile}. + */ + public JobProfile() { + jobid = new BSPJobID(); + } + + /** + * Construct a {@link JobProfile} the userid, jobid, job config-file, + * job-details url and job name. + * + * @param user userid of the person who submitted the job. + * @param jobid id of the job. + * @param jobFile job configuration file. + * @param name user-specified job name. + */ + public JobProfile(String user, BSPJobID jobid, String jobFile, String name) { + this.user = user; + this.jobid = jobid; + this.jobFile = jobFile; + this.name = name; + } + + /** + * Get the user id. + */ + public String getUser() { + return user; + } + + /** + * Get the job id. + */ + public BSPJobID getJobID() { + return jobid; + } + + /** + * Get the configuration file for the job. + */ + public String getJobFile() { + return jobFile; + } + + /** + * Get the user-specified job name. + */ + public String getJobName() { + return name; + } + + public void write(DataOutput out) throws IOException { + jobid.write(out); + Text.writeString(out, jobFile); + Text.writeString(out, user); + Text.writeString(out, name); + } + + public void readFields(DataInput in) throws IOException { + jobid.readFields(in); + this.jobFile = Text.readString(in); + this.user = Text.readString(in); + this.name = Text.readString(in); + } +} Index: core/src/main/java/org/apache/hama/bsp/LaunchTaskAction.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/LaunchTaskAction.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/LaunchTaskAction.java (revision 0) @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the + * {@link org.apache.hama.bsp.GroomServer} to launch a new task. + */ +class LaunchTaskAction extends GroomServerAction { + private Task task; + + public LaunchTaskAction() { + super(ActionType.LAUNCH_TASK); + } + + public LaunchTaskAction(Task task) { + super(ActionType.LAUNCH_TASK); + this.task = task; + } + + public Task getTask() { + return task; + } + + public void write(DataOutput out) throws IOException { + task.write(out); + } + + public void readFields(DataInput in) throws IOException { + task = new BSPTask(); + task.readFields(in); + } + +} Index: core/src/main/java/org/apache/hama/bsp/BSPJobID.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/BSPJobID.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/BSPJobID.java (revision 0) @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; + +import java.io.DataOutput; +import java.io.IOException; +import java.text.NumberFormat; + +import org.apache.hadoop.io.Text; + +/** + * BSPJobID represents the immutable and unique identifier for the job. + */ +public class BSPJobID extends ID implements Comparable { + protected static final String JOB = "job"; + private final Text jtIdentifier; + + protected static final NumberFormat idFormat = NumberFormat.getInstance(); + static { + idFormat.setGroupingUsed(false); + idFormat.setMinimumIntegerDigits(4); + } + + public BSPJobID(String jtIdentifier, int id) { + super(id); + this.jtIdentifier = new Text(jtIdentifier); + } + + public BSPJobID() { + jtIdentifier = new Text(); + } + + public String getJtIdentifier() { + return jtIdentifier.toString(); + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) + return false; + + BSPJobID that = (BSPJobID) o; + return this.jtIdentifier.equals(that.jtIdentifier); + } + + @Override + public int compareTo(ID o) { + BSPJobID that = (BSPJobID) o; + int jtComp = this.jtIdentifier.compareTo(that.jtIdentifier); + if (jtComp == 0) { + return this.id - that.id; + } else + return jtComp; + } + + public StringBuilder appendTo(StringBuilder builder) { + builder.append(SEPARATOR); + builder.append(jtIdentifier); + builder.append(SEPARATOR); + builder.append(idFormat.format(id)); + return builder; + } + + @Override + public int hashCode() { + return jtIdentifier.hashCode() + id; + } + + @Override + public String toString() { + return appendTo(new StringBuilder(JOB)).toString(); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + this.jtIdentifier.readFields(in); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + jtIdentifier.write(out); + } + + public static BSPJobID forName(String str) throws IllegalArgumentException { + if (str == null) + return null; + try { + String[] parts = str.split("_"); + if (parts.length == 3) { + if (parts[0].equals(JOB)) { + return new BSPJobID(parts[1], Integer.parseInt(parts[2])); + } + } + } catch (Exception ex) { + } + throw new IllegalArgumentException("JobId string : " + str + + " is not properly formed"); + } +} Index: core/src/main/java/org/apache/hama/bsp/JobInProgressListener.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/JobInProgressListener.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/JobInProgressListener.java (revision 0) @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.IOException; + +/** + * A listener for changes in a {@link JobInProgress job}'s lifecycle in the + * {@link BSPMaster}. + */ +abstract class JobInProgressListener { + + /** + * Invoked when a new job has been added to the {@link BSPMaster}. + * @param job The job to be added. + * @throws IOException + */ + public abstract void jobAdded(JobInProgress job) throws IOException; + + /** + * Invoked when a job has been removed from the {@link BSPMaster}. + * @param job The job to be removed . + * @throws IOException + */ + public abstract void jobRemoved(JobInProgress job) throws IOException; + +} Index: core/src/main/java/org/apache/hama/bsp/ReinitGroomAction.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/ReinitGroomAction.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/ReinitGroomAction.java (revision 0) @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Represents a directive from the {@link org.apache.hama.bsp.BSPMaster} to the + * {@link org.apache.hama.bsp.GroomServer} to reinitialize itself. + */ +class ReinitGroomAction extends GroomServerAction { + + public ReinitGroomAction() { + super(ActionType.REINIT_GROOM); + } + + public void write(DataOutput out) throws IOException { + } + + public void readFields(DataInput in) throws IOException { + } + +} Index: core/src/main/java/org/apache/hama/bsp/TaskRunner.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/TaskRunner.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/TaskRunner.java (revision 0) @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetSocketAddress; +import java.util.Vector; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.RunJar; + +/** + * Base class that runs a task in a separate process. + */ +public class TaskRunner extends Thread { + + public static final Log LOG = LogFactory.getLog(TaskRunner.class); + + boolean killed = false; + private Process process; + private Task task; + private BSPJob conf; + private GroomServer groomServer; + + public TaskRunner(BSPTask bspTask, GroomServer groom, BSPJob conf) { + this.task = bspTask; + this.conf = conf; + this.groomServer = groom; + } + + public Task getTask() { + return task; + } + + /** + * Called to assemble this task's input. This method is run in the parent + * process before the child is spawned. It should not execute user code, only + * system code. + */ + public boolean prepare() throws IOException { + return true; + } + + public void run() { + try { + String sep = System.getProperty("path.separator"); + File workDir = new File(new File(task.getJobFile()).getParent(), "work"); + boolean isCreated = workDir.mkdirs(); + if(!isCreated) { + LOG.debug("TaskRunner.workDir : " + workDir); + } + + StringBuffer classPath = new StringBuffer(); + // start with same classpath as parent process + classPath.append(System.getProperty("java.class.path")); + classPath.append(sep); + + String jar = conf.getJar(); + if (jar != null) { // if jar exists, it into workDir + RunJar.unJar(new File(jar), workDir); + File[] libs = new File(workDir, "lib").listFiles(); + if (libs != null) { + for (int i = 0; i < libs.length; i++) { + classPath.append(sep); // add libs from jar to classpath + classPath.append(libs[i]); + } + } + classPath.append(sep); + classPath.append(new File(workDir, "classes")); + classPath.append(sep); + classPath.append(workDir); + } + + // Build exec child jmv args. + Vector vargs = new Vector(); + File jvm = // use same jvm as parent + new File(new File(System.getProperty("java.home"), "bin"), "java"); + vargs.add(jvm.toString()); + + // bsp.child.java.opts + String javaOpts = conf.getConf().get("bsp.child.java.opts", "-Xmx200m"); + javaOpts = javaOpts.replace("@taskid@", task.getTaskID().toString()); + + String[] javaOptsSplit = javaOpts.split(" "); + for (int i = 0; i < javaOptsSplit.length; i++) { + vargs.add(javaOptsSplit[i]); + } + + // Add classpath. + vargs.add("-classpath"); + vargs.add(classPath.toString()); + // Add main class and its arguments + vargs.add(GroomServer.Child.class.getName()); // main of Child + + InetSocketAddress addr = groomServer.getTaskTrackerReportAddress(); + vargs.add(addr.getHostName()); + vargs.add(Integer.toString(addr.getPort())); + vargs.add(task.getTaskID().toString()); + vargs.add(groomServer.groomHostName); + + // Run java + runChild((String[]) vargs.toArray(new String[0]), workDir); + } catch (IOException e) { + LOG.error(e); + } + } + + /** + * Run the child process + */ + private void runChild(String[] args, File dir) throws IOException { + this.process = Runtime.getRuntime().exec(args, null, dir); + try { + new Thread() { + public void run() { + logStream(process.getErrorStream()); // copy log output + } + }.start(); + + logStream(process.getInputStream()); // normally empty + + int exit_code = process.waitFor(); + if (!killed && exit_code != 0) { + throw new IOException("Task process exit with nonzero status of " + + exit_code + "."); + } + + } catch (InterruptedException e) { + throw new IOException(e.toString()); + } finally { + kill(); + } + } + + /** + * Kill the child process + */ + public void kill() { + if (process != null) { + process.destroy(); + } + killed = true; + } + + /** + */ + private void logStream(InputStream output) { + try { + BufferedReader in = new BufferedReader(new InputStreamReader(output)); + String line; + while ((line = in.readLine()) != null) { + LOG.info(task.getTaskID() + " " + line); + } + } catch (IOException e) { + LOG.warn(task.getTaskID() + " Error reading child output", e); + } finally { + try { + output.close(); + } catch (IOException e) { + LOG.warn(task.getTaskID() + " Error closing child output", e); + } + } + } + +} Index: core/src/main/java/org/apache/hama/bsp/DirectiveException.java =================================================================== --- core/src/main/java/org/apache/hama/bsp/DirectiveException.java (revision 0) +++ core/src/main/java/org/apache/hama/bsp/DirectiveException.java (revision 0) @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.bsp; + +/** + * A custom exception class for Directive. + */ +public class DirectiveException extends RuntimeException{ + private static final long serialVersionUID = -8052582046894492822L; + + public DirectiveException(){ + super(); + } + + public DirectiveException(String message){ + super(message); + } + + public DirectiveException(String message, Throwable t){ + super(message, t); + } +} Index: core/src/main/java/org/apache/hama/BSPMasterRunner.java =================================================================== --- core/src/main/java/org/apache/hama/BSPMasterRunner.java (revision 0) +++ core/src/main/java/org/apache/hama/BSPMasterRunner.java (revision 0) @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hama.bsp.BSPMaster; + +/** + * This class starts and runs the BSPMaster. + */ +public class BSPMasterRunner extends Configured implements Tool { + + public static final Log LOG = LogFactory.getLog(BSPMasterRunner.class); + + @Override + public int run(String[] args) throws Exception { + StringUtils.startupShutdownMessage(BSPMaster.class, args, LOG); + + if (args.length != 0) { + System.out.println("usage: BSPMasterRunner"); + System.exit(-1); + } + + try { + HamaConfiguration conf = new HamaConfiguration(getConf()); + BSPMaster master = BSPMaster.startMaster(conf); + master.offerService(); + } catch (Throwable e) { + LOG.fatal(StringUtils.stringifyException(e)); + return -1; + } + return 0; + } + + public static void main(String[] args) throws Exception { + int exitCode = ToolRunner.run(new BSPMasterRunner(), args); + System.exit(exitCode); + } + +} Index: core/src/main/java/org/apache/hama/ipc/BSPPeerProtocol.java =================================================================== --- core/src/main/java/org/apache/hama/ipc/BSPPeerProtocol.java (revision 0) +++ core/src/main/java/org/apache/hama/ipc/BSPPeerProtocol.java (revision 0) @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ipc; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.hama.Constants; +import org.apache.hama.bsp.PeerNames; +import org.apache.hama.bsp.Task; +import org.apache.hama.bsp.TaskAttemptID; + +/** + * Protocol that task child process uses to contact its parent process. + */ +public interface BSPPeerProtocol extends HamaRPCProtocolVersion, Closeable, + Constants { + + /** Called when a child task process starts, to get its task. */ + Task getTask(TaskAttemptID taskid) throws IOException; + + /** + * Periodically called by child to check if parent is still alive. + * + * @return True if the task is known + */ + boolean ping(TaskAttemptID taskid) throws IOException; + + /** + * Report that the task is successfully completed. Failure is assumed if the + * task process exits without calling this. + * + * @param taskid task's id + * @param shouldBePromoted whether to promote the task's output or not + */ + void done(TaskAttemptID taskid, boolean shouldBePromoted) throws IOException; + + /** Report that the task encounted a local filesystem error. */ + void fsError(TaskAttemptID taskId, String message) throws IOException; + + void incrementSuperstepCount(TaskAttemptID taskid) throws IOException; + + /** + * @return the all BSPPeer names. + */ + PeerNames getAllPeerNames(); + +} Index: core/src/main/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java =================================================================== --- core/src/main/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java (revision 0) +++ core/src/main/java/org/apache/hama/ipc/HamaRPCProtocolVersion.java (revision 0) @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ipc; + +import org.apache.hadoop.ipc.VersionedProtocol; + +/** + * There is one version id for all the RPC interfaces. If any interface is + * changed, the versionID must be changed here. + */ +public interface HamaRPCProtocolVersion extends VersionedProtocol { + public static final long versionID = 1L; +} Index: core/src/main/java/org/apache/hama/ipc/GroomProtocol.java =================================================================== --- core/src/main/java/org/apache/hama/ipc/GroomProtocol.java (revision 0) +++ core/src/main/java/org/apache/hama/ipc/GroomProtocol.java (revision 0) @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ipc; + +import java.io.IOException; + +import org.apache.hama.bsp.Directive; + +/** + * A protocol for BSPMaster talks to GroomServer. + */ +public interface GroomProtocol extends HamaRPCProtocolVersion { + + /** + * Instruct GroomServer performaning tasks. + * + * @param directive instructs a GroomServer performing necessary + * execution. + * @throws IOException + */ + void dispatch(Directive directive) throws IOException; + +} Index: core/src/main/java/org/apache/hama/ipc/JobSubmissionProtocol.java =================================================================== --- core/src/main/java/org/apache/hama/ipc/JobSubmissionProtocol.java (revision 0) +++ core/src/main/java/org/apache/hama/ipc/JobSubmissionProtocol.java (revision 0) @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ipc; + +import java.io.IOException; + +import org.apache.hama.bsp.ClusterStatus; +import org.apache.hama.bsp.BSPJobID; +import org.apache.hama.bsp.JobProfile; +import org.apache.hama.bsp.JobStatus; +import org.apache.hama.bsp.TaskAttemptID; + +/** + * Protocol that a groom server and the central BSP Master use to communicate. + * This interface will contains several methods: submitJob, killJob, and + * killTask. + */ +public interface JobSubmissionProtocol extends HamaRPCProtocolVersion { + + /** + * Allocate a new id for the job. + * + * @return job id + * @throws IOException + */ + public BSPJobID getNewJobId() throws IOException; + + /** + * Submit a Job for execution. Returns the latest profile for that job. The + * job files should be submitted in system-dir/jobName. + * + * @param jobID + * @param jobFile + * @return jobStatus + * @throws IOException + */ + // public JobStatus submitJob(BSPJobID jobName) throws IOException; + + public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException; + + /** + * Get the current status of the cluster + * + * @param detailed if true then report groom names as well + * @return summary of the state of the cluster + */ + public ClusterStatus getClusterStatus(boolean detailed) throws IOException; + + /** + * Grab a handle to a job that is already known to the BSPMaster. + * + * @return Profile of the job, or null if not found. + */ + public JobProfile getJobProfile(BSPJobID jobid) throws IOException; + + /** + * Grab a handle to a job that is already known to the BSPMaster. + * + * @return Status of the job, or null if not found. + */ + public JobStatus getJobStatus(BSPJobID jobid) throws IOException; + + /** + * A BSP system always operates on a single filesystem. This function returns + * the fs name. ('local' if the localfs; 'addr:port' if dfs). The client can + * then copy files into the right locations prior to submitting the job. + */ + public String getFilesystemName() throws IOException; + + /** + * Get the jobs that are not completed and not failed + * + * @return array of JobStatus for the running/to-be-run jobs. + */ + public JobStatus[] jobsToComplete() throws IOException; + + /** + * Get all the jobs submitted. + * + * @return array of JobStatus for the submitted jobs + */ + public JobStatus[] getAllJobs() throws IOException; + + /** + * Grab the bspmaster system directory path where job-specific files are to be + * placed. + * + * @return the system directory where job-specific files are to be placed. + */ + public String getSystemDir(); + + /** + * Kill the indicated job + */ + public void killJob(BSPJobID jobid) throws IOException; + + /** + * Kill indicated task attempt. + * + * @param taskId the id of the task to kill. + * @param shouldFail if true the task is failed and added to failed tasks + * list, otherwise it is just killed, w/o affecting job failure + * status. + */ + public boolean killTask(TaskAttemptID taskId, boolean shouldFail) + throws IOException; + +} Index: core/src/main/java/org/apache/hama/ipc/MasterProtocol.java =================================================================== --- core/src/main/java/org/apache/hama/ipc/MasterProtocol.java (revision 0) +++ core/src/main/java/org/apache/hama/ipc/MasterProtocol.java (revision 0) @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ipc; + +import java.io.IOException; + +import org.apache.hama.bsp.GroomServerStatus; +import org.apache.hama.bsp.Directive; + +/** + * A new protocol for GroomServers communicate with BSPMaster. This + * protocol paired with WorkerProtocl, let GroomServers enrol with + * BSPMaster, so that BSPMaster can dispatch tasks to GroomServers. + */ +public interface MasterProtocol extends HamaRPCProtocolVersion { + + /** + * A GroomServer register with its status to BSPMaster, which will update + * GroomServers cache. + * + * @param status to be updated in cache. + * @return true if successfully register with BSPMaster; false if fail. + */ + boolean register(GroomServerStatus status) throws IOException; + + /** + * A GroomServer (periodically) reports task statuses back to the BSPMaster. + * @param directive + */ + boolean report(Directive directive) throws IOException; + + public String getSystemDir(); + +} Index: core/src/main/java/org/apache/hama/ipc/package.html =================================================================== --- core/src/main/java/org/apache/hama/ipc/package.html (revision 0) +++ core/src/main/java/org/apache/hama/ipc/package.html (revision 0) @@ -0,0 +1,23 @@ + + + + + +Tools to help define network clients and servers. + + Index: core/src/main/java/org/apache/hama/Constants.java =================================================================== --- core/src/main/java/org/apache/hama/Constants.java (revision 0) +++ core/src/main/java/org/apache/hama/Constants.java (revision 0) @@ -0,0 +1,95 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +/** + * Some constants used in the Hama. + */ +public interface Constants { + + public static final String GROOM_RPC_HOST = "bsp.groom.rpc.hostname"; + + public static final String DEFAULT_GROOM_RPC_HOST = "0.0.0.0"; + + public static final String GROOM_RPC_PORT = "bsp.groom.rpc.port"; + + /** Default port region rpc server listens on. */ + public static final int DEFAULT_GROOM_RPC_PORT = 50000; + + + /////////////////////////////////////// + // Constants for BSP Package + /////////////////////////////////////// + /** default host address */ + public static final String PEER_HOST = "bsp.peer.hostname"; + /** default host address */ + public static final String DEFAULT_PEER_HOST = "0.0.0.0"; + + public static final String PEER_PORT = "bsp.peer.port"; + /** Default port region server listens on. */ + public static final int DEFAULT_PEER_PORT = 61000; + + public static final String PEER_ID = "bsp.peer.id"; + + /** Parameter name for what groom server implementation to use. */ + public static final String GROOM_SERVER_IMPL= "hama.groomserver.impl"; + + /** When we encode strings, we always specify UTF8 encoding */ + static final String UTF8_ENCODING = "UTF-8"; + + /////////////////////////////////////// + // Constants for ZooKeeper + /////////////////////////////////////// + /** zookeeper root */ + public static final String ZOOKEEPER_ROOT = "bsp.zookeeper.root"; + /** zookeeper default root */ + public static final String DEFAULT_ZOOKEEPER_ROOT = "/bsp"; + + /** zookeeper server address */ + public static final String ZOOKEEPER_SERVER_ADDRS = "zookeeper.server"; + /** zookeeper default server address */ + static final String DEFAULT_ZOOKEEPER_SERVER_ADDR = "localhost:21810"; + /** Parameter name for number of times to retry writes to ZooKeeper. */ + public static final String ZOOKEEPER_RETRIES = "zookeeper.retries"; + /** Default number of times to retry writes to ZooKeeper. */ + public static final int DEFAULT_ZOOKEEPER_RETRIES = 5; + /** Parameter name for ZooKeeper pause between retries. In milliseconds. */ + public static final String ZOOKEEPER_PAUSE = "zookeeper.pause"; + /** Default ZooKeeper pause value. In milliseconds. */ + public static final int DEFAULT_ZOOKEEPER_PAUSE = 2 * 1000; + + static final String ZOOKEEPER_CONFIG_NAME = "zoo.cfg"; + static final String ZOOKEEPER_CLIENT_PORT = "hama.zookeeper.property.clientPort"; + static final String ZOOKEEPER_SESSION_TIMEOUT = "hama.zookeeper.session.timeout"; + static final int DEFAULT_ZOOKEEPER_CLIENT_PORT = 21810; + static final String ZOOKEEPER_QUORUM = "hama.zookeeper.quorum"; + /** Cluster is in distributed mode or not */ + static final String CLUSTER_DISTRIBUTED = "hama.cluster.distributed"; + /** Cluster is fully-distributed */ + static final String CLUSTER_IS_DISTRIBUTED = "true"; + + + // Other constants + + /** + * An empty instance. + */ + static final byte [] EMPTY_BYTE_ARRAY = new byte [0]; +} Index: core/src/main/java/org/apache/hama/ZooKeeperRunner.java =================================================================== --- core/src/main/java/org/apache/hama/ZooKeeperRunner.java (revision 0) +++ core/src/main/java/org/apache/hama/ZooKeeperRunner.java (revision 0) @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hama.zookeeper.QuorumPeer; + +/** + * This class starts and runs the ZooKeeperServer. + */ +public class ZooKeeperRunner extends Configured implements Tool { + + public static final Log LOG = LogFactory.getLog(ZooKeeperRunner.class); + + @Override + public int run(String[] args) throws Exception { + QuorumPeer.run(getConf()); + return 0; + } + + public static void main(String[] args) throws Exception { + int exitCode = ToolRunner.run(new ZooKeeperRunner(), args); + System.exit(exitCode); + } + +} Index: core/src/main/java/org/apache/hama/util/ClusterUtil.java =================================================================== --- core/src/main/java/org/apache/hama/util/ClusterUtil.java (revision 0) +++ core/src/main/java/org/apache/hama/util/ClusterUtil.java (revision 0) @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.bsp.BSPMaster; +import org.apache.hama.bsp.GroomServer; + +public class ClusterUtil { + private static final Log LOG = LogFactory.getLog(ClusterUtil.class); + + /** + * Data Structure to hold GroomServer Thread and GroomServer instance + */ + public static class GroomServerThread extends Thread { + private final GroomServer groomServer; + + public GroomServerThread(final GroomServer r, final int index) { + super(r, "GroomServer:" + index); + this.groomServer = r; + } + + /** @return the groom server */ + public GroomServer getGroomServer() { + return this.groomServer; + } + + /** + * Block until the groom server has come online, indicating it is ready + * to be used. + */ + public void waitForServerOnline() { + while (!groomServer.isRunning()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // continue waiting + } + } + } + } + + /** + * Creates a {@link GroomServerThread}. + * Call 'start' on the returned thread to make it run. + * @param c Configuration to use. + * @param hrsc Class to create. + * @param index Used distingushing the object returned. + * @throws IOException + * @return Groom server added. + */ + public static ClusterUtil.GroomServerThread createGroomServerThread(final Configuration c, + final Class hrsc, final int index) + throws IOException { + GroomServer server; + try { + server = hrsc.getConstructor(Configuration.class).newInstance(c); + } catch (Exception e) { + IOException ioe = new IOException(); + ioe.initCause(e); + throw ioe; + } + return new ClusterUtil.GroomServerThread(server, index); + } + + /** + * Start the cluster. + * @param m + * @param conf + * @param groomservers + * @return Address to use contacting master. + * @throws InterruptedException + * @throws IOException + */ + public static String startup(final BSPMaster m, + final List groomservers, Configuration conf) throws IOException, InterruptedException { + if (m != null) { + BSPMaster.startMaster((HamaConfiguration) conf); + } + + if (groomservers != null) { + for (ClusterUtil.GroomServerThread t: groomservers) { + t.start(); + } + } + + return m == null? null: BSPMaster.getAddress(conf).getHostName(); + } + + public static void shutdown(BSPMaster master, + List groomThreads, Configuration conf) { + LOG.debug("Shutting down HAMA Cluster"); + // TODO: + } +} Index: core/src/main/java/org/apache/hama/util/RunJar.java =================================================================== --- core/src/main/java/org/apache/hama/util/RunJar.java (revision 0) +++ core/src/main/java/org/apache/hama/util/RunJar.java (revision 0) @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Array; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.Manifest; + +import org.apache.hadoop.fs.FileUtil; + +/** + * Run a Hama job jar. + */ +public class RunJar { + + /** Unpack a jar file into a directory. */ + public static void unJar(File jarFile, File toDir) throws IOException { + JarFile jar = new JarFile(jarFile); + try { + Enumeration entries = jar.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = (JarEntry) entries.nextElement(); + if (!entry.isDirectory()) { + InputStream in = jar.getInputStream(entry); + try { + File file = new File(toDir, entry.getName()); + file.getParentFile().mkdirs(); + OutputStream out = new FileOutputStream(file); + try { + byte[] buffer = new byte[8192]; + int i; + while ((i = in.read(buffer)) != -1) { + out.write(buffer, 0, i); + } + } finally { + out.close(); + } + } finally { + in.close(); + } + } + } + } finally { + jar.close(); + } + } + + /** + * Run a Hama job jar. If the main class is not in the jar's manifest, then + * it must be provided on the command line. + */ + public static void main(String[] args) throws Throwable { + String usage = "Usage: hama jar [mainClass] args..."; + + if (args.length < 1) { + System.err.println(usage); + System.exit(-1); + } + + int firstArg = 0; + String fileName = args[firstArg++]; + File file = new File(fileName); + String mainClassName = null; + + JarFile jarFile = new JarFile(fileName); + Manifest manifest = jarFile.getManifest(); + if (manifest != null) { + mainClassName = manifest.getMainAttributes().getValue("Main-Class"); + } + jarFile.close(); + + if (mainClassName == null) { + if (args.length < 2) { + System.err.println(usage); + System.exit(-1); + } + mainClassName = args[firstArg++]; + } + mainClassName = mainClassName.replaceAll("/", "."); + + final File workDir = File.createTempFile("hama-unjar", ""); + workDir.delete(); + workDir.mkdirs(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + try { + FileUtil.fullyDelete(workDir); + } catch (IOException e) { + } + } + }); + + unJar(file, workDir); + + List classPath = new ArrayList(); + classPath.add(new File(workDir + "/").toURI().toURL()); + classPath.add(file.toURI().toURL()); + classPath.add(new File(workDir, "classes/").toURI().toURL()); + File[] libs = new File(workDir, "lib").listFiles(); + if (libs != null) { + for (int i = 0; i < libs.length; i++) { + classPath.add(libs[i].toURI().toURL()); + } + } + ClassLoader loader = new URLClassLoader((URL[]) classPath + .toArray(new URL[0])); + + Thread.currentThread().setContextClassLoader(loader); + Class mainClass = loader.loadClass(mainClassName); + Method main = mainClass.getMethod("main", new Class[] { Array.newInstance( + String.class, 0).getClass() }); + String[] newArgs = (String[]) Arrays.asList(args).subList(firstArg, + args.length).toArray(new String[0]); + try { + main.invoke(null, new Object[] { newArgs }); + } catch (InvocationTargetException e) { + throw e.getTargetException(); + } + } + +} Index: core/src/main/java/org/apache/hama/util/VersionInfo.java =================================================================== --- core/src/main/java/org/apache/hama/util/VersionInfo.java (revision 0) +++ core/src/main/java/org/apache/hama/util/VersionInfo.java (revision 0) @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +/** + * A version information class. + */ +public class VersionInfo { + + public static void main(String[] args) { + System.out.println("Apache Hama - 0.4"); + } +} Index: core/src/main/java/org/apache/hama/util/Bytes.java =================================================================== --- core/src/main/java/org/apache/hama/util/Bytes.java (revision 0) +++ core/src/main/java/org/apache/hama/util/Bytes.java (revision 0) @@ -0,0 +1,1226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.util.Comparator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.RawComparator; +import org.apache.hadoop.io.WritableComparator; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hama.Constants; + +/** + * Utility class that handles byte arrays, conversions to/from other types, + * comparisons, hash code generation, manufacturing keys for HashMaps or + * HashSets, etc. + */ +public class Bytes { + + private static final Log LOG = LogFactory.getLog(Bytes.class); + + /** + * Size of boolean in bytes + */ + public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE; + + /** + * Size of byte in bytes + */ + public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN; + + /** + * Size of char in bytes + */ + public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE; + + /** + * Size of double in bytes + */ + public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE; + + /** + * Size of float in bytes + */ + public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE; + + /** + * Size of int in bytes + */ + public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; + + /** + * Size of long in bytes + */ + public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; + + /** + * Size of short in bytes + */ + public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; + + /** + * Estimate of size cost to pay beyond payload in jvm for instance of byte []. + * Estimate based on study of jhat and jprofiler numbers. + */ + // JHat says BU is 56 bytes. + // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?) + public static final int ESTIMATED_HEAP_TAX = 16; + + /** + * Byte array comparator class. + */ + public static class ByteArrayComparator implements RawComparator { + /** + * Constructor + */ + public ByteArrayComparator() { + super(); + } + + public int compare(byte[] left, byte[] right) { + return compareTo(left, right); + } + + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + return compareTo(b1, s1, l1, b2, s2, l2); + } + } + + /** + * Pass this to TreeMaps where byte [] are keys. + */ + public static Comparator BYTES_COMPARATOR = new ByteArrayComparator(); + + /** + * Use comparing byte arrays, byte-by-byte + */ + public static RawComparator BYTES_RAWCOMPARATOR = new ByteArrayComparator(); + + /** + * Read byte-array written with a WritableableUtils.vint prefix. + * + * @param in Input to read from. + * @return byte array read off in + * @throws IOException e + */ + public static byte[] readByteArray(final DataInput in) throws IOException { + int len = WritableUtils.readVInt(in); + if (len < 0) { + throw new NegativeArraySizeException(Integer.toString(len)); + } + byte[] result = new byte[len]; + in.readFully(result, 0, len); + return result; + } + + /** + * Read byte-array written with a WritableableUtils.vint prefix. IOException + * is converted to a RuntimeException. + * + * @param in Input to read from. + * @return byte array read off in + */ + public static byte[] readByteArrayThrowsRuntime(final DataInput in) { + try { + return readByteArray(in); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Write byte-array with a WritableableUtils.vint prefix. + * + * @param out output stream to be written to + * @param b array to write + * @throws IOException e + */ + public static void writeByteArray(final DataOutput out, final byte[] b) + throws IOException { + if (b == null) { + WritableUtils.writeVInt(out, 0); + } else { + writeByteArray(out, b, 0, b.length); + } + } + + /** + * Write byte-array to out with a vint length prefix. + * + * @param out output stream + * @param b array + * @param offset offset into array + * @param length length past offset + * @throws IOException e + */ + public static void writeByteArray(final DataOutput out, final byte[] b, + final int offset, final int length) throws IOException { + WritableUtils.writeVInt(out, length); + out.write(b, offset, length); + } + + /** + * Write byte-array from src to tgt with a vint length prefix. + * + * @param tgt target array + * @param tgtOffset offset into target array + * @param src source array + * @param srcOffset source offset + * @param srcLength source length + * @return New offset in src array. + */ + public static int writeByteArray(final byte[] tgt, final int tgtOffset, + final byte[] src, final int srcOffset, final int srcLength) { + byte[] vint = vintToBytes(srcLength); + System.arraycopy(vint, 0, tgt, tgtOffset, vint.length); + int offset = tgtOffset + vint.length; + System.arraycopy(src, srcOffset, tgt, offset, srcLength); + return offset + srcLength; + } + + /** + * Put bytes at the specified byte array position. + * + * @param tgtBytes the byte array + * @param tgtOffset position in the array + * @param srcBytes array to write out + * @param srcOffset source offset + * @param srcLength source length + * @return incremented offset + */ + public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, + int srcOffset, int srcLength) { + System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength); + return tgtOffset + srcLength; + } + + /** + * Write a single byte out to the specified byte array position. + * + * @param bytes the byte array + * @param offset position in the array + * @param b byte to write out + * @return incremented offset + */ + public static int putByte(byte[] bytes, int offset, byte b) { + bytes[offset] = b; + return offset + 1; + } + + /** + * Returns a new byte array, copied from the passed ByteBuffer. + * + * @param bb A ByteBuffer + * @return the byte array + */ + public static byte[] toBytes(ByteBuffer bb) { + int length = bb.limit(); + byte[] result = new byte[length]; + System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length); + return result; + } + + /** + * @param b Presumed UTF-8 encoded byte array. + * @return String made from b + */ + public static String toString(final byte[] b) { + if (b == null) { + return null; + } + return toString(b, 0, b.length); + } + + /** + * Joins two byte arrays together using a separator. + * + * @param b1 The first byte array. + * @param sep The separator to use. + * @param b2 The second byte array. + */ + public static String toString(final byte[] b1, String sep, final byte[] b2) { + return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length); + } + + /** + * This method will convert utf8 encoded bytes into a string. If an + * UnsupportedEncodingException occurs, this method will eat it and return + * null instead. + * + * @param b Presumed UTF-8 encoded byte array. + * @param off offset into array + * @param len length of utf-8 sequence + * @return String made from b or null + */ + public static String toString(final byte[] b, int off, int len) { + if (b == null) { + return null; + } + if (len == 0) { + return ""; + } + try { + return new String(b, off, len, Constants.UTF8_ENCODING); + } catch (UnsupportedEncodingException e) { + LOG.error("UTF-8 not supported?", e); + return null; + } + } + + /** + * Write a printable representation of a byte array. + * + * @param b byte array + * @return string + * @see #toStringBinary(byte[], int, int) + */ + public static String toStringBinary(final byte[] b) { + return toStringBinary(b, 0, b.length); + } + + /** + * Write a printable representation of a byte array. Non-printable characters + * are hex escaped in the format \\x%02X, eg: \x00 \x05 etc + * + * @param b array to write out + * @param off offset to start at + * @param len length to write + * @return string output + */ + public static String toStringBinary(final byte[] b, int off, int len) { + StringBuilder result = new StringBuilder(); + try { + String first = new String(b, off, len, "ISO-8859-1"); + for (int i = 0; i < first.length(); ++i) { + int ch = first.charAt(i) & 0xFF; + if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') + || (ch >= 'a' && ch <= 'z') || ch == ',' || ch == '_' || ch == '-' + || ch == ':' || ch == ' ' || ch == '<' || ch == '>' || ch == '=' + || ch == '/' || ch == '.') { + result.append(first.charAt(i)); + } else { + result.append(String.format("\\x%02X", ch)); + } + } + } catch (UnsupportedEncodingException e) { + LOG.error("ISO-8859-1 not supported?", e); + } + return result.toString(); + } + + private static boolean isHexDigit(char c) { + return (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9'); + } + + /** + * Takes a ASCII digit in the range A-F0-9 and returns the corresponding + * integer/ordinal value. + * + * @param ch The hex digit. + * @return The converted hex value as a byte. + */ + public static byte toBinaryFromHex(byte ch) { + if (ch >= 'A' && ch <= 'F') + return (byte) ((byte) 10 + (byte) (ch - 'A')); + // else + return (byte) (ch - '0'); + } + + public static byte[] toBytesBinary(String in) { + // this may be bigger than we need, but lets be safe. + byte[] b = new byte[in.length()]; + int size = 0; + for (int i = 0; i < in.length(); ++i) { + char ch = in.charAt(i); + if (ch == '\\') { + // begin hex escape: + char next = in.charAt(i + 1); + if (next != 'x') { + // invalid escape sequence, ignore this one. + b[size++] = (byte) ch; + continue; + } + // ok, take next 2 hex digits. + char hd1 = in.charAt(i + 2); + char hd2 = in.charAt(i + 3); + + // they need to be A-F0-9: + if (!isHexDigit(hd1) || !isHexDigit(hd2)) { + // bogus escape code, ignore: + continue; + } + // turn hex ASCII digit -> number + byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2)); + + b[size++] = d; + i += 3; // skip 3 + } else { + b[size++] = (byte) ch; + } + } + // resize: + byte[] b2 = new byte[size]; + System.arraycopy(b, 0, b2, 0, size); + return b2; + } + + /** + * Converts a string to a UTF-8 byte array. + * + * @param s string + * @return the byte array + */ + public static byte[] toBytes(String s) { + try { + return s.getBytes(Constants.UTF8_ENCODING); + } catch (UnsupportedEncodingException e) { + LOG.error("UTF-8 not supported?", e); + return null; + } + } + + /** + * Convert a boolean to a byte array. True becomes -1 and false becomes 0. + * + * @param b value + * @return b encoded in a byte array. + */ + public static byte[] toBytes(final boolean b) { + return new byte[] { b ? (byte) -1 : (byte) 0 }; + } + + /** + * Reverses {@link #toBytes(boolean)} + * + * @param b array + * @return True or false. + */ + public static boolean toBoolean(final byte[] b) { + if (b.length != 1) { + throw new IllegalArgumentException("Array has wrong size: " + b.length); + } + return b[0] != (byte) 0; + } + + /** + * Convert a long value to a byte array using big-endian. + * + * @param val value to convert + * @return the byte array + */ + public static byte[] toBytes(long val) { + byte[] b = new byte[8]; + for (int i = 7; i > 0; i--) { + b[i] = (byte) val; + val >>>= 8; + } + b[0] = (byte) val; + return b; + } + + /** + * Converts a byte array to a long value. Reverses {@link #toBytes(long)} + * + * @param bytes array + * @return the long value + */ + public static long toLong(byte[] bytes) { + return toLong(bytes, 0, SIZEOF_LONG); + } + + /** + * Converts a byte array to a long value. Assumes there will be + * {@link #SIZEOF_LONG} bytes available. + * + * @param bytes bytes + * @param offset offset + * @return the long value + */ + public static long toLong(byte[] bytes, int offset) { + return toLong(bytes, offset, SIZEOF_LONG); + } + + /** + * Converts a byte array to a long value. + * + * @param bytes array of bytes + * @param offset offset into array + * @param length length of data (must be {@link #SIZEOF_LONG}) + * @return the long value + * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or + * if there's not enough room in the array at the offset indicated. + */ + public static long toLong(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_LONG || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); + } + long l = 0; + for (int i = offset; i < offset + length; i++) { + l <<= 8; + l ^= bytes[i] & 0xFF; + } + return l; + } + + private static IllegalArgumentException explainWrongLengthOrOffset( + final byte[] bytes, final int offset, final int length, + final int expectedLength) { + String reason; + if (length != expectedLength) { + reason = "Wrong length: " + length + ", expected " + expectedLength; + } else { + reason = "offset (" + offset + ") + length (" + length + ") exceed the" + + " capacity of the array: " + bytes.length; + } + return new IllegalArgumentException(reason); + } + + /** + * Put a long value out to the specified byte array position. + * + * @param bytes the byte array + * @param offset position in the array + * @param val long to write out + * @return incremented offset + * @throws IllegalArgumentException if the byte array given doesn't have + * enough room at the offset specified. + */ + public static int putLong(byte[] bytes, int offset, long val) { + if (bytes.length - offset < SIZEOF_LONG) { + throw new IllegalArgumentException("Not enough room to put a long at" + + " offset " + offset + " in a " + bytes.length + " byte array"); + } + for (int i = offset + 7; i > offset; i--) { + bytes[i] = (byte) val; + val >>>= 8; + } + bytes[offset] = (byte) val; + return offset + SIZEOF_LONG; + } + + /** + * Presumes float encoded as IEEE 754 floating-point "single format" + * + * @param bytes byte array + * @return Float made from passed byte array. + */ + public static float toFloat(byte[] bytes) { + return toFloat(bytes, 0); + } + + /** + * Presumes float encoded as IEEE 754 floating-point "single format" + * + * @param bytes array to convert + * @param offset offset into array + * @return Float made from passed byte array. + */ + public static float toFloat(byte[] bytes, int offset) { + return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); + } + + /** + * @param bytes byte array + * @param offset offset to write to + * @param f float value + * @return New offset in bytes + */ + public static int putFloat(byte[] bytes, int offset, float f) { + return putInt(bytes, offset, Float.floatToRawIntBits(f)); + } + + /** + * @param f float value + * @return the float represented as byte [] + */ + public static byte[] toBytes(final float f) { + // Encode it as int + return Bytes.toBytes(Float.floatToRawIntBits(f)); + } + + /** + * @param bytes byte array + * @return Return double made from passed bytes. + */ + public static double toDouble(final byte[] bytes) { + return toDouble(bytes, 0); + } + + /** + * @param bytes byte array + * @param offset offset where double is + * @return Return double made from passed bytes. + */ + public static double toDouble(final byte[] bytes, final int offset) { + return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); + } + + /** + * @param bytes byte array + * @param offset offset to write to + * @param d value + * @return New offset into array bytes + */ + public static int putDouble(byte[] bytes, int offset, double d) { + return putLong(bytes, offset, Double.doubleToLongBits(d)); + } + + /** + * Serialize a double as the IEEE 754 double format output. The resultant + * array will be 8 bytes long. + * + * @param d value + * @return the double represented as byte [] + */ + public static byte[] toBytes(final double d) { + // Encode it as a long + return Bytes.toBytes(Double.doubleToRawLongBits(d)); + } + + /** + * Convert an int value to a byte array + * + * @param val value + * @return the byte array + */ + public static byte[] toBytes(int val) { + byte[] b = new byte[4]; + for (int i = 3; i > 0; i--) { + b[i] = (byte) val; + val >>>= 8; + } + b[0] = (byte) val; + return b; + } + + /** + * Converts a byte array to an int value + * + * @param bytes byte array + * @return the int value + */ + public static int toInt(byte[] bytes) { + return toInt(bytes, 0, SIZEOF_INT); + } + + /** + * Converts a byte array to an int value + * + * @param bytes byte array + * @param offset offset into array + * @return the int value + */ + public static int toInt(byte[] bytes, int offset) { + return toInt(bytes, offset, SIZEOF_INT); + } + + /** + * Converts a byte array to an int value + * + * @param bytes byte array + * @param offset offset into array + * @param length length of int (has to be {@link #SIZEOF_INT}) + * @return the int value + * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or if + * there's not enough room in the array at the offset indicated. + */ + public static int toInt(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_INT || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); + } + int n = 0; + for (int i = offset; i < (offset + length); i++) { + n <<= 8; + n ^= bytes[i] & 0xFF; + } + return n; + } + + /** + * Put an int value out to the specified byte array position. + * + * @param bytes the byte array + * @param offset position in the array + * @param val int to write out + * @return incremented offset + * @throws IllegalArgumentException if the byte array given doesn't have + * enough room at the offset specified. + */ + public static int putInt(byte[] bytes, int offset, int val) { + if (bytes.length - offset < SIZEOF_INT) { + throw new IllegalArgumentException("Not enough room to put an int at" + + " offset " + offset + " in a " + bytes.length + " byte array"); + } + for (int i = offset + 3; i > offset; i--) { + bytes[i] = (byte) val; + val >>>= 8; + } + bytes[offset] = (byte) val; + return offset + SIZEOF_INT; + } + + /** + * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long. + * + * @param val value + * @return the byte array + */ + public static byte[] toBytes(short val) { + byte[] b = new byte[SIZEOF_SHORT]; + b[1] = (byte) val; + val >>= 8; + b[0] = (byte) val; + return b; + } + + /** + * Converts a byte array to a short value + * + * @param bytes byte array + * @return the short value + */ + public static short toShort(byte[] bytes) { + return toShort(bytes, 0, SIZEOF_SHORT); + } + + /** + * Converts a byte array to a short value + * + * @param bytes byte array + * @param offset offset into array + * @return the short value + */ + public static short toShort(byte[] bytes, int offset) { + return toShort(bytes, offset, SIZEOF_SHORT); + } + + /** + * Converts a byte array to a short value + * + * @param bytes byte array + * @param offset offset into array + * @param length length, has to be {@link #SIZEOF_SHORT} + * @return the short value + * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} or + * if there's not enough room in the array at the offset indicated. + */ + public static short toShort(byte[] bytes, int offset, final int length) { + if (length != SIZEOF_SHORT || offset + length > bytes.length) { + throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); + } + short n = 0; + n ^= bytes[offset] & 0xFF; + n <<= 8; + n ^= bytes[offset + 1] & 0xFF; + return n; + } + + /** + * Put a short value out to the specified byte array position. + * + * @param bytes the byte array + * @param offset position in the array + * @param val short to write out + * @return incremented offset + * @throws IllegalArgumentException if the byte array given doesn't have + * enough room at the offset specified. + */ + public static int putShort(byte[] bytes, int offset, short val) { + if (bytes.length - offset < SIZEOF_SHORT) { + throw new IllegalArgumentException("Not enough room to put a short at" + + " offset " + offset + " in a " + bytes.length + " byte array"); + } + bytes[offset + 1] = (byte) val; + val >>= 8; + bytes[offset] = (byte) val; + return offset + SIZEOF_SHORT; + } + + /** + * @param vint Integer to make a vint of. + * @return Vint as bytes array. + */ + public static byte[] vintToBytes(final long vint) { + long i = vint; + int size = WritableUtils.getVIntSize(i); + byte[] result = new byte[size]; + int offset = 0; + if (i >= -112 && i <= 127) { + result[offset] = (byte) i; + return result; + } + + int len = -112; + if (i < 0) { + i ^= -1L; // take one's complement' + len = -120; + } + + long tmp = i; + while (tmp != 0) { + tmp = tmp >> 8; + len--; + } + + result[offset++] = (byte) len; + + len = (len < -120) ? -(len + 120) : -(len + 112); + + for (int idx = len; idx != 0; idx--) { + int shiftbits = (idx - 1) * 8; + long mask = 0xFFL << shiftbits; + result[offset++] = (byte) ((i & mask) >> shiftbits); + } + return result; + } + + /** + * @param buffer buffer to convert + * @return vint bytes as an integer. + */ + public static long bytesToVint(final byte[] buffer) { + int offset = 0; + byte firstByte = buffer[offset++]; + int len = WritableUtils.decodeVIntSize(firstByte); + if (len == 1) { + return firstByte; + } + long i = 0; + for (int idx = 0; idx < len - 1; idx++) { + byte b = buffer[offset++]; + i = i << 8; + i = i | (b & 0xFF); + } + return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); + } + + /** + * Reads a zero-compressed encoded long from input stream and returns it. + * + * @param buffer Binary array + * @param offset Offset into array at which vint begins. + * @throws java.io.IOException e + * @return deserialized long from stream. + */ + public static long readVLong(final byte[] buffer, final int offset) + throws IOException { + byte firstByte = buffer[offset]; + int len = WritableUtils.decodeVIntSize(firstByte); + if (len == 1) { + return firstByte; + } + long i = 0; + for (int idx = 0; idx < len - 1; idx++) { + byte b = buffer[offset + 1 + idx]; + i = i << 8; + i = i | (b & 0xFF); + } + return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); + } + + /** + * @param left left operand + * @param right right operand + * @return 0 if equal, < 0 if left is less than right, etc. + */ + public static int compareTo(final byte[] left, final byte[] right) { + return compareTo(left, 0, left.length, right, 0, right.length); + } + + /** + * Lexographically compare two arrays. + * + * @param b1 left operand + * @param b2 right operand + * @param s1 Where to start comparing in the left buffer + * @param s2 Where to start comparing in the right buffer + * @param l1 How much to compare from the left buffer + * @param l2 How much to compare from the right buffer + * @return 0 if equal, < 0 if left is less than right, etc. + */ + public static int compareTo(byte[] b1, int s1, int l1, byte[] b2, int s2, + int l2) { + // Bring WritableComparator code local + int end1 = s1 + l1; + int end2 = s2 + l2; + for (int i = s1, j = s2; i < end1 && j < end2; i++, j++) { + int a = (b1[i] & 0xff); + int b = (b2[j] & 0xff); + if (a != b) { + return a - b; + } + } + return l1 - l2; + } + + /** + * @param left left operand + * @param right right operand + * @return True if equal + */ + public static boolean equals(final byte[] left, final byte[] right) { + // Could use Arrays.equals? + // noinspection SimplifiableConditionalExpression + if (left == null && right == null) { + return true; + } + return (left == null || right == null || (left.length != right.length) ? false + : compareTo(left, right) == 0); + } + + /** + * @param b bytes to hash + * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the + * passed in array. This method is what + * {@link org.apache.hadoop.io.Text} use calculating hash code. + */ + public static int hashCode(final byte[] b) { + return hashCode(b, b.length); + } + + /** + * @param b value + * @param length length of the value + * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the + * passed in array. This method is what + * {@link org.apache.hadoop.io.Text} use calculating hash code. + */ + public static int hashCode(final byte[] b, final int length) { + return WritableComparator.hashBytes(b, length); + } + + /** + * @param b bytes to hash + * @return A hash of b as an Integer that can be used as key in + * Maps. + */ + public static Integer mapKey(final byte[] b) { + return hashCode(b); + } + + /** + * @param b bytes to hash + * @param length length to hash + * @return A hash of b as an Integer that can be used as key in + * Maps. + */ + public static Integer mapKey(final byte[] b, final int length) { + return hashCode(b, length); + } + + /** + * @param a lower half + * @param b upper half + * @return New array that has a in lower half and b in upper half. + */ + public static byte[] add(final byte[] a, final byte[] b) { + return add(a, b, Constants.EMPTY_BYTE_ARRAY); + } + + /** + * @param a first third + * @param b second third + * @param c third third + * @return New array made from a, b and c + */ + public static byte[] add(final byte[] a, final byte[] b, final byte[] c) { + byte[] result = new byte[a.length + b.length + c.length]; + System.arraycopy(a, 0, result, 0, a.length); + System.arraycopy(b, 0, result, a.length, b.length); + System.arraycopy(c, 0, result, a.length + b.length, c.length); + return result; + } + + /** + * @param a array + * @param length amount of bytes to grab + * @return First length bytes from a + */ + public static byte[] head(final byte[] a, final int length) { + if (a.length < length) { + return null; + } + byte[] result = new byte[length]; + System.arraycopy(a, 0, result, 0, length); + return result; + } + + /** + * @param a array + * @param length amount of bytes to snarf + * @return Last length bytes from a + */ + public static byte[] tail(final byte[] a, final int length) { + if (a.length < length) { + return null; + } + byte[] result = new byte[length]; + System.arraycopy(a, a.length - length, result, 0, length); + return result; + } + + /** + * @param a array + * @param length new array size + * @return Value in a plus length prepended 0 bytes + */ + public static byte[] padHead(final byte[] a, final int length) { + byte[] padding = new byte[length]; + for (int i = 0; i < length; i++) { + padding[i] = 0; + } + return add(padding, a); + } + + /** + * @param a array + * @param length new array size + * @return Value in a plus length appended 0 bytes + */ + public static byte[] padTail(final byte[] a, final int length) { + byte[] padding = new byte[length]; + for (int i = 0; i < length; i++) { + padding[i] = 0; + } + return add(a, padding); + } + + /** + * Split passed range. Expensive operation relatively. Uses BigInteger math. + * Useful splitting ranges for MapReduce jobs. + * + * @param a Beginning of range + * @param b End of range + * @param num Number of times to split range. Pass 1 if you want to split the + * range in two; i.e. one split. + * @return Array of dividing values + */ + public static byte[][] split(final byte[] a, final byte[] b, final int num) { + byte[] aPadded; + byte[] bPadded; + if (a.length < b.length) { + aPadded = padTail(a, b.length - a.length); + bPadded = b; + } else if (b.length < a.length) { + aPadded = a; + bPadded = padTail(b, a.length - b.length); + } else { + aPadded = a; + bPadded = b; + } + if (compareTo(aPadded, bPadded) >= 0) { + throw new IllegalArgumentException("b <= a"); + } + if (num <= 0) { + throw new IllegalArgumentException("num cannot be < 0"); + } + byte[] prependHeader = { 1, 0 }; + BigInteger startBI = new BigInteger(add(prependHeader, aPadded)); + BigInteger stopBI = new BigInteger(add(prependHeader, bPadded)); + BigInteger diffBI = stopBI.subtract(startBI); + BigInteger splitsBI = BigInteger.valueOf(num + 1); + if (diffBI.compareTo(splitsBI) < 0) { + return null; + } + BigInteger intervalBI; + try { + intervalBI = diffBI.divide(splitsBI); + } catch (Exception e) { + LOG.error("Exception caught during division", e); + return null; + } + + byte[][] result = new byte[num + 2][]; + result[0] = a; + + for (int i = 1; i <= num; i++) { + BigInteger curBI = startBI + .add(intervalBI.multiply(BigInteger.valueOf(i))); + byte[] padded = curBI.toByteArray(); + if (padded[1] == 0) + padded = tail(padded, padded.length - 2); + else + padded = tail(padded, padded.length - 1); + result[i] = padded; + } + result[num + 1] = b; + return result; + } + + /** + * @param t operands + * @return Array of byte arrays made from passed array of Text + */ + public static byte[][] toByteArrays(final String[] t) { + byte[][] result = new byte[t.length][]; + for (int i = 0; i < t.length; i++) { + result[i] = Bytes.toBytes(t[i]); + } + return result; + } + + /** + * @param column operand + * @return A byte array of a byte array where first and only entry is + * column + */ + public static byte[][] toByteArrays(final String column) { + return toByteArrays(toBytes(column)); + } + + /** + * @param column operand + * @return A byte array of a byte array where first and only entry is + * column + */ + public static byte[][] toByteArrays(final byte[] column) { + byte[][] result = new byte[1][]; + result[0] = column; + return result; + } + + /** + * Binary search for keys in indexes. + * + * @param arr array of byte arrays to search for + * @param key the key you want to find + * @param offset the offset in the key you want to find + * @param length the length of the key + * @param comparator a comparator to compare. + * @return index of key + */ + public static int binarySearch(byte[][] arr, byte[] key, int offset, + int length, RawComparator comparator) { + int low = 0; + int high = arr.length - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + // we have to compare in this order, because the comparator order + // has special logic when the 'left side' is a special key. + int cmp = comparator.compare(key, offset, length, arr[mid], 0, + arr[mid].length); + // key lives above the midpoint + if (cmp > 0) + low = mid + 1; + // key lives below the midpoint + else if (cmp < 0) + high = mid - 1; + // BAM. how often does this really happen? + else + return mid; + } + return -(low + 1); + } + + /** + * Bytewise binary increment/deincrement of long contained in byte array on + * given amount. + * + * @param value - array of bytes containing long (length <= SIZEOF_LONG) + * @param amount value will be incremented on (deincremented if negative) + * @return array of bytes containing incremented long (length == SIZEOF_LONG) + * @throws IOException - if value.length > SIZEOF_LONG + */ + public static byte[] incrementBytes(byte[] value, long amount) + throws IOException { + byte[] val = value; + if (val.length < SIZEOF_LONG) { + // Hopefully this doesn't happen too often. + byte[] newvalue; + if (val[0] < 0) { + newvalue = new byte[] { -1, -1, -1, -1, -1, -1, -1, -1 }; + } else { + newvalue = new byte[SIZEOF_LONG]; + } + System.arraycopy(val, 0, newvalue, newvalue.length - val.length, + val.length); + val = newvalue; + } else if (val.length > SIZEOF_LONG) { + throw new IllegalArgumentException("Increment Bytes - value too big: " + + val.length); + } + if (amount == 0) + return val; + if (val[0] < 0) { + return binaryIncrementNeg(val, amount); + } + return binaryIncrementPos(val, amount); + } + + /* increment/deincrement for positive value */ + private static byte[] binaryIncrementPos(byte[] value, long amount) { + long amo = amount; + int sign = 1; + if (amount < 0) { + amo = -amount; + sign = -1; + } + for (int i = 0; i < value.length; i++) { + int cur = ((int) amo % 256) * sign; + amo = (amo >> 8); + int val = value[value.length - i - 1] & 0x0ff; + int total = val + cur; + if (total > 255) { + amo += sign; + total %= 256; + } else if (total < 0) { + amo -= sign; + } + value[value.length - i - 1] = (byte) total; + if (amo == 0) + return value; + } + return value; + } + + /* increment/deincrement for negative value */ + private static byte[] binaryIncrementNeg(byte[] value, long amount) { + long amo = amount; + int sign = 1; + if (amount < 0) { + amo = -amount; + sign = -1; + } + for (int i = 0; i < value.length; i++) { + int cur = ((int) amo % 256) * sign; + amo = (amo >> 8); + int val = ((~value[value.length - i - 1]) & 0x0ff) + 1; + int total = cur - val; + if (total >= 0) { + amo += sign; + } else if (total < -256) { + amo -= sign; + total %= 256; + } + value[value.length - i - 1] = (byte) total; + if (amo == 0) + return value; + } + return value; + } + +} Index: core/src/main/java/org/apache/hama/util/RandomVariable.java =================================================================== --- core/src/main/java/org/apache/hama/util/RandomVariable.java (revision 0) +++ core/src/main/java/org/apache/hama/util/RandomVariable.java (revision 0) @@ -0,0 +1,227 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +/** + * The RandomVaraibale Class provides static methods for generating random + * numbers. + */ +public class RandomVariable { + + /** + * Generate a random number between 0 and 1. + * + * @return a double between 0 and 1. + */ + public static double rand() { + double x = Math.random(); + return x; + } + + /** + * Generate a random integer. + * + * @param i0 min of the random variable. + * @param i1 max of the random variable. + * @return an int between i0 and i1. + */ + public static int randInt(int i0, int i1) { + double x = rand(); + int i = i0 + (int) Math.floor((i1 - i0 + 1) * x); + return i; + } + + /** + * Generate a random string using the specified prefix and a fixed length. + * @param prefix + * the specified string prefix. + * @param length + * the length of the string to be appended. + * @return random string. + */ + public static String randString(String prefix, int length) { + StringBuilder result = new StringBuilder(prefix); + for (int i = 0; i < length; i++) { + char ch = (char) ((Math.random() * 26) + 97); + result.append(ch); + } + + return result.toString(); + } + + /** + * Generate a random number from a uniform random variable. + * + * @param min min of the random variable. + * @param max max of the random variable. + * @return a double. + */ + public static double uniform(double min, double max) { + double x = min + (max - min) * rand(); + return x; + } + + /** + * Generate a random number from a discrete random variable. + * + * @param values discrete values. + * @param prob probability of each value. + * @return a double. + */ + public static double dirac(double[] values, double[] prob) { + double[] prob_cumul = new double[values.length]; + prob_cumul[0] = prob[0]; + for (int i = 1; i < values.length; i++) { + prob_cumul[i] = prob_cumul[i - 1] + prob[i]; + } + double y = rand(); + double x = 0; + for (int i = 0; i < values.length - 1; i++) { + if ((y > prob_cumul[i]) && (y < prob_cumul[i + 1])) { + x = values[i]; + } + } + return x; + } + + /** + * Generate a random number from a Gaussian (Normal) random variable. + * + * @param mu mean of the random variable. + * @param sigma standard deviation of the random variable. + * @return a double. + */ + public static double normal(double mu, double sigma) { + double x = mu + sigma * Math.cos(2 * Math.PI * rand()) + * Math.sqrt(-2 * Math.log(rand())); + return x; + } + + /** + * Generate a random number from a Chi-2 random variable. + * + * @param n degrees of freedom of the chi2 random variable. + * @return a double. + */ + public static double chi2(int n) { + double x = 0; + for (int i = 0; i < n; i++) { + double norm = normal(0, 1); + x += norm * norm; + } + return x; + } + + /** + * Generate a random number from a LogNormal random variable. + * + * @param mu mean of the Normal random variable. + * @param sigma standard deviation of the Normal random variable. + * @return a double. + */ + public static double logNormal(double mu, double sigma) { + double x = mu + sigma * Math.cos(2 * Math.PI * rand()) + * Math.sqrt(-2 * Math.log(rand())); + return x; + } + + /** + * Generate a random number from an exponantial random variable (Mean = + * 1/lambda, variance = 1/lambda^2). + * + * @param lambda parmaeter of the exponential random variable. + * @return a double. + */ + public static double exponential(double lambda) { + double x = -1 / lambda * Math.log(rand()); + return x; + } + + /** + * Generate a random number from a symetric triangular random variable. + * + * @param min min of the random variable. + * @param max max of the random variable. + * @return a double. + */ + public static double triangular(double min, double max) { + double x = min / 2 + (max - min) * rand() / 2 + min / 2 + (max - min) + * rand() / 2; + return x; + } + + /** + * Generate a random number from a non-symetric triangular random variable. + * + * @param min min of the random variable. + * @param med value of the random variable with max density. + * @param max max of the random variable. + * @return a double. + */ + public static double triangular(double min, double med, double max) { + double y = rand(); + double x = (y < ((med - min) / (max - min))) ? (min + Math.sqrt(y + * (max - min) * (med - min))) : (max - Math.sqrt((1 - y) * (max - min) + * (max - med))); + return x; + } + + /** + * Generate a random number from a beta random variable. + * + * @param a first parameter of the Beta random variable. + * @param b second parameter of the Beta random variable. + * @return a double. + */ + public static double beta(double a, double b) { + double try_x; + double try_y; + do { + try_x = Math.pow(rand(), 1 / a); + try_y = Math.pow(rand(), 1 / b); + } while ((try_x + try_y) > 1); + return try_x / (try_x + try_y); + } + + /** + * Generate a random number from a Cauchy random variable (Mean = Inf, and + * Variance = Inf). + * + * @param mu median of the Weibull random variable + * @param sigma second parameter of the Cauchy random variable. + * @return a double. + */ + public static double cauchy(double mu, double sigma) { + double x = sigma * Math.tan(Math.PI * (rand() - 0.5)) + mu; + return x; + } + + /** + * Generate a random number from a Weibull random variable. + * + * @param lambda first parameter of the Weibull random variable. + * @param c second parameter of the Weibull random variable. + * @return a double. + */ + public static double weibull(double lambda, double c) { + double x = Math.pow(-Math.log(1 - rand()), 1 / c) / lambda; + return x; + } +} Index: core/src/main/java/org/apache/hama/util/BSPServletUtil.java =================================================================== --- core/src/main/java/org/apache/hama/util/BSPServletUtil.java (revision 0) +++ core/src/main/java/org/apache/hama/util/BSPServletUtil.java (revision 0) @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.util; + +import java.io.IOException; +import java.util.Date; +import java.util.Calendar; + +import org.apache.hadoop.util.ServletUtil; +import org.apache.hama.bsp.JobStatus; + +public class BSPServletUtil extends ServletUtil { + + public static final String HTML_TAIL = "
\n" + + "Hama, " + + Calendar.getInstance().get(Calendar.YEAR) + ".\n" + ""; + + /** + * HTML footer to be added in the jsps. + * + * @return the HTML footer. + */ + public static String htmlFooter() { + return HTML_TAIL; + } + + /** + * Method used to generate the Job table for Job pages. + * + * @param label display heading to be used in the job table. + * @param jobs vector of jobs to be displayed in table. + * @param refresh refresh interval to be used in jobdetails page. + * @param rowId beginning row id to be used in the table. + * @return generated HTML + * @throws IOException + */ + public static String generateJobTable(String label, JobStatus[] jobs, + int refresh, int rowId) throws IOException { + + StringBuffer sb = new StringBuffer(); + + if (jobs.length > 0) { + sb.append("\n"); + sb.append("" + "" + "" + + "" + "" + "\n"); + for (JobStatus status : jobs) { + sb.append("\n"); + } + sb.append("
JobidUserNameSuperStepStarttime
"); + sb.append(status.getJobID()); + sb.append(""); + sb.append(status.getUsername()); + sb.append(""); + sb.append(status.getName()); + sb.append(""); + sb.append(status.getSuperstepCount()); + sb.append(""); + sb.append(new Date(status.getStartTime())); + sb.append("
"); + } else { + sb.append("No jobs found!"); + } + + return sb.toString(); + } + +} Index: core/src/main/java/org/apache/hama/util/package.html =================================================================== --- core/src/main/java/org/apache/hama/util/package.html (revision 0) +++ core/src/main/java/org/apache/hama/util/package.html (revision 0) @@ -0,0 +1,23 @@ + + + + + +Common utilities. + + Index: core/src/main/java/org/apache/hama/HamaConfiguration.java =================================================================== --- core/src/main/java/org/apache/hama/HamaConfiguration.java (revision 0) +++ core/src/main/java/org/apache/hama/HamaConfiguration.java (revision 0) @@ -0,0 +1,61 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama; + +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +/** + * Adds Hama configuration files to a Configuration. + */ +public class HamaConfiguration extends Configuration { + /** constructor */ + public HamaConfiguration() { + super(); + addHamaResources(); + } + + public HamaConfiguration(Path confFile) { + super(); + this.addResource(confFile); + } + + /** + * Create a clone of passed configuration. + * + * @param c Configuration to clone. + */ + public HamaConfiguration(final Configuration c) { + this(); + for (Entry e : c) { + set(e.getKey(), e.getValue()); + } + } + + /** + * Adds Hama configuration files to a Configuration + */ + private void addHamaResources() { + Configuration.addDefaultResource("hama-default.xml"); + Configuration.addDefaultResource("hama-site.xml"); + } +} Index: core/src/main/java/org/apache/hama/package.html =================================================================== --- core/src/main/java/org/apache/hama/package.html (revision 0) +++ core/src/main/java/org/apache/hama/package.html (revision 0) @@ -0,0 +1,23 @@ + + + + + +Hama base package. + + Index: core/src/main/java/org/apache/overview.html =================================================================== --- core/src/main/java/org/apache/overview.html (revision 0) +++ core/src/main/java/org/apache/overview.html (revision 0) @@ -0,0 +1,152 @@ + + + + + + Hama + + +Hama is a distributed computing framework based on BSP (Bulk Synchronous Parallel) computing techniques for massive scientific computations. + +

Requirements

+
    +
  • hadoop-0.20.x for HDFS
  • +
  • Sun Java JDK 1.6.x or higher version
  • +
+ +

Startup script

+The $HAMA_HOME/bin directory contains some script used to start up the Hama daemons. +
    +
  • +start-bspd.sh - Starts all Hama daemons, the BSPMaster, GroomServers and Zookeeper. +
  • +
+ +

Configuration files

+ +The $HAMA_HOME/conf directory contains some configuration files for Hama. These are: +
    +
  • +hama-env.sh - This file contains some environment variable settings used by Hama. You can use these to affect some aspects of Hama daemon behavior, +such as where log files are stored, the maximum amount of heap used etc. The only variable you should need to change in this file is JAVA_HOME, +which specifies the path to the Java 1.5.x installation used by Hama. +
  • +
  • +groomservers - This file lists the hosts, one per line, where the GroomServer daemons will run. By default this contains the single entry localhost.
  • +
  • +hama-default.xml - This file contains generic default settings for Hama daemons. Do not modify this file. +
  • +
  • +hama-site.xml - This file contains site specific settings for all Hama daemons and BSP jobs. +This file is empty by default. Settings in this file override those in hama-default.xml. +This file should contain settings that must be respected by all servers and clients in a Hama installation. +
  • +
+ +

Setting up Hama

+This section describes how to get started by setting up a Hama cluster. +
    +
  • +BSPMaster and Zookeeper settings - Figure out where to run your HDFS namenode and BSPMaster. +Set the variable bsp.master.address to the BSPMaster's intended host:port. +Set the variable fs.default.name to the HDFS Namenode's intended host:port. +
  • +
+ +An example of a hama-site.xml file: +
+<!--?xml version="1.0"?-->
+<!--?xml-stylesheet type="text/xsl" href="configuration.xsl"?-->
+<configuration>
+  <property>
+    <name>bsp.master.address</name>
+    <value>host1.mydomain.com:40000</value>
+    <description>The address of the bsp master server. Either the
+    literal string "local" or a host:port for distributed mode
+    </description>
+  </property>
+  
+  <property>
+    <name>fs.default.name</name>
+    <value>hdfs://host1.mydomain.com:9000/</value>
+    <description>
+      The name of the default file system. Either the literal string
+      "local" or a host:port for HDFS.
+    </description>
+  </property>
+  
+  <property>
+    <name>hama.zookeeper.quorum</name>
+    <value>host1.mydomain.com</value>
+    <description>Comma separated list of servers in the ZooKeeper Quorum.
+    For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+    By default this is set to localhost for local and pseudo-distributed modes
+    of operation. For a fully-distributed setup, this should be set to a full
+    list of ZooKeeper quorum servers. If HAMA_MANAGES_ZK is set in hama-env.sh
+    this is the list of servers which we will start/stop zookeeper on.
+    </description>
+  </property>
+</configuration>
+
+ +If you are managing your own ZooKeeper, you have to specify the port number as below: + +
+  <property>
+    <name>hama.zookeeper.property.clientPort</name>
+    <value>2181</value>
+  </property>
+
+ + + +

Starting a Hama cluster

+ +Run the command: +
+% $HAMA_HOME/bin/start-bspd.sh
+
+ +This will startup a BSPMaster, GroomServers and Zookeeper on your machine. + +

Stopping a Hama cluster

+ +Run the command: +
+% $HAMA_HOME/bin/stop-bspd.sh
+
+ +to stop all the daemons running on your cluster. + +

Run the BSP Examples

+Run the command: +
+% $HAMA_HOME/bin/hama jar hama-examples-0.x.0-incubating.jar
+
+ +

Hama Web Interfaces

+ +The web UI provides information about BSP job statistics of the Hama cluster, running/completed/failed jobs. +

+By default, it’s available at http://localhost:40013 + + + + Index: core/src/main/webapp/bspmaster/machines.jsp =================================================================== --- core/src/main/webapp/bspmaster/machines.jsp (revision 0) +++ core/src/main/webapp/bspmaster/machines.jsp (revision 0) @@ -0,0 +1,61 @@ + +<%@ page contentType="text/html; charset=UTF-8" import="javax.servlet.*" + import="javax.servlet.http.*" import="java.io.*" import="java.util.*" + import="java.text.DecimalFormat" import="org.apache.hama.bsp.*" + import="org.apache.hama.util.*"%> +<%!private static final long serialVersionUID = 1L;%> +<% + BSPMaster tracker = (BSPMaster) application + .getAttribute("bsp.master"); + ClusterStatus status = tracker.getClusterStatus(true); + String trackerName = tracker.getBSPMasterName(); + String type = request.getParameter("type"); +%> +<%!public void generateGroomsTable(JspWriter out, String type, + ClusterStatus status, BSPMaster master) throws IOException { + out.print("
\n"); + out.print("\n"); + out.print("\n"); + out.print("" + "" + + "\n"); + for (Map.Entry entry : status.getActiveGroomNames() + .entrySet()) { + out.print("" + "\n"); + } + out.print("
Groom Servers
NameHost# running tasks
"); + out.print(entry.getValue() + ""); + out.print(entry.getValue() + "" + 1 + "
\n"); + out.print("
\n"); + }%> + + + +<%=trackerName%> Hama Machine List + + +

<%=trackerName%> Hama Machine List

+ +

Grooms

+<% + generateGroomsTable(out, type, status, tracker); +%> + +<% + out.println(BSPServletUtil.htmlFooter()); +%> \ No newline at end of file Index: core/src/main/webapp/bspmaster/bspjob.jsp =================================================================== --- core/src/main/webapp/bspmaster/bspjob.jsp (revision 0) +++ core/src/main/webapp/bspmaster/bspjob.jsp (revision 0) @@ -0,0 +1,66 @@ + +<%@ page contentType="text/html; charset=UTF-8" import="javax.servlet.*" + import="javax.servlet.http.*" import="java.io.*" import="java.util.*" + import="java.text.DecimalFormat" import="org.apache.hama.bsp.*" + import="org.apache.hama.util.*"%> +<%!private static final long serialVersionUID = 1L;%> +<% + BSPMaster tracker = (BSPMaster) application + .getAttribute("bsp.master"); + String idString = request.getParameter("jobid"); + JobStatus status = tracker.getJobStatus(BSPJobID.forName(idString)); + JobStatus.State state = status.getState(); +%> + + + +Hama BSP Job Summary + + +

<%=status.getName()%>

+ + State: <%=state.toString() %> + +

+ + + + + + + + + + + + + + + + + +
NameUserSuperStepStartTimeFinishTime
<%=status.getName() %><%=status.getUsername() %><%=status.getSuperstepCount() %><%=new Date(status.getStartTime()).toString() %> + <% if(status.getFinishTime() != 0L) {out.write(new Date(status.getFinishTime()).toString());} %> +
+ +
+ Back to BSPMaster + + <% + out.println(BSPServletUtil.htmlFooter()); + %> \ No newline at end of file Index: core/src/main/webapp/bspmaster/index.html =================================================================== --- core/src/main/webapp/bspmaster/index.html (revision 0) +++ core/src/main/webapp/bspmaster/index.html (revision 0) @@ -0,0 +1,36 @@ + + + + + +Hama Administration + + + + +

Hama Administration

+ + + + + + Index: core/src/main/webapp/bspmaster/bspmaster.jsp =================================================================== --- core/src/main/webapp/bspmaster/bspmaster.jsp (revision 0) +++ core/src/main/webapp/bspmaster/bspmaster.jsp (revision 0) @@ -0,0 +1,82 @@ + +<%@ page contentType="text/html; charset=UTF-8" import="javax.servlet.*" + import="javax.servlet.http.*" import="java.io.*" import="java.util.*" + import="java.text.DecimalFormat" import="org.apache.hama.bsp.*" + import="org.apache.hama.util.*"%> +<%!private static final long serialVersionUID = 1L;%> +<% + BSPMaster tracker = (BSPMaster) application + .getAttribute("bsp.master"); + ClusterStatus status = tracker.getClusterStatus(true); + String trackerName = tracker.getBSPMasterName(); + JobStatus[] runningJobs = tracker.jobsToComplete(); + JobStatus[] allJobs = tracker.getAllJobs(); +%> +<%!private static DecimalFormat percentFormat = new DecimalFormat("##0.00"); + + public void generateSummaryTable(JspWriter out, ClusterStatus status, + BSPMaster tracker) throws IOException { + String tasksPerNode = status.getGroomServers() > 0 ? percentFormat + .format(((double) (status.getMaxTasks()) / status + .getGroomServers())) : "-"; + out.print("\n" + + "" + "" + + "" + + "\n"); + out.print("
Groom ServersBSP Task CapacityAvg. Tasks/NodeBlacklisted Nodes
" + + status.getActiveGroomNames().size() + "" + + status.getMaxTasks() + "" + tasksPerNode + + "" + 0 + + "" + "
\n"); + + out.print("
"); + }%> + + + + +<%=trackerName%> Hama BSP Administration + + + + +

<%=trackerName%> Hama BSP Administration

+ +State: +<%=status.getBSPMasterState()%>
+Started: +<%=new Date(tracker.getStartTime())%>
+Identifier: +<%=tracker.getBSPMasterIdentifier()%>
+ +
+<% + generateSummaryTable(out, status, tracker); +%> +
+ +

Running Jobs

+<%=BSPServletUtil.generateJobTable("Running", runningJobs, + 30, 0)%> +
+

All Jobs History

+<%=BSPServletUtil.generateJobTable("All", allJobs, + 30, 0)%> +<% + out.println(BSPServletUtil.htmlFooter()); +%> \ No newline at end of file Index: core/bin/grooms.sh =================================================================== --- core/bin/grooms.sh (revision 0) +++ core/bin/grooms.sh (revision 0) @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Run a shell command on all slave hosts. +# +# Environment Variables +# +# HAMA_GROOMS File naming remote hosts. +# Default is ${HAMA_CONF_DIR}/groomservers. +# HAMA_CONF_DIR Alternate conf dir. Default is ${HAMA_HOME}/conf. +# HAMA_GROOM_SLEEP Seconds to sleep between spawning remote commands. +# HAMA_SSH_OPTS Options passed to ssh when running remote commands. +## + +usage="Usage: grooms.sh [--config confdir] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hama-config.sh + +# If the groomservers file is specified in the command line, +# then it takes precedence over the definition in +# hama-env.sh. Save it here. +HOSTLIST=$HAMA_GROOMS + +if [ -f "${HAMA_CONF_DIR}/hama-env.sh" ]; then + . "${HAMA_CONF_DIR}/hama-env.sh" +fi + +if [ "$HOSTLIST" = "" ]; then + if [ "$HAMA_GROOMS" = "" ]; then + export HOSTLIST="${HAMA_CONF_DIR}/groomservers" + else + export HOSTLIST="${HAMA_GROOMS}" + fi +fi + +for groom in `cat "$HOSTLIST"|sed "s/#.*$//;/^$/d"`; do + ssh $HAMA_SSH_OPTS $groom $"${@// /\\ }" \ + 2>&1 | sed "s/^/$groom: /" & + if [ "$HAMA_GROOM_SLEEP" != "" ]; then + sleep $HAMA_GROOM_SLEEP + fi +done + +wait Index: core/bin/zookeepers.sh =================================================================== --- core/bin/zookeepers.sh (revision 0) +++ core/bin/zookeepers.sh (revision 0) @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2009 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all zookeeper hosts. +# +# Environment Variables +# +# HAMA_CONF_DIR Alternate hama conf dir. Default is ${HAMA_HOME}/conf. +# HAMA_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: zookeepers [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hama-config.sh + +if [ "$HAMA_MANAGES_ZK" = "" ]; then + HAMA_MANAGES_ZK=true +fi + +if [ "$HAMA_MANAGES_ZK" = "true" ]; then + hosts=`"$bin"/hama org.apache.hama.zookeeper.ZKServerTool` + cmd=$"${@// /\\ }" + for zookeeper in $hosts; do + ssh $HAMA_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & + done +fi + +wait Index: core/bin/hama-daemon.sh =================================================================== --- core/bin/hama-daemon.sh (revision 0) +++ core/bin/hama-daemon.sh (revision 0) @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Runs a Hama command as a daemon. +# +# Environment Variables +# +# HAMA_CONF_DIR Alternate conf dir. Default is ${HAMA_HOME}/conf. +# HAMA_LOG_DIR Where log files are stored. PWD by default. +# HAMA_MASTER host:path where hama code should be rsync'd from +# HAMA_PID_DIR The pid files are stored. /tmp by default. +# HAMA_IDENT_STRING A string representing this instance of hama. $USER by default +# HAMA_NICENESS The scheduling priority for daemons. Defaults to 0. +## + +usage="Usage: hama-daemon.sh [--config ] [--hosts hostlistfile] (start|stop) " + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hama-config.sh + +# get arguments +startStop=$1 +shift +command=$1 +shift + +hama_rotate_log () +{ + log=$1; + num=5; + if [ -n "$2" ]; then + num=$2 + fi + if [ -f "$log" ]; then # rotate logs + while [ $num -gt 1 ]; do + prev=`expr $num - 1` + [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" + num=$prev + done + mv "$log" "$log.$num"; + fi +} + +if [ -f "${HAMA_CONF_DIR}/hama-env.sh" ]; then + . "${HAMA_CONF_DIR}/hama-env.sh" +fi + +# get log directory +if [ "$HAMA_LOG_DIR" = "" ]; then + export HAMA_LOG_DIR="$HAMA_HOME/logs" +fi +mkdir -p "$HAMA_LOG_DIR" + +if [ "$HAMA_PID_DIR" = "" ]; then + HAMA_PID_DIR=/tmp +fi + +if [ "$HAMA_IDENT_STRING" = "" ]; then + export HAMA_IDENT_STRING="$USER" +fi + +# some variables +export HAMA_LOGFILE=hama-$HAMA_IDENT_STRING-$command-$HOSTNAME.log +export HAMA_ROOT_LOGGER="INFO,DRFA" +log=$HAMA_LOG_DIR/hama-$HAMA_IDENT_STRING-$command-$HOSTNAME.out +pid=$HAMA_PID_DIR/hama-$HAMA_IDENT_STRING-$command.pid + +# Set default scheduling priority +if [ "$HAMA_NICENESS" = "" ]; then + export HAMA_NICENESS=0 +fi + +case $startStop in + + (start) + + mkdir -p "$HAMA_PID_DIR" + + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo $command running as process `cat $pid`. Stop it first. + exit 1 + fi + fi + + if [ "$HAMA_MASTER" != "" ]; then + echo rsync from $HAMA_MASTER + rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $HAMA_MASTER/ "$HAMA_HOME" + fi + + hama_rotate_log $log + echo starting $command, logging to $log + cd "$HAMA_HOME" + nohup nice -n $HAMA_NICENESS "$HAMA_HOME"/bin/hama --config $HAMA_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & + echo $! > $pid + sleep 1; head "$log" + ;; + + (stop) + + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo stopping $command + kill `cat $pid` + else + echo no $command to stop + fi + else + echo no $command to stop + fi + ;; + + (*) + echo $usage + exit 1 + ;; + +esac Index: core/bin/hama-config.sh =================================================================== --- core/bin/hama-config.sh (revision 0) +++ core/bin/hama-config.sh (revision 0) @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# included in all the hama scripts with source command +# should not be executable directly +# also should not be passed any arguments, since we need original $* + +# resolve links - $0 may be a softlink + +this="$0" +while [ -h "$this" ]; do + ls=`ls -ld "$this"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + this="$link" + else + this=`dirname "$this"`/"$link" + fi +done + +# convert relative path to absolute path +bin=`dirname "$this"` +script=`basename "$this"` +bin=`cd "$bin"; pwd` +this="$bin/$script" + +# the root of the Hama installation +export HAMA_HOME=`dirname "$this"`/.. + +#check to see if the conf dir is given as an optional argument +if [ $# -gt 1 ] +then + if [ "--config" = "$1" ] + then + shift + confdir=$1 + shift + HAMA_CONF_DIR=$confdir + fi +fi + +# Allow alternate conf dir location. +HAMA_CONF_DIR="${HAMA_CONF_DIR:-$HAMA_HOME/conf}" + +# Source the hama-env.sh. +# Will have JAVA_HOME and HAMA_MANAGES_ZK defined. +if [ -f "${HAMA_CONF_DIR}/hama-env.sh" ]; then + . "${HAMA_CONF_DIR}/hama-env.sh" +fi + +#check to see it is specified whether to use the grooms or the +# masters file +if [ $# -gt 1 ] +then + if [ "--hosts" = "$1" ] + then + shift + grooms=$1 + shift + export HAMA_SLAVES="${HAMA_CONF_DIR}/$grooms" + fi +fi Index: core/bin/hama-daemons.sh =================================================================== --- core/bin/hama-daemons.sh (revision 0) +++ core/bin/hama-daemons.sh (revision 0) @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Run a Hama command on all slave hosts. + +usage="Usage: hama-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..." + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. $bin/hama-config.sh + +remote_cmd="cd ${HAMA_HOME}; $bin/hama-daemon.sh --config ${HAMA_CONF_DIR} $@" +args="--config ${HAMA_CONF_DIR} $remote_cmd" +command=$2 + +case $command in + (zookeeper) + exec "$bin/zookeepers.sh" $args + ;; + (*) + exec "$bin/grooms.sh" $args + ;; +esac \ No newline at end of file Index: core/bin/stop-bspd.sh =================================================================== --- core/bin/stop-bspd.sh (revision 0) +++ core/bin/stop-bspd.sh (revision 0) @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Stop hama map reduce daemons. Run this on master node. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hama-config.sh + +"$bin"/hama-daemon.sh --config $HAMA_CONF_DIR stop bspmaster +"$bin"/hama-daemons.sh --config $HAMA_CONF_DIR stop groom +"$bin"/hama-daemons.sh --config "${HAMA_CONF_DIR}" stop zookeeper Index: core/bin/hama =================================================================== --- core/bin/hama (revision 0) +++ core/bin/hama (revision 0) @@ -0,0 +1,186 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The Hama command script +# +# Environment Variables +# +# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +# +# HAMA_CLASSPATH Extra Java CLASSPATH entries. +# +# HAMA_HEAPSIZE The maximum amount of heap to use, in MB. +# Default is 1000. +# +# HAMA_OPTS Extra Java runtime options. +# +# HAMA_NAMENODE_OPTS These options are added to HAMA_OPTS +# HAMA_CLIENT_OPTS when the respective command is run. +# HAMA_{COMMAND}_OPTS etc HAMA_JT_OPTS applies to JobTracker +# for e.g. HAMA_CLIENT_OPTS applies to +# more than one command (fs, dfs, fsck, +# dfsadmin etc) +# +# HAMA_CONF_DIR Alternate conf dir. Default is ${HAMA_HOME}/conf. +# +# HAMA_ROOT_LOGGER The root appender. Default is INFO,console +# + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hama-config.sh + +cygwin=false +case "`uname`" in +CYGWIN*) cygwin=true;; +esac + +# if no args specified, show usage +if [ $# = 0 ]; then + echo "Usage: hama [--config confdir] COMMAND" + echo "where COMMAND is one of:" + echo " bspmaster run the BSP Master node" + echo " groom run the Groom node" + echo " zookeeper run a Zookeeper server" + echo " job manipulate BSP jobs" + echo " jar run a jar file" + echo " or" + echo " CLASSNAME run the class named CLASSNAME" + echo "Most commands print help when invoked w/o parameters." + exit 1 +fi + +# get arguments +COMMAND=$1 +shift + +if [ -f "${HAMA_CONF_DIR}/hama-env.sh" ]; then + . "${HAMA_CONF_DIR}/hama-env.sh" +fi + +# some Java parameters +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi + +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +# check envvars which might override default args +if [ "$HAMA_HEAPSIZE" != "" ]; then + #echo "run with heapsize $HAMA_HEAPSIZE" + JAVA_HEAP_MAX="-Xmx""$HAMA_HEAPSIZE""m" + #echo $JAVA_HEAP_MAX +fi + +# CLASSPATH initially contains $HAMA_CONF_DIR +CLASSPATH="${HAMA_CONF_DIR}" +CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar + +# for developers, add Hama classes to CLASSPATH +if [ -d "$HAMA_HOME/target/classes" ]; then + CLASSPATH=${CLASSPATH}:$HAMA_HOME/target/classes +fi +if [ -d "$HAMA_HOME/target/test-classes/classes" ]; then + CLASSPATH=${CLASSPATH}:$HAMA_HOME/target/test-classes +fi + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + +# for releases, add core hama jar to CLASSPATH +for f in $HAMA_HOME/hama-**.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +# add libs to CLASSPATH +for f in $HAMA_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +# add user-specified CLASSPATH last +if [ "$HAMA_CLASSPATH" != "" ]; then + CLASSPATH=${CLASSPATH}:${HAMA_CLASSPATH} +fi + +# default log directory & file +if [ "$HAMA_LOG_DIR" = "" ]; then + HAMA_LOG_DIR="$HAMA_HOME/logs" +fi +if [ "$HAMA_LOGFILE" = "" ]; then + HAMA_LOGFILE='hama.log' +fi + +# default policy file for service-level authorization +if [ "$HAMA_POLICYFILE" = "" ]; then + HAMA_POLICYFILE="hama-policy.xml" +fi + +# restore ordinary behaviour +unset IFS + +# figure out which class to run +if [ "$COMMAND" = "bspmaster" ] ; then + CLASS='org.apache.hama.BSPMasterRunner' + BSP_OPTS="$BSP_OPTS $BSP_BSPMASTER_OPTS" +elif [ "$COMMAND" = "groom" ] ; then + CLASS='org.apache.hama.GroomServerRunner' + BSP_OPTS="$BSP_OPTS $BSP_GROOMSERVER_OPTS" +elif [ "$COMMAND" = "zookeeper" ] ; then + CLASS='org.apache.hama.ZooKeeperRunner' +elif [ "$COMMAND" = "job" ] ; then + CLASS='org.apache.hama.bsp.BSPJobClient' +elif [ "$COMMAND" = "jar" ] ; then + CLASS=org.apache.hama.util.RunJar + BSP_OPTS="$BSP_OPTS" +else + CLASS=$COMMAND +fi + +# cygwin path translation +if $cygwin; then + CLASSPATH=`cygpath -p -w "$CLASSPATH"` + HAMA_HOME=`cygpath -w "$HAMA_HOME"` + HAMA_LOG_DIR=`cygpath -w "$HAMA_LOG_DIR"` + TOOL_PATH=`cygpath -p -w "$TOOL_PATH"` +fi + +# cygwin path translation +if $cygwin; then + JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` +fi + +HAMA_OPTS="$HAMA_OPTS -Dhama.log.dir=$HAMA_LOG_DIR" +HAMA_OPTS="$HAMA_OPTS -Dhama.log.file=$HAMA_LOGFILE" +HAMA_OPTS="$HAMA_OPTS -Dhama.home.dir=$HAMA_HOME" +HAMA_OPTS="$HAMA_OPTS -Dhama.id.str=$HAMA_IDENT_STRING" +HAMA_OPTS="$HAMA_OPTS -Dhama.root.logger=${HAMA_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + HAMA_OPTS="$HAMA_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" +fi +HAMA_OPTS="$HAMA_OPTS -Dhama.policy.file=$HAMA_POLICYFILE" + +# run it +exec "$JAVA" $JAVA_HEAP_MAX $HAMA_OPTS -classpath "$CLASSPATH" $CLASS "$@" Index: core/bin/start-bspd.sh =================================================================== --- core/bin/start-bspd.sh (revision 0) +++ core/bin/start-bspd.sh (revision 0) @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start hama map reduce daemons. Run this on master node. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hama-config.sh + +# start bsp daemons +# start zookeeper first to minimize connection errors at startup +"$bin"/hama-daemons.sh --config "${HAMA_CONF_DIR}" start zookeeper +"$bin"/hama-daemon.sh --config $HAMA_CONF_DIR start bspmaster +"$bin"/hama-daemons.sh --config $HAMA_CONF_DIR start groom Index: core/pom.xml =================================================================== --- core/pom.xml (revision 0) +++ core/pom.xml (revision 0) @@ -0,0 +1,127 @@ + + + + + + org.apache.hama + hama-parent + 0.4.0-incubating-SNAPSHOT + + + 4.0.0 + org.apache.hama + hama-core + Apache Hama Core + 0.4.0-incubating-SNAPSHOT + jar + + + scm:svn:http://svn.apache.org/repos/asf/incubator/hama/core + + + scm:svn:https://svn.apache.org/repos/asf/incubator/hama/core + + + + + + commons-logging + commons-logging + + + commons-cli + commons-cli + + + org.mortbay.jetty + jetty + + + org.apache.tomcat + servlet-api + + + org.mortbay.jetty + jetty-annotations + + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + jsp-api-2.1 + + + org.mortbay.jetty + jsp-2.1 + + + org.apache.hadoop + hadoop-core + + + org.apache.hadoop + hadoop-test + + + junit + junit + test + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + provided + + + org.apache.hadoop + zookeeper + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + ${basedir}/target/lib + false + true + true + + + + + + hama-core-${project.version} + + Index: pom.xml =================================================================== --- pom.xml (revision 1152138) +++ pom.xml (working copy) @@ -16,22 +16,21 @@ --> - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + org.apache apache 8 - + 4.0.0 org.apache.hama - hama-core - Apache Hama + hama-parent + Apache Hama parent POM 0.4.0-incubating-SNAPSHOT - Apache Hama http://incubator.apache.org/hama - jar + pom 3.0 @@ -84,7 +83,7 @@ scm:svn:https://svn.apache.org/repos/asf/incubator/hama - + 1.1.1 @@ -99,95 +98,98 @@ 1.7.1 - - - commons-logging - commons-logging - ${commons-logging.version} - - - commons-cli - commons-cli - ${commons-cli.version} - - - org.mortbay.jetty - jetty - ${jetty.version} - - - org.apache.tomcat - servlet-api - ${servlet-api.version} - - - org.mortbay.jetty - jetty-annotations - ${jetty.version} - - - org.mortbay.jetty - jetty-util - ${jetty.version} - - - org.mortbay.jetty - jsp-api-2.1 - ${jetty.version} - - - org.mortbay.jetty - jsp-2.1 - ${jetty.version} - - - org.apache.ant - ant - ${ant.version} - - - org.apache.ant - ant-launcher - ${ant.version} - - - org.apache.hadoop - hadoop-core - ${hadoop.version} - - - org.apache.hadoop - hadoop-test - ${hadoop.version} - - - junit - junit - ${junit.version} - test - - - org.slf4j - slf4j-log4j12 - ${slf4j-log4j12.version} - - - log4j - log4j - ${log4j.version} - provided - - - org.apache.hadoop - zookeeper - ${zookeeper.version} - - - - + + + + commons-logging + commons-logging + ${commons-logging.version} + + + commons-cli + commons-cli + ${commons-cli.version} + + + org.mortbay.jetty + jetty + ${jetty.version} + + + org.apache.tomcat + servlet-api + ${servlet-api.version} + + + org.mortbay.jetty + jetty-annotations + ${jetty.version} + + + org.mortbay.jetty + jetty-util + ${jetty.version} + + + org.mortbay.jetty + jsp-api-2.1 + ${jetty.version} + + + org.mortbay.jetty + jsp-2.1 + ${jetty.version} + + + org.apache.ant + ant + ${ant.version} + + + org.apache.ant + ant-launcher + ${ant.version} + + + org.apache.hadoop + hadoop-core + ${hadoop.version} + + + org.apache.hadoop + hadoop-test + ${hadoop.version} + + + junit + junit + ${junit.version} + + + org.slf4j + slf4j-log4j12 + ${slf4j-log4j12.version} + + + log4j + log4j + ${log4j.version} + + + org.apache.hadoop + zookeeper + ${zookeeper.version} + + + + + + core + examples + site + + - ${source.dir} - src/test org.apache.maven.plugins @@ -210,115 +212,4 @@ - - - default - - true - - - src/java - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${basedir}/lib - false - true - true - - - - - - hama-core-${project.version} - - - - examples - - src/examples - - - hama-examples-${project.version} - - - - maven-surefire-plugin - - true - - - - - - - maven-shade-plugin - - - package - - shade - - - - - org.apache.hama.examples.ExampleDriver - - - - - - - - - - - org.apache.hama - hama-core - ${project.version} - - - - - forrest - - - - maven-antrun-plugin - - - package - - - - - - - - - - - - run - - - - - - - - Property changes on: examples ___________________________________________________________________ Added: svn:ignore + *.iml target Index: examples/src/main/java/org/apache/hama/examples/PiEstimator.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/PiEstimator.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/PiEstimator.java (revision 0) @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.bsp.BSP; +import org.apache.hama.bsp.BSPJob; +import org.apache.hama.bsp.BSPJobClient; +import org.apache.hama.bsp.BSPPeer; +import org.apache.hama.bsp.ClusterStatus; +import org.apache.hama.bsp.DoubleMessage; +import org.apache.zookeeper.KeeperException; + +public class PiEstimator { + private static String MASTER_TASK = "master.task."; + private static Path TMP_OUTPUT = new Path("/tmp/pi-example/output"); + + public static class MyEstimator extends BSP { + public static final Log LOG = LogFactory.getLog(MyEstimator.class); + private Configuration conf; + private String masterTask; + private static final int iterations = 10000; + + public void bsp(BSPPeer bspPeer) throws IOException, + KeeperException, InterruptedException { + int in = 0, out = 0; + for (int i = 0; i < iterations; i++) { + double x = 2.0 * Math.random() - 1.0, y = 2.0 * Math.random() - 1.0; + if ((Math.sqrt(x * x + y * y) < 1.0)) { + in++; + } else { + out++; + } + } + + double data = 4.0 * (double) in / (double) iterations; + DoubleMessage estimate = new DoubleMessage(bspPeer.getPeerName(), data); + + bspPeer.send(masterTask, estimate); + bspPeer.sync(); + + if (bspPeer.getPeerName().equals(masterTask)) { + double pi = 0.0; + int numPeers = bspPeer.getNumCurrentMessages(); + DoubleMessage received; + while ((received = (DoubleMessage) bspPeer.getCurrentMessage()) != null) { + pi += received.getData(); + } + + pi = pi / numPeers; + writeResult(pi); + } + } + + private void writeResult(double pi) throws IOException { + FileSystem fileSys = FileSystem.get(conf); + + SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, + TMP_OUTPUT, DoubleWritable.class, DoubleWritable.class, + CompressionType.NONE); + writer.append(new DoubleWritable(pi), new DoubleWritable(0)); + writer.close(); + } + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration conf) { + this.conf = conf; + this.masterTask = conf.get(MASTER_TASK); + } + + } + + private static void initTempDir(FileSystem fileSys) throws IOException { + if (fileSys.exists(TMP_OUTPUT)) { + fileSys.delete(TMP_OUTPUT, true); + } + } + + private static void printOutput(FileSystem fileSys, HamaConfiguration conf) + throws IOException { + SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, TMP_OUTPUT, + conf); + DoubleWritable output = new DoubleWritable(); + DoubleWritable zero = new DoubleWritable(); + reader.next(output, zero); + reader.close(); + + System.out.println("Estimated value of PI is " + output); + } + + public static void main(String[] args) throws InterruptedException, + IOException, ClassNotFoundException { + // BSP job configuration + HamaConfiguration conf = new HamaConfiguration(); + + BSPJob bsp = new BSPJob(conf, PiEstimator.class); + // Set the job name + bsp.setJobName("Pi Estimation Example"); + bsp.setBspClass(MyEstimator.class); + + BSPJobClient jobClient = new BSPJobClient(conf); + ClusterStatus cluster = jobClient.getClusterStatus(true); + + if (args.length > 0) { + bsp.setNumBspTask(Integer.parseInt(args[0])); + } else { + // Set to maximum + bsp.setNumBspTask(cluster.getMaxTasks()); + } + + // Choose one as a master + for (String peerName : cluster.getActiveGroomNames().values()) { + conf.set(MASTER_TASK, peerName); + break; + } + + FileSystem fileSys = FileSystem.get(conf); + initTempDir(fileSys); + + long startTime = System.currentTimeMillis(); + + if (bsp.waitForCompletion(true)) { + printOutput(fileSys, conf); + + System.out.println("Job Finished in " + + (double) (System.currentTimeMillis() - startTime) / 1000.0 + + " seconds"); + } + } +} Index: examples/src/main/java/org/apache/hama/examples/graph/PageRankBase.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/graph/PageRankBase.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/graph/PageRankBase.java (revision 0) @@ -0,0 +1,236 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples.graph; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.bsp.BSP; +import org.apache.hama.bsp.BSPPeer; + +public abstract class PageRankBase extends BSP { + public static final Log LOG = LogFactory.getLog(PageRankBase.class); + + protected static int MAX_ITERATIONS = 30; + protected static final String MASTER_TASK = "master.groom"; + protected static double ALPHA; + protected static int numOfVertices; + protected static double DAMPING_FACTOR = 0.85; + protected static double EPSILON = 0.001; + + static HashMap> mapAdjacencyList( + Configuration conf, BSPPeer peer) throws FileNotFoundException, + IOException { + FileSystem fs = FileSystem.get(conf); + HashMap> adjacencyList = new HashMap>(); + Path p = new Path(conf.get("in.path." + peer.getPeerName().split(":")[0])); + LOG.info(p.toString()); + SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf); + Text key = new Text(); + Text value = new Text(); + while (reader.next(key, value)) { + PageRankVertex k = new PageRankVertex(key.toString()); + PageRankVertex v = new PageRankVertex(value.toString()); + if (!adjacencyList.containsKey(k)) { + adjacencyList.put(k, new LinkedList()); + adjacencyList.get(k).add(v); + } else { + adjacencyList.get(k).add(v); + } + } + reader.close(); + return adjacencyList; + } + + static HamaConfiguration partitionExample(Path out, HamaConfiguration conf) + throws IOException { + + String[] groomNames = conf.get(ShortestPaths.BSP_PEERS).split(";"); + int sizeOfCluster = groomNames.length; + + // setup the paths where the grooms can find their input + List partPaths = new ArrayList(sizeOfCluster); + List writers = new ArrayList( + sizeOfCluster); + FileSystem fs = FileSystem.get(conf); + for (String entry : groomNames) { + partPaths.add(new Path(out.getParent().toString() + Path.SEPARATOR + + ShortestPaths.PARTED + Path.SEPARATOR + + entry.split(ShortestPaths.NAME_VALUE_SEPARATOR)[0])); + } + // create a seq writer for that + for (Path p : partPaths) { + // LOG.info(p.toString()); + fs.delete(p, true); + writers.add(SequenceFile + .createWriter(fs, conf, p, Text.class, Text.class)); + } + + /** + * 1:twitter.com
+ * 2:google.com
+ * 3:facebook.com
+ * 4:yahoo.com
+ * 5:nasa.gov
+ * 6:stackoverflow.com
+ * 7:youtube.com + */ + + String[] realNames = new String[] { null, "twitter.com", "google.com", + "facebook.com", "yahoo.com", "nasa.gov", "stackoverflow.com", + "youtube.com" }; + + String[] lineArray = new String[] { "1;2;3", "2", "3;1;2;5", "4;5;6", + "5;4;6", "6;4", "7;2;4" }; + + int numLines = 0; + for (String line : lineArray) { + String[] arr = line.split(";"); + String vId = arr[0]; + int indexKey = Integer.valueOf(vId); + LinkedList list = new LinkedList(); + for (int i = 0; i < arr.length; i++) { + int index = Integer.valueOf(arr[i]); + list.add(realNames[index]); + } + + int mod = Math.abs(realNames[indexKey].hashCode() % sizeOfCluster); + for (String value : list) { + writers.get(mod).append(new Text(realNames[indexKey]), new Text(value)); + } + numLines++; + } + + for (SequenceFile.Writer w : writers) + w.close(); + + for (Path p : partPaths) { + conf.set("in.path." + p.getName(), p.toString()); + } + conf.set("num.vertices", "" + numLines); + + return conf; + } + + static HamaConfiguration partitionTextFile(Path in, HamaConfiguration conf) + throws IOException { + + String[] groomNames = conf.get(ShortestPaths.BSP_PEERS).split(";"); + int sizeOfCluster = groomNames.length; + + // setup the paths where the grooms can find their input + List partPaths = new ArrayList(sizeOfCluster); + List writers = new ArrayList( + sizeOfCluster); + FileSystem fs = FileSystem.get(conf); + for (String entry : groomNames) { + partPaths.add(new Path(in.getParent().toString() + Path.SEPARATOR + + ShortestPaths.PARTED + Path.SEPARATOR + + entry.split(ShortestPaths.NAME_VALUE_SEPARATOR)[0])); + } + // create a seq writer for that + for (Path p : partPaths) { + // LOG.info(p.toString()); + fs.delete(p, true); + writers.add(SequenceFile + .createWriter(fs, conf, p, Text.class, Text.class)); + } + + BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(in))); + + String line = null; + long numLines = 0; + while ((line = br.readLine()) != null) { + String[] arr = line.split("\t"); + String vId = arr[0]; + LinkedList list = new LinkedList(); + for (int i = 0; i < arr.length; i++) { + list.add(arr[i]); + } + + int mod = Math.abs(vId.hashCode() % sizeOfCluster); + for (String value : list) { + writers.get(mod).append(new Text(vId), new Text(value)); + } + numLines++; + } + + for (SequenceFile.Writer w : writers) + w.close(); + + for (Path p : partPaths) { + conf.set("in.path." + p.getName(), p.toString()); + } + conf.set("num.vertices", "" + numLines); + + return conf; + } + + static void savePageRankMap(BSPPeer peer, Configuration conf, + Map tentativePagerank) throws IOException { + FileSystem fs = FileSystem.get(conf); + Path outPath = new Path(conf.get("out.path") + Path.SEPARATOR + "temp" + + Path.SEPARATOR + + peer.getPeerName().split(ShortestPaths.NAME_VALUE_SEPARATOR)[0]); + fs.delete(outPath, true); + final SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, + outPath, Text.class, DoubleWritable.class); + for (Entry row : tentativePagerank.entrySet()) { + out.append(new Text(row.getKey().getUrl()), new DoubleWritable(row + .getValue())); + } + LOG.info("Closing..."); + out.close(); + LOG.info("Closed!"); + } + + static void printOutput(FileSystem fs, Configuration conf) throws IOException { + LOG.info("-------------------- RESULTS --------------------"); + FileStatus[] stati = fs.listStatus(new Path(conf.get("out.path") + + Path.SEPARATOR + "temp")); + for (FileStatus status : stati) { + Path path = status.getPath(); + SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); + Text key = new Text(); + DoubleWritable value = new DoubleWritable(); + while (reader.next(key, value)) { + LOG.info(key.toString() + " | " + value.get()); + } + reader.close(); + } + } + +} Index: examples/src/main/java/org/apache/hama/examples/graph/ShortestPathsGraphLoader.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/graph/ShortestPathsGraphLoader.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/graph/ShortestPathsGraphLoader.java (revision 0) @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples.graph; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public class ShortestPathsGraphLoader { + + static Map> loadGraph() { + + Map> adjacencyList = new HashMap>(); + String[] cities = new String[] { "Frankfurt", "Mannheim", "Wuerzburg", + "Stuttgart", "Kassel", "Karlsruhe", "Erfurt", "Nuernberg", "Augsburg", + "Muenchen" }; + + int id = 1; + for (String city : cities) { + if (city.equals("Frankfurt")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(85, "Mannheim")); + list.add(new ShortestPathVertex(173, "Kassel")); + list.add(new ShortestPathVertex(217, "Wuerzburg")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Stuttgart")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(183, "Nuernberg")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Kassel")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(502, "Muenchen")); + list.add(new ShortestPathVertex(173, "Frankfurt")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Erfurt")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(186, "Wuerzburg")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Wuerzburg")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(217, "Frankfurt")); + list.add(new ShortestPathVertex(168, "Erfurt")); + list.add(new ShortestPathVertex(103, "Nuernberg")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Mannheim")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(80, "Karlsruhe")); + list.add(new ShortestPathVertex(85, "Frankfurt")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Karlsruhe")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(250, "Augsburg")); + list.add(new ShortestPathVertex(80, "Mannheim")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Augsburg")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(250, "Karlsruhe")); + list.add(new ShortestPathVertex(84, "Muenchen")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Nuernberg")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(183, "Stuttgart")); + list.add(new ShortestPathVertex(167, "Muenchen")); + list.add(new ShortestPathVertex(103, "Wuerzburg")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } else if (city.equals("Muenchen")) { + List list = new LinkedList(); + list.add(new ShortestPathVertex(167, "Nuernberg")); + list.add(new ShortestPathVertex(173, "Kassel")); + list.add(new ShortestPathVertex(84, "Augsburg")); + adjacencyList.put(new ShortestPathVertex(0, city), list); + } + id++; + } + return adjacencyList; + } + +} Index: examples/src/main/java/org/apache/hama/examples/graph/PageRankVertex.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/graph/PageRankVertex.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/graph/PageRankVertex.java (revision 0) @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples.graph; + +public class PageRankVertex { + + private final int id; + private final String url; + + public PageRankVertex(String url) { + super(); + this.id = url.hashCode(); + this.url = url; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + id; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + PageRankVertex other = (PageRankVertex) obj; + if (id != other.id) + return false; + return true; + } + + @Override + public String toString() { + return url; + } + + public int getId() { + return id; + } + + public String getUrl() { + return url; + } + +} Index: examples/src/main/java/org/apache/hama/examples/graph/ShortestPathVertex.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/graph/ShortestPathVertex.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/graph/ShortestPathVertex.java (revision 0) @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples.graph; + +public final class ShortestPathVertex { + + private final int id; + private final String name; + private final int weight; + private Integer cost; + + public ShortestPathVertex(int weight, String name) { + super(); + this.id = name.hashCode(); + this.weight = weight; + this.name = name; + } + + public ShortestPathVertex(int weight, String name, Integer cost) { + super(); + this.id = name.hashCode(); + this.weight = weight; + this.name = name; + this.cost = cost; + } + + public String getName() { + return name; + } + + public Integer getCost() { + return cost; + } + + public void setCost(Integer cost) { + this.cost = cost; + } + + public int getId() { + return id; + } + + public int getWeight() { + return weight; + } + + @Override + public String toString() { + return name; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ShortestPathVertex other = (ShortestPathVertex) obj; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + return true; + } + +} Index: examples/src/main/java/org/apache/hama/examples/graph/ShortestPathsBase.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/graph/ShortestPathsBase.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/graph/ShortestPathsBase.java (revision 0) @@ -0,0 +1,277 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples.graph; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.bsp.BSP; +import org.apache.hama.bsp.BSPPeer; + +public abstract class ShortestPathsBase extends BSP { + + public static final String BSP_PEERS = "bsp.peers"; + public static final String SHORTEST_PATHS_START_VERTEX_ID = "shortest.paths.start.vertex.id"; + public static final String PARTED = "parted"; + public static final String IN_PATH = "in.path."; + public static final String OUT_PATH = "out.path"; + public static final String NAME_VALUE_SEPARATOR = ":"; + public static final String MASTER_TASK = "master.groom"; + + /** + * When finished we just writing a sequencefile of the vertex name and the + * cost. + * + * @param peer The peer we got through the BSP method. + * @param adjacencyList + * @throws IOException + */ + static void saveVertexMap(Configuration conf, BSPPeer peer, + Map> adjacencyList) + throws IOException { + Path outPath = new Path(conf.get(OUT_PATH) + Path.SEPARATOR + + peer.getPeerName().split(":")[0]); + FileSystem fs = FileSystem.get(conf); + fs.delete(outPath, true); + final SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, + outPath, Text.class, IntWritable.class); + for (ShortestPathVertex vertex : adjacencyList.keySet()) { + out.append(new Text(vertex.getName()), new IntWritable(vertex.getCost())); + } + out.close(); + } + + /** + * Just a reader of the vertexMap in DFS. Output going to STDOUT. + * + * @param fs + * @param conf + * @throws IOException + */ + static void printOutput(FileSystem fs, Configuration conf) throws IOException { + System.out.println("-------------------- RESULTS --------------------"); + FileStatus[] stati = fs.listStatus(new Path(conf.get(OUT_PATH))); + for (FileStatus status : stati) { + Path path = status.getPath(); + SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); + Text key = new Text(); + IntWritable value = new IntWritable(); + while (reader.next(key, value)) { + System.out.println(key.toString() + " | " + value.get()); + } + reader.close(); + } + } + + /** + * + * The adjacencylist contains two text fields on each line. The key component + * is the name of a vertex, the value is a ":" separated Text field that + * contains the name of the adjacent vertex leftmost and the weight on the + * rightmost side. + * + *
+   *    K               V 
+ * Vertex[Text] AdjacentVertex : Weight [Text] + *
+ * + * @param adjacencyList + * @param vertexLookupMap + */ + static void mapAdjacencyList(Configuration conf, BSPPeer peer, + Map> adjacencyList, + Map vertexLookupMap) + throws FileNotFoundException, IOException { + FileSystem fs = FileSystem.get(conf); + Path p = new Path(conf.get(IN_PATH + + peer.getPeerName().split(NAME_VALUE_SEPARATOR)[0])); + SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf); + Text key = new Text(); // name of the vertex + Text value = new Text(); // name of the adjacent vertex : weight + while (reader.next(key, value)) { + // a key vertex has weight 0 to itself + ShortestPathVertex keyVertex = new ShortestPathVertex(0, key.toString(), + Integer.MAX_VALUE); + String[] nameWeight = value.toString().split(NAME_VALUE_SEPARATOR); + if (!adjacencyList.containsKey(keyVertex)) { + LinkedList list = new LinkedList(); + list.add(new ShortestPathVertex(Integer.valueOf(nameWeight[1]), + nameWeight[0], Integer.MAX_VALUE)); + adjacencyList.put(keyVertex, list); + vertexLookupMap.put(keyVertex.getName(), keyVertex); + } else { + adjacencyList.get(keyVertex).add( + new ShortestPathVertex(Integer.valueOf(nameWeight[1]), + nameWeight[0], Integer.MAX_VALUE)); + } + } + reader.close(); + } + + /** + * Partitioning for in memory adjacency lists. + * + * @param adjacencyList + * @param status + * @param conf + * @return + * @throws IOException + */ + static HamaConfiguration partition( + Map> adjacencyList, + HamaConfiguration conf) throws IOException { + + String[] groomNames = conf.get(BSP_PEERS).split(";"); + + int sizeOfCluster = groomNames.length; + + // setup the paths where the grooms can find their input + List partPaths = new ArrayList(sizeOfCluster); + List writers = new ArrayList( + sizeOfCluster); + FileSystem fs = FileSystem.get(conf); + Path fileToPartition = new Path(conf.get(OUT_PATH)); + for (String entry : groomNames) { + partPaths.add(new Path(fileToPartition.getParent().toString() + + Path.SEPARATOR + PARTED + Path.SEPARATOR + + entry.split(NAME_VALUE_SEPARATOR)[0])); + } + // create a seq writer for that + for (Path p : partPaths) { + // System.out.println(p.toString()); + fs.delete(p, true); + writers.add(SequenceFile + .createWriter(fs, conf, p, Text.class, Text.class)); + } + + for (Entry> entry : adjacencyList + .entrySet()) { + // a key vertex has weight 0 to itself + ShortestPathVertex keyVertex = entry.getKey(); + // just mod the id + int mod = Math.abs(keyVertex.getId() % sizeOfCluster); + // append it to the right sequenceFile + for (ShortestPathVertex value : entry.getValue()) + writers.get(mod) + .append( + new Text(keyVertex.getName()), + new Text(value.getName() + NAME_VALUE_SEPARATOR + + value.getWeight())); + } + + for (SequenceFile.Writer w : writers) + w.close(); + + for (Path p : partPaths) { + conf.set(IN_PATH + p.getName(), p.toString()); + } + return conf; + } + + /** + * Partitioning for sequencefile partitioned adjacency lists. + * + * The adjacencylist contains two text fields on each line. The key component + * is the name of a vertex, the value is a ":" separated Text field that + * contains the name of the adjacent vertex leftmost and the weight on the + * rightmost side. + * + *
+   *    K               V 
+ * Vertex[Text] AdjacentVertex : Weight [Text] + *
+ * + * @param fileToPartition + * @param status + * @param conf + * @return + * @throws IOException + */ + static HamaConfiguration partition(Path fileToPartition, + HamaConfiguration conf, boolean skipPartitioning) throws IOException { + + String[] groomNames = conf.get(BSP_PEERS).split(";"); + int sizeOfCluster = groomNames.length; + + // setup the paths where the grooms can find their input + List partPaths = new ArrayList(sizeOfCluster); + List writers = new ArrayList( + sizeOfCluster); + FileSystem fs = FileSystem.get(conf); + for (String entry : groomNames) { + partPaths.add(new Path(fileToPartition.getParent().toString() + + Path.SEPARATOR + PARTED + Path.SEPARATOR + + entry.split(NAME_VALUE_SEPARATOR)[0])); + } + + if (!skipPartitioning) { + + // create a seq writer for that + for (Path p : partPaths) { + // System.out.println(p.toString()); + fs.delete(p, true); + writers.add(SequenceFile.createWriter(fs, conf, p, Text.class, + Text.class)); + } + + // parse our file + if (!fs.exists(fileToPartition)) + throw new FileNotFoundException("File " + fileToPartition + + " wasn't found!"); + + SequenceFile.Reader reader = new SequenceFile.Reader(fs, fileToPartition, + conf); + Text key = new Text(); // name of the vertex + Text value = new Text(); // name of the adjacent vertex : weight + + while (reader.next(key, value)) { + // a key vertex has weight 0 to itself + ShortestPathVertex keyVertex = new ShortestPathVertex(0, key.toString()); + // just mod the id + int mod = Math.abs(keyVertex.getId() % sizeOfCluster); + // append it to the right sequenceFile + writers.get(mod).append(new Text(keyVertex.getName()), new Text(value)); + } + + reader.close(); + + for (SequenceFile.Writer w : writers) + w.close(); + } + + for (Path p : partPaths) { + conf.set(IN_PATH + p.getName(), p.toString()); + } + + return conf; + } + +} Index: examples/src/main/java/org/apache/hama/examples/graph/PageRank.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/graph/PageRank.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/graph/PageRank.java (revision 0) @@ -0,0 +1,251 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples.graph; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.bsp.BSPJob; +import org.apache.hama.bsp.BSPJobClient; +import org.apache.hama.bsp.BSPPeer; +import org.apache.hama.bsp.ClusterStatus; +import org.apache.hama.bsp.DoubleMessage; +import org.apache.zookeeper.KeeperException; + +public class PageRank extends PageRankBase { + public static final Log LOG = LogFactory.getLog(PageRank.class); + + private Configuration conf; + + private HashMap> adjacencyList; + private final HashMap lookupMap = new HashMap(); + private final HashMap tentativePagerank = new HashMap(); + // backup of the last pagerank to determine the error + private final HashMap lastTentativePagerank = new HashMap(); + private String[] peerNames; + + @Override + public void bsp(BSPPeer peer) throws IOException, KeeperException, + InterruptedException { + String master = conf.get(MASTER_TASK); + // setup the datasets + adjacencyList = PageRankBase.mapAdjacencyList(getConf(), peer); + // init the pageranks to 1/n where n is the number of all vertices + for (PageRankVertex vertex : adjacencyList.keySet()) { + tentativePagerank.put(vertex, Double.valueOf(1.0 / numOfVertices)); + lookupMap.put(vertex.getUrl(), vertex); + } + // while the error not converges against epsilon do the pagerank stuff + double error = 1.0; + int iteration = 0; + // if MAX_ITERATIONS are set to 0, ignore the iterations and just go + // with the error + while ((MAX_ITERATIONS > 0 && iteration < MAX_ITERATIONS) + || error >= EPSILON) { + peer.sync(); + + if (iteration >= 1) { + // copy the old pagerank to the backup + copyTentativePageRankToBackup(); + // sum up all incoming messages for a vertex + HashMap sumMap = new HashMap(); + DoubleMessage msg = null; + while ((msg = (DoubleMessage) peer.getCurrentMessage()) != null) { + PageRankVertex k = lookupMap.get(msg.getTag()); + if (!sumMap.containsKey(k)) { + sumMap.put(k, msg.getData()); + } else { + sumMap.put(k, msg.getData() + sumMap.get(k)); + } + } + // pregel formula: + // ALPHA = 0.15 / NumVertices() + // P(i) = ALPHA + 0.85 * sum + for (Entry entry : sumMap.entrySet()) { + tentativePagerank.put(entry.getKey(), ALPHA + + (entry.getValue() * DAMPING_FACTOR)); + } + + // determine the error and send this to the master + double err = determineError(); + error = broadcastError(peer, master, err); + } + // in every step send the tentative pagerank of a vertex to its + // adjacent vertices + for (PageRankVertex vertex : adjacencyList.keySet()) { + sendMessageToNeighbors(peer, vertex); + } + + iteration++; + } + + // Clears all queues entries. + peer.clear(); + // finally save the chunk of pageranks + PageRankBase.savePageRankMap(peer, conf, lastTentativePagerank); + LOG.info("Finished with iteration " + iteration + "!"); + } + + private double broadcastError(BSPPeer peer, String master, + double error) throws IOException, KeeperException, InterruptedException { + peer.send(master, new DoubleMessage("", error)); + peer.sync(); + if (peer.getPeerName().equals(master)) { + double errorSum = 0.0; + int count = 0; + DoubleMessage message; + while ((message = (DoubleMessage) peer.getCurrentMessage()) != null) { + errorSum += message.getData(); + count++; + } + double avgError = errorSum / (double) count; + // LOG.info("Average error: " + avgError); + for (String name : peer.getAllPeerNames()) { + peer.send(name, new DoubleMessage("", avgError)); + } + } + + peer.sync(); + DoubleMessage message = (DoubleMessage) peer.getCurrentMessage(); + return message.getData(); + } + + private double determineError() { + double error = 0.0; + for (Entry entry : tentativePagerank.entrySet()) { + error += Math.abs(lastTentativePagerank.get(entry.getKey()) + - entry.getValue()); + } + return error; + } + + private void copyTentativePageRankToBackup() { + for (Entry entry : tentativePagerank.entrySet()) { + lastTentativePagerank.put(entry.getKey(), entry.getValue()); + } + } + + private void sendMessageToNeighbors(BSPPeer peer, PageRankVertex v) + throws IOException { + List outgoingEdges = adjacencyList.get(v); + for (PageRankVertex adjacent : outgoingEdges) { + int mod = Math.abs(adjacent.getId() % peerNames.length); + // send a message of the tentative pagerank divided by the size of + // the outgoing edges to all adjacents + peer.send(peerNames[mod], new DoubleMessage(adjacent.getUrl(), + tentativePagerank.get(v) / outgoingEdges.size())); + } + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + numOfVertices = Integer.parseInt(conf.get("num.vertices")); + DAMPING_FACTOR = Double.parseDouble(conf.get("damping.factor")); + ALPHA = (1 - DAMPING_FACTOR) / (double) numOfVertices; + EPSILON = Double.parseDouble(conf.get("epsilon.error")); + MAX_ITERATIONS = Integer.parseInt(conf.get("max.iterations")); + peerNames = conf.get(ShortestPaths.BSP_PEERS).split(";"); + } + + @Override + public Configuration getConf() { + return conf; + } + + public static void printUsage() { + System.out.println("PageRank Example:"); + System.out + .println(" "); + } + + public static void main(String[] args) throws IOException, + InterruptedException, ClassNotFoundException { + if (args.length == 0) { + printUsage(); + System.exit(-1); + } + + HamaConfiguration conf = new HamaConfiguration(new Configuration()); + // set the defaults + conf.set("damping.factor", "0.85"); + conf.set("epsilon.error", "0.000001"); + + if (args.length < 2) { + System.out.println("You have to provide a damping factor and an error!"); + System.out.println("Try using 0.85 0.001 as parameter!"); + System.exit(-1); + } else { + conf.set("damping.factor", args[0]); + conf.set("epsilon.error", args[1]); + LOG.info("Set damping factor to " + args[0]); + LOG.info("Set epsilon error to " + args[1]); + if (args.length > 2) { + conf.set("out.path", args[2]); + LOG.info("Set output path to " + args[2]); + if (args.length == 4) { + conf.set("in.path", args[3]); + LOG.info("Using custom input at " + args[3]); + } else { + LOG.info("Running default example graph!"); + } + } else { + conf.set("out.path", "pagerank/output"); + LOG.info("Set output path to default of pagerank/output!"); + } + } + + BSPJobClient jobClient = new BSPJobClient(conf); + ClusterStatus cluster = jobClient.getClusterStatus(true); + StringBuilder sb = new StringBuilder(); + for (String peerName : cluster.getActiveGroomNames().values()) { + conf.set(MASTER_TASK, peerName); + sb.append(peerName + ";"); + } + + // put every peer into the configuration + conf.set(ShortestPaths.BSP_PEERS, sb.toString()); + // leave the iterations on default + conf.set("max.iterations", "0"); + + if (conf.get("in.path") == null) { + conf = PageRankBase + .partitionExample(new Path(conf.get("out.path")), conf); + } else { + conf = PageRankBase + .partitionTextFile(new Path(conf.get("in.path")), conf); + } + + BSPJob job = new BSPJob(conf); + job.setNumBspTask(cluster.getGroomServers()); + job.setBspClass(PageRank.class); + job.setJarByClass(PageRank.class); + job.setJobName("Pagerank"); + if (job.waitForCompletion(true)) { + PageRankBase.printOutput(FileSystem.get(conf), conf); + } + } +} Index: examples/src/main/java/org/apache/hama/examples/graph/ShortestPaths.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/graph/ShortestPaths.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/graph/ShortestPaths.java (revision 0) @@ -0,0 +1,254 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples.graph; + +import java.io.IOException; +import java.util.Deque; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.bsp.BSPJob; +import org.apache.hama.bsp.BSPJobClient; +import org.apache.hama.bsp.BSPPeer; +import org.apache.hama.bsp.BooleanMessage; +import org.apache.hama.bsp.ClusterStatus; +import org.apache.hama.bsp.IntegerMessage; +import org.apache.hama.examples.RandBench; +import org.apache.zookeeper.KeeperException; + +public class ShortestPaths extends ShortestPathsBase { + public static final Log LOG = LogFactory.getLog(ShortestPaths.class); + + private Configuration conf; + private Map> adjacencyList = new HashMap>(); + private Map vertexLookupMap = new HashMap(); + private String[] peerNames; + + @Override + public void bsp(BSPPeer peer) throws IOException, KeeperException, + InterruptedException { + LOG.info("Mapping graph into ram..."); + // map our input into ram + mapAdjacencyList(conf, peer, adjacencyList, vertexLookupMap); + LOG.info("Finished! Starting graph initialization..."); + // parse the configuration to get the peerNames + parsePeerNames(conf); + // get our master groom + String master = conf.get(MASTER_TASK); + + // initial message bypass + ShortestPathVertex v = vertexLookupMap.get(conf + .get(SHORTEST_PATHS_START_VERTEX_ID)); + if (v != null) { + v.setCost(0); + sendMessageToNeighbors(peer, v); + } + + LOG.info("Finished! Starting main loop..."); + boolean updated = true; + while (updated) { + int updatesMade = 0; + peer.sync(); + + IntegerMessage msg = null; + Deque updatedQueue = new LinkedList(); + while ((msg = (IntegerMessage) peer.getCurrentMessage()) != null) { + ShortestPathVertex vertex = vertexLookupMap.get(msg.getTag()); + // check if we need an distance update + if (vertex.getCost() > msg.getData()) { + updatesMade++; + updatedQueue.add(vertex); + vertex.setCost(msg.getData()); + } + } + // synchonize with all grooms if there were updates + updated = broadcastUpdatesMade(peer, master, updatesMade); + // send updates to the adjacents of the updated vertices + for (ShortestPathVertex vertex : updatedQueue) { + sendMessageToNeighbors(peer, vertex); + } + } + LOG.info("Finished!"); + // finished, finally save our map to DFS. + saveVertexMap(conf, peer, adjacencyList); + } + + /** + * Parses the peer names to fix inconsistency in bsp peer names from context. + * + * @param conf + */ + private void parsePeerNames(Configuration conf) { + peerNames = conf.get(BSP_PEERS).split(";"); + } + + /** + * This method broadcasts to a master groom how many updates were made. He + * simply sums them up and sends a message back to the grooms if sum is + * greater than zero. + * + * @param peer The peer we got through the BSP method. + * @param master The assigned master groom name. + * @param updates How many updates were made? + * @return True if we need another iteration, False if no updates can be made + * anymore. + * @throws IOException + * @throws KeeperException + * @throws InterruptedException + */ + private boolean broadcastUpdatesMade(BSPPeer peer, String master, + int updates) throws IOException, KeeperException, InterruptedException { + peer.send(master, new IntegerMessage(peer.getPeerName(), updates)); + peer.sync(); + if (peer.getPeerName().equals(master)) { + int count = 0; + IntegerMessage message; + while ((message = (IntegerMessage) peer.getCurrentMessage()) != null) { + count += message.getData(); + LOG.info("Received " + message.getData() + " updates from " + + message.getTag() + " in SuperStep " + peer.getSuperstepCount()); + } + + for (String name : peer.getAllPeerNames()) { + peer.send(name, new BooleanMessage("", count > 0 ? true : false)); + } + } + + peer.sync(); + BooleanMessage message = (BooleanMessage) peer.getCurrentMessage(); + return message.getData(); + } + + /** + * This method takes advantage of our partitioning: it uses the vertexID + * (simply hash of the name) to determine the host where the message belongs + * to.
+ * It sends the current cost to the adjacent vertex + the edge weight. If cost + * will be infinity we just going to send infinity, because summing the weight + * will cause an integer overflow resulting in negative weights. + * + * @param peer The peer we got through the BSP method. + * @param id The vertex to all adjacent vertices the new cost has to be send. + * @throws IOException + */ + private void sendMessageToNeighbors(BSPPeer peer, + ShortestPathVertex id) throws IOException { + List outgoingEdges = adjacencyList.get(id); + for (ShortestPathVertex adjacent : outgoingEdges) { + int mod = Math.abs((adjacent.getId() % peer.getAllPeerNames().length)); + peer.send(peerNames[mod], new IntegerMessage(adjacent.getName(), id + .getCost() == Integer.MAX_VALUE ? id.getCost() : id.getCost() + + adjacent.getWeight())); + } + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return conf; + } + + public static void printUsage() { + System.out.println("Single Source Shortest Path Example:"); + System.out + .println(" "); + } + + public static void main(String[] args) throws IOException, + InterruptedException, ClassNotFoundException { + + printUsage(); + + // BSP job configuration + HamaConfiguration conf = new HamaConfiguration(); + conf.set(SHORTEST_PATHS_START_VERTEX_ID, "Frankfurt"); + System.out.println("Setting default start vertex to \"Frankfurt\"!"); + conf.set(OUT_PATH, "sssp/output"); + boolean skipPartitioning = false; + Path adjacencyListPath = null; + + if (args.length > 0) { + conf.set(SHORTEST_PATHS_START_VERTEX_ID, args[0]); + System.out.println("Setting start vertex to " + args[0] + "!"); + + if (args.length > 1) { + conf.set(OUT_PATH, args[1]); + System.out.println("Using new output folder: " + args[1]); + } + + if (args.length > 2) { + adjacencyListPath = new Path(args[2]); + } + + if (args.length > 3) { + skipPartitioning = Boolean.valueOf(args[3]); + } + + } + + Map> adjacencyList = null; + if (adjacencyListPath == null) + adjacencyList = ShortestPathsGraphLoader.loadGraph(); + + BSPJob bsp = new BSPJob(conf, RandBench.class); + // Set the job name + bsp.setJobName("Single Source Shortest Path"); + bsp.setBspClass(ShortestPaths.class); + + // Set the task size as a number of GroomServer + BSPJobClient jobClient = new BSPJobClient(conf); + ClusterStatus cluster = jobClient.getClusterStatus(true); + StringBuilder sb = new StringBuilder(); + for (String peerName : cluster.getActiveGroomNames().values()) { + conf.set(MASTER_TASK, peerName); + sb.append(peerName); + sb.append(";"); + } + LOG.info("Master is: " + conf.get(MASTER_TASK)); + conf.set(BSP_PEERS, sb.toString()); + LOG.info("Starting data partitioning..."); + if (adjacencyList == null) + conf = partition(adjacencyListPath, conf, skipPartitioning); + else + conf = partition(adjacencyList, conf); + LOG.info("Finished!"); + + bsp.setNumBspTask(cluster.getGroomServers()); + + long startTime = System.currentTimeMillis(); + if (bsp.waitForCompletion(true)) { + System.out.println("Job Finished in " + + (double) (System.currentTimeMillis() - startTime) / 1000.0 + + " seconds"); + printOutput(FileSystem.get(conf), conf); + } + } + +} Index: examples/src/main/java/org/apache/hama/examples/SerializePrinting.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/SerializePrinting.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/SerializePrinting.java (revision 0) @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples; + +import java.io.IOException; +import java.util.Date; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.SequenceFile.CompressionType; +import org.apache.hadoop.io.Text; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.bsp.BSP; +import org.apache.hama.bsp.BSPJob; +import org.apache.hama.bsp.BSPJobClient; +import org.apache.hama.bsp.BSPPeer; +import org.apache.hama.bsp.ClusterStatus; +import org.apache.zookeeper.KeeperException; + +public class SerializePrinting { + private static String TMP_OUTPUT = "/tmp/test-example/"; + + public static class HelloBSP extends BSP { + public static final Log LOG = LogFactory.getLog(HelloBSP.class); + private Configuration conf; + private final static int PRINT_INTERVAL = 1000; + private FileSystem fileSys; + private int num; + + public void bsp(BSPPeer bspPeer) throws IOException, + KeeperException, InterruptedException { + + int i = 0; + for (String otherPeer : bspPeer.getAllPeerNames()) { + String peerName = bspPeer.getPeerName(); + if (peerName.equals(otherPeer)) { + writeLogToFile(peerName, i); + } + + Thread.sleep(PRINT_INTERVAL); + bspPeer.sync(); + i++; + } + } + + private void writeLogToFile(String string, int i) throws IOException { + SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, + new Path(TMP_OUTPUT + i), LongWritable.class, Text.class, + CompressionType.NONE); + writer.append(new LongWritable(System.currentTimeMillis()), new Text( + "Hello BSP from " + (i + 1) + " of " + num + ": " + string)); + writer.close(); + } + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration conf) { + this.conf = conf; + num = Integer.parseInt(conf.get("bsp.peers.num")); + try { + fileSys = FileSystem.get(conf); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + + private static void printOutput(FileSystem fileSys, ClusterStatus cluster, + HamaConfiguration conf) throws IOException { + System.out.println("Each task printed the \"Hello World\" as below:"); + for (int i = 0; i < cluster.getGroomServers(); i++) { + SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, new Path( + TMP_OUTPUT + i), conf); + LongWritable timestamp = new LongWritable(); + Text message = new Text(); + reader.next(timestamp, message); + System.out.println(new Date(timestamp.get()) + ": " + message); + reader.close(); + } + } + + private static void initTempDir(FileSystem fileSys) throws IOException { + if (fileSys.exists(new Path(TMP_OUTPUT))) { + fileSys.delete(new Path(TMP_OUTPUT), true); + } + } + + public static void main(String[] args) throws InterruptedException, + IOException, ClassNotFoundException { + // BSP job configuration + HamaConfiguration conf = new HamaConfiguration(); + + BSPJob bsp = new BSPJob(conf, SerializePrinting.class); + // Set the job name + bsp.setJobName("Serialize Printing"); + bsp.setBspClass(HelloBSP.class); + + // Set the task size as a number of GroomServer + BSPJobClient jobClient = new BSPJobClient(conf); + ClusterStatus cluster = jobClient.getClusterStatus(false); + bsp.setNumBspTask(cluster.getGroomServers()); + + FileSystem fileSys = FileSystem.get(conf); + initTempDir(fileSys); + + if (bsp.waitForCompletion(true)) { + printOutput(fileSys, cluster, conf); + } + } + +} Index: examples/src/main/java/org/apache/hama/examples/RandBench.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/RandBench.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/RandBench.java (revision 0) @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples; + +import java.io.IOException; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hama.HamaConfiguration; +import org.apache.hama.bsp.BSP; +import org.apache.hama.bsp.BSPJob; +import org.apache.hama.bsp.BSPJobClient; +import org.apache.hama.bsp.BSPMessage; +import org.apache.hama.bsp.BSPPeer; +import org.apache.hama.bsp.ByteMessage; +import org.apache.hama.bsp.ClusterStatus; +import org.apache.hama.util.Bytes; +import org.apache.zookeeper.KeeperException; + +public class RandBench { + private static final String SIZEOFMSG = "msg.size"; + private static final String N_COMMUNICATIONS = "communications.num"; + private static final String N_SUPERSTEPS = "supersteps.num"; + + public static class RandBSP extends BSP { + public static final Log LOG = LogFactory.getLog(RandBSP.class); + private Configuration conf; + private Random r = new Random(); + private int sizeOfMsg; + private int nCommunications; + private int nSupersteps; + + @Override + public void bsp(BSPPeer bspPeer) throws IOException, + KeeperException, InterruptedException { + byte[] dummyData = new byte[sizeOfMsg]; + BSPMessage msg = null; + String[] peers = bspPeer.getAllPeerNames(); + String peerName = bspPeer.getPeerName(); + + for (int i = 0; i < nSupersteps; i++) { + + for (int j = 0; j < nCommunications; j++) { + String tPeer = peers[r.nextInt(peers.length)]; + String tag = peerName + " to " + tPeer; + msg = new ByteMessage(Bytes.toBytes(tag), dummyData); + bspPeer.send(tPeer, msg); + } + + bspPeer.sync(); + + ByteMessage received; + while ((received = (ByteMessage) bspPeer.getCurrentMessage()) != null) { + LOG.info(Bytes.toString(received.getTag()) + " : " + + received.getData().length); + } + + } + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + this.sizeOfMsg = conf.getInt(SIZEOFMSG, 1); + this.nCommunications = conf.getInt(N_COMMUNICATIONS, 1); + this.nSupersteps = conf.getInt(N_SUPERSTEPS, 1); + } + + @Override + public Configuration getConf() { + return conf; + } + + } + + public static void main(String[] args) throws Exception { + if (args.length < 3) { + System.out.println("Usage: "); + System.exit(-1); + } + + // BSP job configuration + HamaConfiguration conf = new HamaConfiguration(); + + conf.setInt(SIZEOFMSG, Integer.parseInt(args[0])); + conf.setInt(N_COMMUNICATIONS, Integer.parseInt(args[1])); + conf.setInt(N_SUPERSTEPS, Integer.parseInt(args[2])); + + BSPJob bsp = new BSPJob(conf, RandBench.class); + // Set the job name + bsp.setJobName("Random Communication Benchmark"); + bsp.setBspClass(RandBSP.class); + + // Set the task size as a number of GroomServer + BSPJobClient jobClient = new BSPJobClient(conf); + ClusterStatus cluster = jobClient.getClusterStatus(false); + bsp.setNumBspTask(cluster.getMaxTasks()); + + long startTime = System.currentTimeMillis(); + bsp.waitForCompletion(true); + System.out.println("Job Finished in " + + (double) (System.currentTimeMillis() - startTime) / 1000.0 + + " seconds"); + } +} Index: examples/src/main/java/org/apache/hama/examples/ExampleDriver.java =================================================================== --- examples/src/main/java/org/apache/hama/examples/ExampleDriver.java (revision 0) +++ examples/src/main/java/org/apache/hama/examples/ExampleDriver.java (revision 0) @@ -0,0 +1,42 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.examples; + +import org.apache.hadoop.util.ProgramDriver; +import org.apache.hama.examples.graph.PageRank; +import org.apache.hama.examples.graph.ShortestPaths; + +public class ExampleDriver { + + public static void main(String[] args) { + ProgramDriver pgd = new ProgramDriver(); + try { + pgd.addClass("pi", PiEstimator.class, "Pi Estimator"); + pgd.addClass("bench", RandBench.class, "Random Communication Benchmark"); + pgd.addClass("test", SerializePrinting.class, "Serialize Printing Test"); + pgd.addClass("sssp", ShortestPaths.class, "Single Source Shortest Path"); + pgd.addClass("pagerank", PageRank.class, "PageRank"); + + pgd.driver(args); + } catch (Throwable e) { + e.printStackTrace(); + } + } +} Index: examples/pom.xml =================================================================== --- examples/pom.xml (revision 0) +++ examples/pom.xml (revision 0) @@ -0,0 +1,76 @@ + + + + + + org.apache.hama + hama-parent + 0.4.0-incubating-SNAPSHOT + + + 4.0.0 + org.apache.hama + hama-examples + Apache Hama Examples + 0.4.0-incubating-SNAPSHOT + jar + + + + org.apache.hama + hama-core + ${project.version} + + + + hama-examples-${project.version} + + + + maven-surefire-plugin + + true + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 1.4 + + + package + + shade + + + + + org.apache.hama.examples.ExampleDriver + + + + + + + + +