Index: src/examples/org/apache/hama/examples/AbstractExample.java =================================================================== --- src/examples/org/apache/hama/examples/AbstractExample.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/AbstractExample.java (작업 사본) @@ -1,52 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hama.HamaConfiguration; - -public abstract class AbstractExample { - protected static final HamaConfiguration conf = new HamaConfiguration(); - protected static List ARGS; - - public static void parseArgs(String[] args) { - List other_args = new ArrayList(); - for (int i = 0; i < args.length; ++i) { - try { - if ("-m".equals(args[i])) { - conf.setNumMapTasks(Integer.parseInt(args[++i])); - } else if ("-r".equals(args[i])) { - conf.setNumReduceTasks(Integer.parseInt(args[++i])); - } else { - other_args.add(args[i]); - } - } catch (NumberFormatException except) { - System.out.println("ERROR: Integer expected instead of " + args[i]); - } catch (ArrayIndexOutOfBoundsException except) { - System.out.println("ERROR: Required parameter missing from " - + args[i - 1]); - } - } - - ARGS = other_args; - } -} Index: src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java =================================================================== --- src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/CosineSimilarityMatrix.java (작업 사본) @@ -1,110 +0,0 @@ -package org.apache.hama.examples; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.IdentityTableReducer; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hama.Constants; -import org.apache.hama.HamaAdmin; -import org.apache.hama.HamaAdminImpl; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.DenseVector; -import org.apache.hama.util.BytesUtil; - -/** - * Cosine Similarity MapReduce - * - *

- * This is EXAMPLE code. You will need to change it to work for your context. - *

- * - *

- * ./bin/hama examples similarity INPUT_MATRIX OUTPUT_NAME
- * 
- */ -public class CosineSimilarityMatrix { - - public static class ComputeSimilarityMapper extends - TableMapper implements Configurable { - private Configuration conf = null; - private DenseMatrix matrix; - - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - DenseVector v = new DenseVector(value); - - Put put = new Put(key.get()); - for (int i = 0; i < matrix.getRows(); i++) { - double dotProduct = matrix.getRow(i).dot(v); - if (BytesUtil.getRowIndex(key.get()) == i) { - dotProduct = 0; - } - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String - .valueOf(i)), Bytes.toBytes(dotProduct)); - } - - context.write(key, put); - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - try { - matrix = new DenseMatrix(new HamaConfiguration(conf), conf - .get("input.matrix")); - } catch (IOException e) { - e.printStackTrace(); - } - - } - } - - private static Job configureJob(HamaConfiguration conf, String[] args) - throws IOException { - HamaAdmin admin = new HamaAdminImpl(conf); - - Job job = new Job(conf, "set MR job test"); - job.getConfiguration().set("input.matrix", admin.getPath(args[0])); - - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - - TableMapReduceUtil.initTableMapperJob(admin.getPath(args[0]), scan, - ComputeSimilarityMapper.class, ImmutableBytesWritable.class, Put.class, job); - TableMapReduceUtil.initTableReducerJob(args[1], IdentityTableReducer.class, - job); - job.setNumReduceTasks(0); - return job; - } - - /** - * - * - * @param args - * @throws Exception - */ - public static void main(String[] args) throws Exception { - if (args.length < 2) { - System.out.println("Usage: "); - } - - HamaConfiguration conf = new HamaConfiguration(); - Job job = configureJob(conf, args); - System.exit(job.waitForCompletion(true) ? 0 : 1); - } -} Index: src/examples/org/apache/hama/examples/ExampleDriver.java =================================================================== --- src/examples/org/apache/hama/examples/ExampleDriver.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/ExampleDriver.java (작업 사본) @@ -28,14 +28,6 @@ try { pgd.addClass("pi", PiEstimator.class, "Pi Estimator"); - /* - pgd.addClass("sprint", SerializePrinting.class, "Serialize Printing"); - pgd.addClass("rand", RandomMatrix.class, "Generate matrix with random elements."); - pgd.addClass("mult", MatrixMultiplication.class, "Mat-Mat Multiplication."); - pgd.addClass("similarity", CosineSimilarityMatrix.class, "Cosine Similarity Matrix."); - pgd.addClass("norms", MatrixNorm.class, "Matrix Norms."); - */ - pgd.driver(args); } catch (Throwable e) { e.printStackTrace(); Index: src/examples/org/apache/hama/examples/JacobiEigen.java =================================================================== --- src/examples/org/apache/hama/examples/JacobiEigen.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/JacobiEigen.java (작업 사본) @@ -1,366 +0,0 @@ -package org.apache.hama.examples; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.IdentityTableReducer; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; -import org.apache.hama.Constants; -import org.apache.hama.HamaAdmin; -import org.apache.hama.HamaAdminImpl; -import org.apache.hama.examples.mapreduce.DummyMapper; -import org.apache.hama.examples.mapreduce.JacobiInitMap; -import org.apache.hama.examples.mapreduce.Pair; -import org.apache.hama.examples.mapreduce.PivotInputFormat; -import org.apache.hama.examples.mapreduce.PivotMap; -import org.apache.hama.examples.mapreduce.RotationInputFormat; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.Matrix; -import org.apache.hama.util.BytesUtil; - -public class JacobiEigen extends AbstractExample { - - /** - * EigenValue Constants - */ - /** a matrix copy of the original copy collected in "eicol" family * */ - public static final String EICOL = "eicol"; - - /** a column family collect all values and statuses used during computation * */ - public static final String EI = "eival"; - public static final String EIVAL = "value"; - public static final String EICHANGED = "changed"; - - /** a column identify the index of the max absolute value each row * */ - public static final String EIIND = "ind"; - - /** a matrix collect all the eigen vectors * */ - public static final String EIVEC = "eivec"; - public static final String MATRIX = "hama.jacobieigenvalue.matrix"; - - /** parameters for pivot * */ - public static final String PIVOTROW = "hama.jacobi.pivot.row"; - public static final String PIVOTCOL = "hama.jacobi.pivot.col"; - public static final String PIVOTSIN = "hama.jacobi.pivot.sin"; - public static final String PIVOTCOS = "hama.jacobi.pivot.cos"; - - private static HTable table; - - public static void main(String[] args) throws IOException { - if (args.length < 2) { - System.out - .println("add [-m maps] [-r reduces] [max_iteration]"); - System.exit(-1); - } else { - parseArgs(args); - } - - String matrixA = ARGS.get(0); - - HamaAdmin admin = new HamaAdminImpl(conf); - Matrix a = admin.getMatrix(matrixA); - - if (ARGS.size() > 1) { - jacobiEigenValue((DenseMatrix) a, Integer.parseInt(ARGS.get(1))); - } else { - jacobiEigenValue((DenseMatrix) a, Integer.MAX_VALUE); - } - } - - /** - * Compute all the eigen values. Note: all the eigen values are collected in - * the "eival:value" column, and the eigen vector of a specified eigen value - * is collected in the "eivec:" column family in the same row. - * - * TODO: we may need to expose the interface to access the eigen values and - * vectors - * - * @param imax limit the loops of the computation - * @throws IOException - */ - public static void jacobiEigenValue(DenseMatrix m, int imax) - throws IOException { - table = new HTable(conf, m.getPath()); - /* - * Initialization A M/R job is used for initialization(such as, preparing a - * matrx copy of the original in "eicol:" family.) - */ - // initialization - Job job = new Job(conf, "JacobiEigen initialization MR job" + m.getPath()); - - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - - TableMapReduceUtil.initTableMapperJob(m.getPath(), scan, - JacobiInitMap.class, ImmutableBytesWritable.class, Put.class, job); - TableMapReduceUtil.initTableReducerJob(m.getPath(), - IdentityTableReducer.class, job); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - final FileSystem fs = FileSystem.get(conf); - Pair pivotPair = new Pair(); - DoubleWritable pivotWritable = new DoubleWritable(); - Put put; - - // loop - int size = m.getRows(); - int state = size; - int pivot_row, pivot_col; - double pivot; - double s, c, t, y; - - int icount = 0; - while (state != 0 && icount < imax) { - icount = icount + 1; - /* - * Find the pivot and its index(pivot_row, pivot_col) A M/R job is used to - * scan all the "eival:ind" to get the max absolute value of each row, and - * do a MAX aggregation of these max values to get the max value in the - * matrix. - */ - Path outDir = new Path(new Path(m.getType() + "_TMP_FindPivot_dir_" - + System.currentTimeMillis()), "out"); - if (fs.exists(outDir)) - fs.delete(outDir, true); - - job = new Job(conf, "Find Pivot MR job" + m.getPath()); - - scan = new Scan(); - scan.addFamily(Bytes.toBytes(EI)); - - job.setInputFormatClass(PivotInputFormat.class); - job.setMapOutputKeyClass(Pair.class); - job.setMapOutputValueClass(DoubleWritable.class); - job.setMapperClass(PivotMap.class); - job.getConfiguration().set(PivotInputFormat.INPUT_TABLE, m.getPath()); - job.getConfiguration().set(PivotInputFormat.SCAN, - PivotInputFormat.convertScanToString(scan)); - - job.setOutputKeyClass(Pair.class); - job.setOutputValueClass(DoubleWritable.class); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - SequenceFileOutputFormat.setOutputPath(job, outDir); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - // read outputs - Path inFile = new Path(outDir, "part-r-00000"); - SequenceFile.Reader reader = new SequenceFile.Reader(fs, inFile, conf); - try { - reader.next(pivotPair, pivotWritable); - pivot_row = pivotPair.getRow(); - pivot_col = pivotPair.getColumn(); - pivot = pivotWritable.get(); - } finally { - reader.close(); - } - fs.delete(outDir, true); - fs.delete(outDir.getParent(), true); - - if (pivot_row == 0 && pivot_col == 0) - break; // stop the iterations - - /* - * Calculation Compute the rotation parameters of next rotation. - */ - Get get = new Get(BytesUtil.getRowIndex(pivot_row)); - get.addFamily(Bytes.toBytes(EI)); - Result r = table.get(get); - double e1 = Bytes.toDouble(r.getValue(Bytes.toBytes(EI), Bytes - .toBytes(EIVAL))); - - get = new Get(BytesUtil.getRowIndex(pivot_col)); - get.addFamily(Bytes.toBytes(EI)); - r = table.get(get); - double e2 = Bytes.toDouble(r.getValue(Bytes.toBytes(EI), Bytes - .toBytes(EIVAL))); - - y = (e2 - e1) / 2; - t = Math.abs(y) + Math.sqrt(pivot * pivot + y * y); - s = Math.sqrt(pivot * pivot + t * t); - c = t / s; - s = pivot / s; - t = (pivot * pivot) / t; - if (y < 0) { - s = -s; - t = -t; - } - - /* - * Upate the pivot and the eigen values indexed by the pivot - */ - put = new Put(BytesUtil.getRowIndex(pivot_row)); - put.add(Bytes.toBytes(EICOL), Bytes.toBytes(String - .valueOf(pivot_col)), Bytes.toBytes(0.0)); - table.put(put); - - state = update(pivot_row, -t, state); - state = update(pivot_col, t, state); - - /* - * Rotation the matrix - */ - job = new Job(conf, "Rotation Matrix MR job" + m.getPath()); - - scan = new Scan(); - scan.addFamily(Bytes.toBytes(EI)); - - job.getConfiguration().setInt(PIVOTROW, pivot_row); - job.getConfiguration().setInt(PIVOTCOL, pivot_col); - job.getConfiguration().set(PIVOTSIN, String.valueOf(s)); - job.getConfiguration().set(PIVOTCOS, String.valueOf(c)); - - job.setInputFormatClass(RotationInputFormat.class); - job.setMapOutputKeyClass(NullWritable.class); - job.setMapOutputValueClass(NullWritable.class); - job.setMapperClass(DummyMapper.class); - job.getConfiguration().set(RotationInputFormat.INPUT_TABLE, m.getPath()); - job.getConfiguration().set(RotationInputFormat.SCAN, - PivotInputFormat.convertScanToString(scan)); - job.setOutputFormatClass(NullOutputFormat.class); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - // rotate eigenvectors - for (int i = 0; i < size; i++) { - get = new Get(BytesUtil.getRowIndex(pivot_row)); - e1 = Bytes.toDouble(table.get(get).getValue( - Bytes.toBytes(EIVEC), Bytes.toBytes(String.valueOf(i)))); - - get = new Get(BytesUtil.getRowIndex(pivot_col)); - e2 = Bytes.toDouble(table.get(get).getValue( - Bytes.toBytes(EIVEC), Bytes.toBytes(String.valueOf(i)))); - - double pivotR = (c * e1 - s * e2); - put = new Put(BytesUtil.getRowIndex(pivot_row)); - put.add(Bytes.toBytes(EIVEC), Bytes - .toBytes(String.valueOf(i)), Bytes.toBytes(pivotR)); - table.put(put); - - double eivC = (s * e1 + c * e2); - put = new Put(BytesUtil.getRowIndex(pivot_col)); - put.add(Bytes.toBytes(EIVEC), Bytes - .toBytes(String.valueOf(i)), Bytes.toBytes(eivC)); - table.put(put); - } - - // update index array - maxind(pivot_row, size); - maxind(pivot_col, size); - } - } - - static void maxind(int row, int size) throws IOException { - int m = row + 1; - Get get = null; - if (row + 2 < size) { - get = new Get(BytesUtil.getRowIndex(row)); - - double max = Bytes.toDouble(table.get(get).getValue( - Bytes.toBytes(EICOL), Bytes.toBytes(String.valueOf(m)))); - double val; - for (int i = row + 2; i < size; i++) { - get = new Get(BytesUtil.getRowIndex(row)); - val = Bytes.toDouble(table.get(get).getValue( - Bytes.toBytes(EICOL), Bytes.toBytes(String.valueOf(i)))); - if (Math.abs(val) > Math.abs(max)) { - m = i; - max = val; - } - } - } - - Put put = new Put(BytesUtil.getRowIndex(row)); - put.add(Bytes.toBytes(EI), Bytes.toBytes("ind"), Bytes - .toBytes(String.valueOf(m))); - table.put(put); - } - - static int update(int row, double value, int state) throws IOException { - Get get = new Get(BytesUtil.getRowIndex(row)); - double e = Bytes.toDouble(table.get(get).getValue( - Bytes.toBytes(EI), Bytes.toBytes(EIVAL))); - int changed = BytesUtil.bytesToInt(table.get(get).getValue( - Bytes.toBytes(EI), Bytes.toBytes("changed"))); - double y = e; - e += value; - - Put put = new Put(BytesUtil.getRowIndex(row)); - put.add(Bytes.toBytes(EI), Bytes.toBytes(EIVAL), Bytes - .toBytes(e)); - - if (changed == 1 && (Math.abs(y - e) < .0000001)) { // y == e) { - changed = 0; - put.add(Bytes.toBytes(EI), Bytes.toBytes(EICHANGED), - Bytes.toBytes(String.valueOf(changed))); - - state--; - } else if (changed == 0 && (Math.abs(y - e) > .0000001)) { - changed = 1; - put.add(Bytes.toBytes(EI), Bytes.toBytes(EICHANGED), - Bytes.toBytes(String.valueOf(changed))); - - state++; - } - table.put(put); - return state; - } - - // for test - static boolean verifyEigenValue(double[] e, double[][] E) throws IOException { - boolean success = true; - double e1, ev; - Get get = null; - for (int i = 0; i < e.length; i++) { - get = new Get(BytesUtil.getRowIndex(i)); - e1 = Bytes.toDouble(table.get(get).getValue(Bytes.toBytes(EI), - Bytes.toBytes(EIVAL))); - success &= ((Math.abs(e1 - e[i]) < .0000001)); - if (!success) - return success; - - for (int j = 0; j < E[i].length; j++) { - get = new Get(BytesUtil.getRowIndex(i)); - ev = Bytes.toDouble(table.get(get).getValue( - Bytes.toBytes(EIVEC), Bytes.toBytes(String.valueOf(j)))); - success &= ((Math.abs(ev - E[i][j]) < .0000001)); - if (!success) - return success; - } - } - return success; - } -} Index: src/examples/org/apache/hama/examples/MatrixMultiplication.java =================================================================== --- src/examples/org/apache/hama/examples/MatrixMultiplication.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/MatrixMultiplication.java (작업 사본) @@ -1,252 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hama.Constants; -import org.apache.hama.HamaAdmin; -import org.apache.hama.HamaAdminImpl; -import org.apache.hama.examples.mapreduce.BlockID; -import org.apache.hama.examples.mapreduce.BlockMultMap; -import org.apache.hama.examples.mapreduce.BlockMultReduce; -import org.apache.hama.examples.mapreduce.CollectBlocksMapper; -import org.apache.hama.examples.mapreduce.DenseMatrixVectorMultMap; -import org.apache.hama.examples.mapreduce.DenseMatrixVectorMultReduce; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.Matrix; -import org.apache.hama.util.RandomVariable; - -public class MatrixMultiplication extends AbstractExample { - public static void main(String[] args) throws IOException { - if (args.length < 2) { - System.out - .println("mult [-m maps] [-r reduces] [blocks]"); - System.exit(-1); - } else { - parseArgs(args); - } - - String matrixA = ARGS.get(0); - String matrixB = ARGS.get(1); - - HamaAdmin admin = new HamaAdminImpl(conf); - Matrix a = admin.getMatrix(matrixA); - Matrix b = admin.getMatrix(matrixB); - - if (!a.getType().equals(b.getType())) { - System.out.println(a.getType() + " != " + b.getType()); - System.exit(-1); - } - - Matrix c = null; - if (a.getType().equals("SparseMatrix")) { - System.out - .println("NOTE: Not implemented M/R based sparse matrix multiplication."); - System.exit(-1); - } else { - if (ARGS.size() > 2) { - c = mult(a, b, Integer.parseInt(ARGS.get(2))); - } else { - c = mult(a, b); - } - } - - for (int i = 0; i < 2; i++) { - System.out.println("c(" + 0 + ", " + i + ") : " + c.get(0, i)); - } - System.out.println("..."); - } - - /** - * C = A*B using iterative method - * - * @param B - * @return C - * @throws IOException - */ - public static DenseMatrix mult(Matrix A, Matrix B) throws IOException { - ensureForMultiplication(A, B); - int columns = 0; - if (B.getColumns() == 1 || A.getColumns() == 1) - columns = 1; - else - columns = A.getColumns(); - - DenseMatrix result = new DenseMatrix(conf, A.getRows(), columns); - List jobId = new ArrayList(); - - for (int i = 0; i < A.getRows(); i++) { - Job job = new Job(conf, "multiplication MR job : " + result.getPath() - + " " + i); - - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - job.getConfiguration() - .set(DenseMatrixVectorMultMap.MATRIX_A, A.getPath()); - job.getConfiguration().setInt(DenseMatrixVectorMultMap.ITH_ROW, i); - - TableMapReduceUtil.initTableMapperJob(B.getPath(), scan, - DenseMatrixVectorMultMap.class, IntWritable.class, MapWritable.class, - job); - TableMapReduceUtil.initTableReducerJob(result.getPath(), - DenseMatrixVectorMultReduce.class, job); - try { - job.waitForCompletion(false); - jobId.add(job); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } - - while (checkAllJobs(jobId) == false) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - return result; - } - - /** - * C = A * B using Blocking algorithm - * - * @param B - * @param blocks the number of blocks - * @return C - * @throws IOException - */ - public static DenseMatrix mult(Matrix A, Matrix B, int blocks) - throws IOException { - ensureForMultiplication(A, B); - - String collectionTable = "collect_" + RandomVariable.randMatrixPath(); - HTableDescriptor desc = new HTableDescriptor(collectionTable); - desc.addFamily(new HColumnDescriptor(Bytes.toBytes(Constants.BLOCK))); - new HBaseAdmin(conf).createTable(desc); - - collectBlocksMapRed(A, collectionTable, blocks, true); - collectBlocksMapRed(B, collectionTable, blocks, false); - - DenseMatrix result = new DenseMatrix(conf, A.getRows(), A.getColumns()); - - Job job = new Job(conf, "multiplication MR job : " + result.getPath()); - - Scan scan = new Scan(); - scan.addFamily(Bytes.toBytes(Constants.BLOCK)); - - TableMapReduceUtil.initTableMapperJob(collectionTable, scan, - BlockMultMap.class, BlockID.class, BytesWritable.class, job); - TableMapReduceUtil.initTableReducerJob(result.getPath(), - BlockMultReduce.class, job); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - new HamaAdminImpl(conf, new HBaseAdmin(conf)).delete(collectionTable); - return result; - } - - /** - * Collect Blocks - * - * @param path a input path - * @param collectionTable the collection table - * @param blockNum the number of blocks - * @param bool - * @throws IOException - */ - public static void collectBlocksMapRed(Matrix m, String collectionTable, - int blockNum, boolean bool) throws IOException { - double blocks = Math.pow(blockNum, 0.5); - if (!String.valueOf(blocks).endsWith(".0")) - throw new IOException("can't divide."); - - int block_size = (int) blocks; - Job job = new Job(conf, "Blocking MR job" + m.getPath()); - - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - - job.getConfiguration().set(CollectBlocksMapper.BLOCK_SIZE, - String.valueOf(block_size)); - job.getConfiguration().set(CollectBlocksMapper.ROWS, - String.valueOf(m.getRows())); - job.getConfiguration().set(CollectBlocksMapper.COLUMNS, - String.valueOf(m.getColumns())); - job.getConfiguration().setBoolean(CollectBlocksMapper.MATRIX_POS, bool); - - TableMapReduceUtil.initTableMapperJob(m.getPath(), scan, - org.apache.hama.examples.mapreduce.CollectBlocksMapper.class, BlockID.class, - MapWritable.class, job); - TableMapReduceUtil.initTableReducerJob(collectionTable, - org.apache.hama.examples.mapreduce.CollectBlocksReducer.class, job); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } - - public static void ensureForMultiplication(Matrix A, Matrix m) - throws IOException { - if (A.getColumns() != m.getRows()) { - throw new IOException("A's columns should equal with B's rows while A*B."); - } - } - - public static boolean checkAllJobs(List jobId) throws IOException { - Iterator it = jobId.iterator(); - boolean allTrue = true; - while (it.hasNext()) { - if (!it.next().isComplete()) { - allTrue = false; - } - } - - return allTrue; - } -} Index: src/examples/org/apache/hama/examples/MatrixNorm.java =================================================================== --- src/examples/org/apache/hama/examples/MatrixNorm.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/MatrixNorm.java (작업 사본) @@ -1,249 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; -import org.apache.hama.Constants; -import org.apache.hama.HamaAdmin; -import org.apache.hama.HamaAdminImpl; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.examples.mapreduce.MatrixNormMapReduce; -import org.apache.hama.matrix.Matrix; -import org.apache.hama.matrix.Matrix.Norm; - -public class MatrixNorm extends AbstractExample { - public static void main(String[] args) throws IOException { - if (args.length < 1) { - System.out.println("norm [-m maps] [-r reduces] "); - System.out.println("arguments: type - one | infinity | frobenius | maxvalue"); - System.exit(-1); - } else { - parseArgs(args); - } - - HamaAdmin admin = new HamaAdminImpl(conf); - Matrix a = admin.getMatrix(ARGS.get(0)); - if(ARGS.get(1).equalsIgnoreCase("one")) { - System.out.println("The maximum absolute column sum of matrix '" + ARGS.get(0) - + "' is " + norm(a, Norm.One)); - } else if(ARGS.get(1).equalsIgnoreCase("infinity")) { - System.out.println("The maximum absolute row sum of matrix '" + ARGS.get(0) - + "' is " + norm(a, Norm.Infinity)); - } else if(ARGS.get(1).equalsIgnoreCase("frobenius")) { - System.out.println("The root of sum of squares of matrix '" + ARGS.get(0) - + "' is " + norm(a, Norm.Frobenius)); - } else { - System.out.println("The max absolute cell value of matrix '" + ARGS.get(0) - + "' is " + norm(a, Norm.Maxvalue)); - } - } - - /** - * Computes the given norm of the matrix - * - * @param type - * @return norm of the matrix - * @throws IOException - */ - public static double norm(Matrix a, Norm type) throws IOException { - if (type == Norm.One) - return getNorm1(a); - else if (type == Norm.Frobenius) - return getFrobenius(a); - else if (type == Norm.Infinity) - return getInfinity(a); - else - return getMaxvalue(a); - } - - - public static double getNorm1(Matrix a) throws IOException { - final FileSystem fs = FileSystem.get(conf); - Path outDir = new Path(new Path(a.getType() + "_TMP_norm1_dir_" - + System.currentTimeMillis()), "out"); - if (fs.exists(outDir)) - fs.delete(outDir, true); - - Job job = new Job(conf, "norm1 MR job : " + a.getPath()); - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - - TableMapReduceUtil.initTableMapperJob(a.getPath(), scan, - MatrixNormMapReduce.MatrixOneNormMapper.class, IntWritable.class, - DoubleWritable.class, job); - - job.setCombinerClass(MatrixNormMapReduce.MatrixOneNormCombiner.class); - job.setReducerClass(MatrixNormMapReduce.MatrixOneNormReducer.class); - job.setNumReduceTasks(1); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(IntWritable.class); - job.setOutputValueClass(DoubleWritable.class); - SequenceFileOutputFormat.setOutputPath(job, outDir); - - try { - job.waitForCompletion(true); - System.out.println(job.reduceProgress()); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - // read outputs - double result = readOutput(conf, fs, outDir); - fs.delete(outDir.getParent(), true); - return result; - } - - protected static double getMaxvalue(Matrix a) throws IOException { - final FileSystem fs = FileSystem.get(conf); - Path outDir = new Path(new Path(a.getType() + "_TMP_normMaxValue_dir_" - + System.currentTimeMillis()), "out"); - if (fs.exists(outDir)) - fs.delete(outDir, true); - - Job job = new Job(conf, "MaxValue Norm MR job : " + a.getPath()); - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - - TableMapReduceUtil.initTableMapperJob(a.getPath(), scan, - MatrixNormMapReduce.MatrixMaxValueNormMapper.class, IntWritable.class, - DoubleWritable.class, job); - - job.setCombinerClass(MatrixNormMapReduce.MatrixMaxValueNormReducer.class); - job.setReducerClass(MatrixNormMapReduce.MatrixMaxValueNormReducer.class); - job.setNumReduceTasks(1); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(IntWritable.class); - job.setOutputValueClass(DoubleWritable.class); - SequenceFileOutputFormat.setOutputPath(job, outDir); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - // read outputs - double result = readOutput(conf, fs, outDir); - fs.delete(outDir.getParent(), true); - return result; - } - - protected static double getInfinity(Matrix a) throws IOException { - final FileSystem fs = FileSystem.get(conf); - Path outDir = new Path(new Path(a.getType() + "_TMP_normInifity_dir_" - + System.currentTimeMillis()), "out"); - if (fs.exists(outDir)) - fs.delete(outDir, true); - - Job job = new Job(conf, "Infinity Norm MR job : " + a.getPath()); - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - - TableMapReduceUtil.initTableMapperJob(a.getPath(), scan, - MatrixNormMapReduce.MatrixInfinityNormMapper.class, IntWritable.class, - DoubleWritable.class, job); - - job.setCombinerClass(MatrixNormMapReduce.MatrixInfinityNormReduce.class); - job.setReducerClass(MatrixNormMapReduce.MatrixInfinityNormReduce.class); - job.setNumReduceTasks(1); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(IntWritable.class); - job.setOutputValueClass(DoubleWritable.class); - SequenceFileOutputFormat.setOutputPath(job, outDir); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - // read outputs - double result = readOutput(conf, fs, outDir); - fs.delete(outDir.getParent(), true); - return result; - } - - protected static double getFrobenius(Matrix a) throws IOException { - final FileSystem fs = FileSystem.get(conf); - Path outDir = new Path(new Path(a.getType() + "_TMP_normFrobenius_dir_" - + System.currentTimeMillis()), "out"); - if (fs.exists(outDir)) - fs.delete(outDir, true); - - Job job = new Job(conf, "Frobenius Norm MR job : " + a.getPath()); - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - - TableMapReduceUtil.initTableMapperJob(a.getPath(), scan, - MatrixNormMapReduce.MatrixFrobeniusNormMapper.class, IntWritable.class, - DoubleWritable.class, job); - - job.setCombinerClass(MatrixNormMapReduce.MatrixFrobeniusNormCombiner.class); - job.setReducerClass(MatrixNormMapReduce.MatrixFrobeniusNormReducer.class); - job.setNumReduceTasks(1); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(IntWritable.class); - job.setOutputValueClass(DoubleWritable.class); - SequenceFileOutputFormat.setOutputPath(job, outDir); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - // read outputs - double result = readOutput(conf, fs, outDir); - fs.delete(outDir.getParent(), true); - return result; - } - - public static double readOutput(HamaConfiguration config, FileSystem fs, Path outDir) - throws IOException { - Path inFile = new Path(outDir, "part-r-00000"); - IntWritable numInside = new IntWritable(); - DoubleWritable result = new DoubleWritable(); - SequenceFile.Reader reader = new SequenceFile.Reader(fs, inFile, config); - try { - reader.next(numInside, result); - } finally { - reader.close(); - } - return result.get(); - } -} Index: src/examples/org/apache/hama/examples/PiEstimator.java =================================================================== --- src/examples/org/apache/hama/examples/PiEstimator.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/PiEstimator.java (작업 사본) @@ -55,7 +55,7 @@ byte[] myData = Bytes.toBytes(4.0 * (double) in / (double) iterations); BSPMessage estimate = new BSPMessage(tagName, myData); - bspPeer.send(new InetSocketAddress("slave.udanax.org", 61000), estimate); + bspPeer.send(new InetSocketAddress("localhost", 61000), estimate); bspPeer.sync(); double pi = 0.0; Index: src/examples/org/apache/hama/examples/RandomMatrix.java =================================================================== --- src/examples/org/apache/hama/examples/RandomMatrix.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/RandomMatrix.java (작업 사본) @@ -1,205 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.examples.mapreduce.RandomMatrixMapper; -import org.apache.hama.examples.mapreduce.RandomMatrixReducer; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.Matrix; -import org.apache.hama.matrix.SparseMatrix; - -public class RandomMatrix extends AbstractExample { - static private String TABLE_PREFIX; - static private Path TMP_DIR; - - public static void main(String[] args) throws IOException { - if (args.length < 3) { - System.out - .println("rand [-m maps] [-r reduces] "); - System.out - .println("ex) rand -m 10 -r 10 2000 2000 30.5% matrixA"); - System.exit(-1); - } else { - parseArgs(args); - } - - int row = Integer.parseInt(ARGS.get(0)); - int column = Integer.parseInt(ARGS.get(1)); - double percent = Double.parseDouble(ARGS.get(2).substring(0, ARGS.get(2).length()-1)); - - Matrix a; - if(percent == 100) - a = random_mapred(conf, row, column); - else - a = random_mapred(conf, row, column, percent); - - a.save(ARGS.get(3)); - } - - - /** - * Generate matrix with random elements using Map/Reduce - * - * @param conf configuration object - * @param m the number of rows. - * @param n the number of columns. - * @return an m-by-n matrix with uniformly distributed random elements. - * @throws IOException - */ - public static DenseMatrix random_mapred(HamaConfiguration conf, int m, int n) - throws IOException { - TABLE_PREFIX = "DenseMatrix"; - TMP_DIR = new Path(TABLE_PREFIX + "_TMP_dir"); - DenseMatrix rand = new DenseMatrix(conf, m, n); - - Job job = new Job(conf, "random matrix MR job : " + rand.getPath()); - final Path inDir = new Path(TMP_DIR, "in"); - FileInputFormat.setInputPaths(job, inDir); - job.setMapperClass(RandomMatrixMapper.class); - - job.setMapOutputKeyClass(IntWritable.class); - job.setMapOutputValueClass(MapWritable.class); - - job.getConfiguration().setInt("matrix.column", n); - job.getConfiguration().set("matrix.type", TABLE_PREFIX); - job.getConfiguration().set("matrix.density", "100"); - - job.setInputFormatClass(SequenceFileInputFormat.class); - final FileSystem fs = FileSystem.get(job.getConfiguration()); - int interval = m / conf.getNumMapTasks(); - - // generate an input file for each map task - for (int i = 0; i < conf.getNumMapTasks(); ++i) { - final Path file = new Path(inDir, "part" + i); - final IntWritable start = new IntWritable(i * interval); - IntWritable end = null; - if ((i + 1) != conf.getNumMapTasks()) { - end = new IntWritable(((i * interval) + interval) - 1); - } else { - end = new IntWritable(m - 1); - } - final SequenceFile.Writer writer = SequenceFile.createWriter(fs, job - .getConfiguration(), file, IntWritable.class, IntWritable.class, - CompressionType.NONE); - try { - writer.append(start, end); - } finally { - writer.close(); - } - System.out.println("Wrote input for Map #" + i); - } - - job.setOutputFormatClass(TableOutputFormat.class); - job.setReducerClass(RandomMatrixReducer.class); - job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, rand.getPath()); - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(Writable.class); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ClassNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - fs.delete(TMP_DIR, true); - return rand; - } - - public static SparseMatrix random_mapred(HamaConfiguration conf, int m, - int n, double percent) throws IOException { - TABLE_PREFIX = "SparseMatrix"; - TMP_DIR = new Path(TABLE_PREFIX + "_TMP_dir"); - SparseMatrix rand = new SparseMatrix(conf, m, n); - - Job job = new Job(conf, "random matrix MR job : " + rand.getPath()); - final Path inDir = new Path(TMP_DIR, "in"); - FileInputFormat.setInputPaths(job, inDir); - job.setMapperClass(RandomMatrixMapper.class); - - job.setMapOutputKeyClass(IntWritable.class); - job.setMapOutputValueClass(MapWritable.class); - - job.getConfiguration().setInt("matrix.column", n); - job.getConfiguration().set("matrix.type", TABLE_PREFIX); - job.getConfiguration().set("matrix.density", String.valueOf(percent)); - - job.setInputFormatClass(SequenceFileInputFormat.class); - final FileSystem fs = FileSystem.get(job.getConfiguration()); - int interval = m / conf.getNumMapTasks(); - - // generate an input file for each map task - for (int i = 0; i < conf.getNumMapTasks(); ++i) { - final Path file = new Path(inDir, "part" + i); - final IntWritable start = new IntWritable(i * interval); - IntWritable end = null; - if ((i + 1) != conf.getNumMapTasks()) { - end = new IntWritable(((i * interval) + interval) - 1); - } else { - end = new IntWritable(m - 1); - } - final SequenceFile.Writer writer = SequenceFile.createWriter(fs, job - .getConfiguration(), file, IntWritable.class, IntWritable.class, - CompressionType.NONE); - try { - writer.append(start, end); - } finally { - writer.close(); - } - System.out.println("Wrote input for Map #" + i); - } - - job.setOutputFormatClass(TableOutputFormat.class); - job.setReducerClass(RandomMatrixReducer.class); - job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, rand.getPath()); - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(Writable.class); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ClassNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - fs.delete(TMP_DIR, true); - return rand; - } - -} Index: src/examples/org/apache/hama/examples/mapreduce/BlockID.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/BlockID.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/BlockID.java (작업 사본) @@ -1,150 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.mapreduce; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.WritableComparable; -import org.apache.log4j.Logger; - -/** A WritableComparable for BlockIDs. */ -@SuppressWarnings("unchecked") -public class BlockID implements WritableComparable { - static final Logger LOG = Logger.getLogger(BlockID.class); - public static final int PAD_SIZE = 15; - private int row; - private int column; - private int seq = -1; - - public BlockID() { - } - - public BlockID(int row, int column) { - set(row, column); - } - - public BlockID(byte[] bytes) throws IOException { - String rKey = Bytes.toString(bytes); - String keys[] = null; - if (rKey.substring(0, 8).equals("00000000")) { - int i = 8; - while (rKey.charAt(i) == '0') { - i++; - } - keys = rKey.substring(i, rKey.length()).split("[,]"); - } else { - int i = 0; - while (rKey.charAt(i) == '0') { - i++; - } - keys = rKey.substring(i, rKey.length()).split("[,]"); - } - - try { - this.row = Integer.parseInt(keys[1]); - String[] columns = keys[2].split("[-]"); - this.column = Integer.parseInt(columns[0]); - } catch (ArrayIndexOutOfBoundsException e) { - throw new ArrayIndexOutOfBoundsException(rKey + "\n" + e); - } - } - - public BlockID(int row, int column, int seq) { - set(row, column); - this.seq = seq; - } - - public void set(int row, int column) { - this.row = row; - this.column = column; - } - - public int getRow() { - return row; - } - - public int getColumn() { - return column; - } - - public void readFields(DataInput in) throws IOException { - BlockID value = new BlockID(Bytes.readByteArray(in)); - this.row = value.getRow(); - this.column = value.getColumn(); - } - - public void write(DataOutput out) throws IOException { - Bytes.writeByteArray(out, Bytes.toBytes(this.toString())); - } - - /** - * Make BlockID's string representation be same format. - */ - public String toString() { - int zeros = PAD_SIZE - String.valueOf(row).length() - - String.valueOf(column).length(); - StringBuffer buf = new StringBuffer(); - for (int i = 0; i < zeros; ++i) { - buf.append("0"); - } - - if(seq > -1) { - return buf.toString() + "," + row + "," + column + "-" + seq; - } else { - return buf.toString() + "," + row + "," + column; - } - } - - @Override - public int hashCode() { - // simply use a prime number - // may be need a more balance hash function - return row * 37 + column; - } - - public int compareTo(Object o) { - int thisRow = this.row; - int thatRow = ((BlockID) o).row; - int thisColumn = this.column; - int thatColumn = ((BlockID) o).column; - - if (thisRow != thatRow) { - return (thisRow < thatRow ? -1 : 1); - } else { - return (thisColumn < thatColumn ? -1 : (thisColumn == thatColumn ? 0 : 1)); - } - } - - @Override - public boolean equals(Object o) { - if (o == null) - return false; - if (!(o instanceof BlockID)) - return false; - return compareTo(o) == 0; - } - - public byte[] getBytes() { - return Bytes.toBytes(this.toString()); - } -} Index: src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java (작업 사본) @@ -1,24 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hama.Constants; -import org.apache.hama.matrix.SubMatrix; - -public class BlockMultMap extends TableMapper { - private byte[] COLUMN = Bytes.toBytes(Constants.BLOCK); - - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - SubMatrix a = new SubMatrix(value.getValue(COLUMN, Bytes.toBytes("a"))); - SubMatrix b = new SubMatrix(value.getValue(COLUMN, Bytes.toBytes("b"))); - - SubMatrix c = a.mult(b); - context.write(new BlockID(key.get()), new BytesWritable(c.getBytes())); - } -} Index: src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java (작업 사본) @@ -1,45 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableReducer; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.Constants; -import org.apache.hama.matrix.SubMatrix; -import org.apache.hama.util.BytesUtil; - -public class BlockMultReduce extends - TableReducer { - - @Override - public void reduce(BlockID key, Iterable values, - Context context) throws IOException, InterruptedException { - SubMatrix s = null; - for (BytesWritable value : values) { - SubMatrix b = new SubMatrix(value.getBytes()); - if (s == null) { - s = b; - } else { - s = s.add(b); - } - } - - int startRow = key.getRow() * s.getRows(); - int startColumn = key.getColumn() * s.getColumns(); - - for (int i = 0; i < s.getRows(); i++) { - Put put = new Put(BytesUtil.getRowIndex(i + startRow)); - for (int j = 0; j < s.getColumns(); j++) { - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(j + startColumn)), - Bytes.toBytes(s.get(i, j))); - } - - context.write(new ImmutableBytesWritable(BytesUtil.getRowIndex(key - .getRow())), put); - } - } -} Index: src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java (작업 사본) @@ -1,62 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.io.MapWritable; -import org.apache.hama.matrix.DenseVector; -import org.apache.hama.util.BytesUtil; - -public class CollectBlocksMapper extends TableMapper - implements Configurable { - private Configuration conf = null; - /** Parameter of the path of the matrix to be blocked * */ - public static final String BLOCK_SIZE = "hama.blocking.size"; - public static final String ROWS = "hama.blocking.rows"; - public static final String COLUMNS = "hama.blocking.columns"; - public static final String MATRIX_POS = "a.or.b"; - - private int mBlockNum; - private int mBlockRowSize; - private int mBlockColSize; - private int mRows; - private int mColumns; - - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - int startColumn, endColumn, blkRow = BytesUtil.getRowIndex(key.get()) - / mBlockRowSize, i = 0; - DenseVector dv = new DenseVector(BytesUtil.getRowIndex(key.get()), value); - - do { - startColumn = i * mBlockColSize; - endColumn = startColumn + mBlockColSize - 1; - if (endColumn >= mColumns) // the last sub vector - endColumn = mColumns - 1; - context.write(new BlockID(blkRow, i), dv.subVector(startColumn, endColumn).getEntries()); - - i++; - } while (endColumn < (mColumns - 1)); - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - - mBlockNum = Integer.parseInt(conf.get(BLOCK_SIZE, "")); - mRows = Integer.parseInt(conf.get(ROWS, "")); - mColumns = Integer.parseInt(conf.get(COLUMNS, "")); - - mBlockRowSize = mRows / mBlockNum; - mBlockColSize = mColumns / mBlockNum; - } -} Index: src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java (작업 사본) @@ -1,106 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableReducer; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.Constants; -import org.apache.hama.matrix.DenseVector; -import org.apache.hama.matrix.SubMatrix; - -public class CollectBlocksReducer extends - TableReducer implements Configurable { - private Configuration conf = null; - private int mBlockNum; - private int mBlockRowSize; - private int mBlockColSize; - private int mRows; - private int mColumns; - private boolean matrixPos; - - public void reduce(BlockID key, Iterable values, - Context context) throws IOException, InterruptedException { - // the block's base offset in the original matrix - int colBase = key.getColumn() * mBlockColSize; - int rowBase = key.getRow() * mBlockRowSize; - - // the block's size : rows & columns - int smRows = mBlockRowSize; - if ((rowBase + mBlockRowSize - 1) >= mRows) - smRows = mRows - rowBase; - int smCols = mBlockColSize; - if ((colBase + mBlockColSize - 1) >= mColumns) - smCols = mColumns - colBase; - - // construct the matrix - SubMatrix subMatrix = new SubMatrix(smRows, smCols); - // i, j is the current offset in the sub-matrix - int i = 0, j = 0; - for (MapWritable value : values) { - DenseVector vw = new DenseVector(value); - // check the size is suitable - if (vw.size() != smCols) - throw new IOException("Block Column Size dismatched."); - i = vw.getRow() - rowBase; - - if (i >= smRows || i < 0) - throw new IOException("Block Row Size dismatched."); - - // put the subVector to the subMatrix - for (j = 0; j < smCols; j++) { - subMatrix.set(i, j, vw.get(colBase + j)); - } - } - //BlockWritable outValue = new BlockWritable(subMatrix); - - // It'll used for only matrix multiplication. - if (matrixPos) { - for (int x = 0; x < mBlockNum; x++) { - int r = (key.getRow() * mBlockNum) * mBlockNum; - int seq = (x * mBlockNum) + key.getColumn() + r; - BlockID bkID = new BlockID(key.getRow(), x, seq); - Put put = new Put(bkID.getBytes()); - put.add(Bytes.toBytes(Constants.BLOCK), - Bytes.toBytes("a"), - subMatrix.getBytes()); - context.write(new ImmutableBytesWritable(bkID.getBytes()), put); - } - } else { - for (int x = 0; x < mBlockNum; x++) { - int seq = (x * mBlockNum * mBlockNum) + (key.getColumn() * mBlockNum) - + key.getRow(); - BlockID bkID = new BlockID(x, key.getColumn(), seq); - Put put = new Put(bkID.getBytes()); - put.add(Bytes.toBytes(Constants.BLOCK), - Bytes.toBytes("b"), - subMatrix.getBytes()); - context.write(new ImmutableBytesWritable(bkID.getBytes()), put); - } - } - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - - mBlockNum = Integer.parseInt(conf.get(CollectBlocksMapper.BLOCK_SIZE, "")); - mRows = Integer.parseInt(conf.get(CollectBlocksMapper.ROWS, "")); - mColumns = Integer.parseInt(conf.get(CollectBlocksMapper.COLUMNS, "")); - - mBlockRowSize = mRows / mBlockNum; - mBlockColSize = mColumns / mBlockNum; - - matrixPos = conf.getBoolean(CollectBlocksMapper.MATRIX_POS, true); - } -} Index: src/examples/org/apache/hama/examples/mapreduce/DenseMatrixVectorMultMap.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/DenseMatrixVectorMultMap.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/DenseMatrixVectorMultMap.java (작업 사본) @@ -1,73 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.DenseVector; -import org.apache.hama.util.BytesUtil; - -public class DenseMatrixVectorMultMap extends - TableMapper implements Configurable { - private Configuration conf = null; - protected DenseVector currVector; - public static final String ITH_ROW = "ith.row"; - public static final String MATRIX_A = "hama.multiplication.matrix.a"; - private IntWritable nKey = new IntWritable(); - - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - double ithjth = currVector.get(BytesUtil.getRowIndex(key.get())); - if (ithjth != 0) { - DenseVector scaled = new DenseVector(value).scale(ithjth); - context.write(nKey, scaled.getEntries()); - } - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - DenseMatrix matrix_a; - try { - matrix_a = new DenseMatrix(new HamaConfiguration(conf), conf.get(MATRIX_A, - "")); - int ithRow = conf.getInt(ITH_ROW, 0); - nKey.set(ithRow); - currVector = matrix_a.getRow(ithRow); - } catch (IOException e) { - e.printStackTrace(); - } - } - -} Index: src/examples/org/apache/hama/examples/mapreduce/DenseMatrixVectorMultReduce.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/DenseMatrixVectorMultReduce.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/DenseMatrixVectorMultReduce.java (작업 사본) @@ -1,66 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; -import java.util.Map; - -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableReducer; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.Constants; -import org.apache.hama.matrix.DenseVector; -import org.apache.hama.util.BytesUtil; - -public class DenseMatrixVectorMultReduce extends - TableReducer { - - @Override - public void reduce(IntWritable key, Iterable values, - Context context) throws IOException, InterruptedException { - DenseVector sum = new DenseVector(); - - for (MapWritable value : values) { - DenseVector nVector = new DenseVector(value); - - if (sum.size() == 0) { - sum.zeroFill(nVector.size()); - sum.add(nVector); - } else { - sum.add(nVector); - } - } - - Put put = new Put(BytesUtil.getRowIndex(key.get())); - for (Map.Entry e : sum.getEntries().entrySet()) { - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String - .valueOf(((IntWritable) e.getKey()).get())), Bytes - .toBytes(((DoubleWritable) e.getValue()).get())); - } - - context.write(new ImmutableBytesWritable(BytesUtil.getRowIndex(key.get())), - put); - } -} Index: src/examples/org/apache/hama/examples/mapreduce/DummyMapper.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/DummyMapper.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/DummyMapper.java (작업 사본) @@ -1,15 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapreduce.Mapper; - -public class DummyMapper extends Mapper { - /** The dummy function. */ - public void map(K key, V val, OutputCollector output, Reporter reporter) - throws IOException { - // do nothing - } -} Index: src/examples/org/apache/hama/examples/mapreduce/JacobiInitMap.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/JacobiInitMap.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/JacobiInitMap.java (작업 사본) @@ -1,83 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; -import java.util.Map; -import java.util.NavigableMap; - -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hama.Constants; -import org.apache.hama.examples.JacobiEigen; -import org.apache.hama.util.BytesUtil; - -/** - * The matrix will be modified during computing eigen value. So a new matrix - * will be created to prevent the original matrix being modified. To reduce the - * network transfer, we copy the "column" family in the original matrix to a - * "eicol" family. All the following modification will be done over "eicol" - * family. - * - * And the output Eigen Vector Arrays "eivec", and the output eigen value array - * "eival:value", and the temp status array "eival:changed", "eival:ind" will be - * created. - * - * Also "eival:state" will record the state of the rotation state of a matrix - */ -public class JacobiInitMap extends TableMapper { - - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - int row, col; - row = BytesUtil.getRowIndex(key.get()); - Put put = new Put(BytesUtil.getRowIndex(row)); - - double val; - double maxVal = Double.MIN_VALUE; - int maxInd = row + 1; - - boolean init = true; - - NavigableMap map = value - .getFamilyMap(Constants.COLUMNFAMILY); - for (Map.Entry e : map.entrySet()) { - val = Bytes.toDouble(e.getValue()); - col = BytesUtil.bytesToInt(e.getKey()); - // copy the original matrix to "EICOL" family - put.add(Bytes.toBytes(JacobiEigen.EICOL), Bytes.toBytes(String.valueOf(col)), Bytes - .toBytes(val)); - // make the "EIVEC" a dialog matrix - put.add(Bytes.toBytes(JacobiEigen.EIVEC), Bytes.toBytes(String.valueOf(col)), Bytes - .toBytes(col == row ? new Double(1) : new Double(0))); - - if (col == row) { - put.add(Bytes.toBytes(JacobiEigen.EI), Bytes.toBytes(JacobiEigen.EIVAL), Bytes - .toBytes(val)); - } - // find the max index - if (col > row) { - if (init) { - maxInd = col; - maxVal = val; - init = false; - } else { - if (Math.abs(val) > Math.abs(maxVal)) { - maxVal = val; - maxInd = col; - } - } - } - } - - // index array - put.add(Bytes.toBytes(JacobiEigen.EI), Bytes.toBytes(JacobiEigen.EIIND), Bytes - .toBytes(String.valueOf(maxInd))); - // Changed Array set to be true during initialization - put.add(Bytes.toBytes(JacobiEigen.EI), Bytes.toBytes(JacobiEigen.EICHANGED), Bytes - .toBytes(String.valueOf(1))); - - context.write(key, put); - } -} Index: src/examples/org/apache/hama/examples/mapreduce/MatrixNormMapReduce.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/MatrixNormMapReduce.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/MatrixNormMapReduce.java (작업 사본) @@ -1,221 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; -import java.util.Map; -import java.util.NavigableMap; - -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.mapreduce.Reducer; -import org.apache.hama.Constants; -import org.apache.hama.util.BytesUtil; -import org.apache.log4j.Logger; - -/** A Catalog class collect all the mr classes to compute the matrix's norm */ -public class MatrixNormMapReduce { - public final static IntWritable nKey = new IntWritable(-1); - - /** Infinity Norm */ - public static class MatrixInfinityNormMapper extends - TableMapper { - private DoubleWritable nValue = new DoubleWritable(); - - @Override - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - - double rowSum = 0; - NavigableMap v = value - .getFamilyMap(Constants.COLUMNFAMILY); - for (Map.Entry e : v.entrySet()) { - rowSum += Math.abs(Bytes.toDouble(e.getValue())); - } - - nValue.set(rowSum); - context.write(MatrixNormMapReduce.nKey, nValue); - } - } - - /** - * Matrix Infinity Norm Reducer - */ - public static class MatrixInfinityNormReduce extends - Reducer { - static final Logger LOG = Logger.getLogger(MatrixInfinityNormReduce.class); - private double max = 0; - private DoubleWritable nValue = new DoubleWritable(); - - public void reduce(IntWritable key, Iterable values, - Context context) throws IOException, InterruptedException { - for (DoubleWritable val : values) { - max = Math.max(val.get(), max); - } - - nValue.set(max); - context.write(MatrixNormMapReduce.nKey, nValue); - } - } - - /** One Norm Mapper */ - public static class MatrixOneNormMapper extends - TableMapper { - private IntWritable newkey = new IntWritable(); - private DoubleWritable nValue = new DoubleWritable(); - - @Override - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - - NavigableMap v = value - .getFamilyMap(Constants.COLUMNFAMILY); - for (Map.Entry e : v.entrySet()) { - newkey.set(BytesUtil.bytesToInt(e.getKey())); - nValue.set(Bytes.toDouble(e.getValue())); - context.write(newkey, nValue); - } - } - } - - /** One Norm Combiner * */ - public static class MatrixOneNormCombiner extends - Reducer { - private DoubleWritable nValue = new DoubleWritable(); - - @Override - public void reduce(IntWritable key, Iterable values, - Context context) throws IOException, InterruptedException { - - double partialColSum = 0; - for (DoubleWritable val : values) { - partialColSum += val.get(); - } - - nValue.set(partialColSum); - context.write(key, nValue); - } - } - - /** One Norm Reducer * */ - public static class MatrixOneNormReducer extends - Reducer { - private double max = 0; - - @Override - public void reduce(IntWritable key, Iterable values, - Context context) throws IOException, InterruptedException { - double colSum = 0; - - for (DoubleWritable val : values) { - colSum += val.get(); - } - - max = Math.max(Math.abs(colSum), max); - } - - public void cleanup(Context context) throws IOException, - InterruptedException { - context.write(MatrixNormMapReduce.nKey, new DoubleWritable(max)); - } - } - - /** Frobenius Norm Mapper */ - public static class MatrixFrobeniusNormMapper extends - TableMapper { - private DoubleWritable nValue = new DoubleWritable(); - - @Override - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - double rowSqrtSum = 0; - - NavigableMap v = value - .getFamilyMap(Constants.COLUMNFAMILY); - for (Map.Entry e : v.entrySet()) { - double cellValue = Bytes.toDouble(e.getValue()); - rowSqrtSum += (cellValue * cellValue); - } - - nValue.set(rowSqrtSum); - context.write(MatrixNormMapReduce.nKey, nValue); - } - } - - /** Frobenius Norm Combiner */ - public static class MatrixFrobeniusNormCombiner extends - Reducer { - private double sqrtSum = 0; - private DoubleWritable nValue = new DoubleWritable(); - - @Override - public void reduce(IntWritable key, Iterable values, - Context context) throws IOException, InterruptedException { - for (DoubleWritable val : values) { - sqrtSum += val.get(); - } - - nValue.set(sqrtSum); - context.write(MatrixNormMapReduce.nKey, nValue); - } - } - - /** Frobenius Norm Reducer */ - public static class MatrixFrobeniusNormReducer extends - Reducer { - private double sqrtSum = 0; - - @Override - public void reduce(IntWritable key, Iterable values, - Context context) throws IOException, InterruptedException { - for (DoubleWritable val : values) { - sqrtSum += val.get(); - } - - context.write(MatrixNormMapReduce.nKey, new DoubleWritable(Math - .sqrt(sqrtSum))); - } - } - - /** MaxValue Norm Mapper * */ - public static class MatrixMaxValueNormMapper extends - TableMapper { - private DoubleWritable nValue = new DoubleWritable(); - - @Override - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - double max = 0; - - NavigableMap v = value - .getFamilyMap(Constants.COLUMNFAMILY); - for (Map.Entry e : v.entrySet()) { - double cellValue = Bytes.toDouble(e.getValue()); - max = cellValue > max ? cellValue : max; - } - - nValue.set(max); - context.write(MatrixNormMapReduce.nKey, nValue); - } - } - - /** MaxValue Norm Reducer */ - public static class MatrixMaxValueNormReducer extends - Reducer { - private double max = 0; - private DoubleWritable nValue = new DoubleWritable(); - - @Override - public void reduce(IntWritable key, Iterable values, - Context context) throws IOException, InterruptedException { - for (DoubleWritable val : values) { - max = Math.max(val.get(), max); - } - - nValue.set(max); - context.write(MatrixNormMapReduce.nKey, nValue); - } - } -} Index: src/examples/org/apache/hama/examples/mapreduce/Pair.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/Pair.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/Pair.java (작업 사본) @@ -1,65 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.WritableComparable; - -/** A Pair stands for (row, column) pair **/ -public class Pair implements WritableComparable { - - int row, col; - - public Pair() {} - - public Pair(int row_, int col_) { - set(row_, col_); - } - - public int getRow() { return row; } - public int getColumn() { return col; } - - public void setRow(int row_) { row = row_; } - public void setColumn(int col_) { col = col_; } - public void set(int row_, int col_) { - row = row_; - col = col_; - } - - @Override - public void readFields(DataInput in) throws IOException { - row = in.readInt(); - col = in.readInt(); - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(row); - out.writeInt(col); - } - - @Override - public int compareTo(Pair p) { - return row == p.row ? col - p.col : row - p.row; - } - - @Override - public boolean equals(Object obj) { - Pair pair = (Pair)obj; - return compareTo(pair) == 0; - } - - @Override - public int hashCode() { - return row; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('(').append(row).append(',').append(col).append(')'); - return sb.toString(); - } - -} Index: src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java (작업 사본) @@ -1,285 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.mapreduce.TableSplit; -import org.apache.hadoop.hbase.util.Base64; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.util.StringUtils; -import org.apache.hama.examples.JacobiEigen; -import org.apache.hama.util.BytesUtil; - -public class PivotInputFormat extends InputFormat - implements Configurable { - final static Log LOG = LogFactory.getLog(PivotInputFormat.class); - - /** Job parameter that specifies the output table. */ - public static final String INPUT_TABLE = "hama.mapreduce.inputtable"; - /** Space delimited list of columns. */ - public static final String SCAN = "hama.mapreduce.scan"; - - /** The configuration. */ - private Configuration conf = null; - - /** Holds the details for the internal scanner. */ - private Scan scan = null; - /** The table to scan. */ - private HTable table = null; - /** The reader scanning the table, can be a custom one. */ - private PivotRecordReader pivotRecordReader = null; - - @Override - public List getSplits(JobContext context) throws IOException { - if (table == null) { - throw new IOException("No table was provided."); - } - byte[][] startKeys = table.getStartKeys(); - if (startKeys == null || startKeys.length == 0) { - throw new IOException("Expecting at least one region."); - } - int realNumSplits = startKeys.length; - InputSplit[] splits = new InputSplit[realNumSplits]; - int middle = startKeys.length / realNumSplits; - int startPos = 0; - for (int i = 0; i < realNumSplits; i++) { - int lastPos = startPos + middle; - lastPos = startKeys.length % realNumSplits > i ? lastPos + 1 : lastPos; - String regionLocation = table.getRegionLocation(startKeys[startPos]) - .getServerAddress().getHostname(); - splits[i] = new TableSplit(this.table.getTableName(), - startKeys[startPos], ((i + 1) < realNumSplits) ? startKeys[lastPos] - : HConstants.EMPTY_START_ROW, regionLocation); - LOG.info("split: " + i + "->" + splits[i]); - startPos = lastPos; - } - return Arrays.asList(splits); - } - - protected static class PivotRecordReader extends - RecordReader { - private int totalRows; - private int processedRows; - private int size; - boolean mocked = true; - - private ResultScanner scanner = null; - private Scan scan = null; - private HTable htable = null; - private byte[] lastRow = null; - private Pair key = null; - private DoubleWritable value = null; - - @Override - public void close() { - this.scanner.close(); - } - - public void setScan(Scan scan) { - this.scan = scan; - } - - public void setHTable(HTable htable) { - this.htable = htable; - } - - public void init() throws IOException { - restart(scan.getStartRow()); - } - - public void restart(byte[] firstRow) throws IOException { - Scan newScan = new Scan(scan); - newScan.setStartRow(firstRow); - this.scanner = this.htable.getScanner(newScan); - } - - @Override - public Pair getCurrentKey() throws IOException, InterruptedException { - return key; - } - - @Override - public DoubleWritable getCurrentValue() throws IOException, - InterruptedException { - return value; - } - - @Override - public float getProgress() throws IOException, InterruptedException { - if (totalRows <= 0) { - return 0; - } else { - return Math.min(1.0f, processedRows / (float) totalRows); - } - } - - @Override - public void initialize(InputSplit split, TaskAttemptContext context) - throws IOException, InterruptedException { - } - - @Override - public boolean nextKeyValue() throws IOException, InterruptedException { - if (key == null) - key = new Pair(); - if (value == null) - value = new DoubleWritable(); - - Result vv; - try { - vv = this.scanner.next(); - } catch (IOException e) { - LOG.debug("recovered from " + StringUtils.stringifyException(e)); - restart(lastRow); - scanner.next(); // skip presumed already mapped row - vv = scanner.next(); - } - - boolean hasMore = vv != null && vv.size() > 0; - if (hasMore) { - - byte[] row = vv.getRow(); - - int rowId = BytesUtil.getRowIndex(row); - if (rowId == size - 1) { // skip the last row - if (mocked) { - key.set(Integer.MAX_VALUE, Integer.MAX_VALUE); - mocked = false; - return true; - } else { - return false; - } - } - - byte[] col = vv.getValue(Bytes - .toBytes(JacobiEigen.EI), Bytes - .toBytes(JacobiEigen.EIIND)); - int colId = BytesUtil.bytesToInt(col); - double val = 0; - - Get get = new Get(BytesUtil.getRowIndex(rowId)); - byte[] cell = htable.get(get).getValue( - Bytes.toBytes(JacobiEigen.EICOL), - Bytes.toBytes(String.valueOf(colId))); - if (cell != null) { - val = Bytes.toDouble(cell); - } - - key.set(rowId, colId); - value.set(val); - - lastRow = row; - } else { - if (mocked) { - key.set(Integer.MAX_VALUE, Integer.MAX_VALUE); - mocked = false; - return true; - } else { - return false; - } - } - - return hasMore; - } - } - - @Override - public RecordReader createRecordReader( - InputSplit split, TaskAttemptContext context) throws IOException { - TableSplit tSplit = (TableSplit) split; - PivotRecordReader trr = this.pivotRecordReader; - // if no table record reader was provided use default - if (trr == null) { - trr = new PivotRecordReader(); - } - Scan sc = new Scan(this.scan); - sc.setStartRow(tSplit.getStartRow()); - sc.setStopRow(tSplit.getEndRow()); - trr.setScan(sc); - trr.setHTable(table); - trr.init(); - return trr; - } - - protected HTable getHTable() { - return this.table; - } - - protected void setHTable(HTable table) { - this.table = table; - } - - public Scan getScan() { - if (this.scan == null) - this.scan = new Scan(); - return scan; - } - - public void setScan(Scan scan) { - this.scan = scan; - } - - protected void setTableRecordReader(PivotRecordReader pivotRecordReader) { - this.pivotRecordReader = pivotRecordReader; - } - - @Override - public Configuration getConf() { - return conf; - } - - public void setConf(Configuration configuration) { - this.conf = configuration; - String tableName = conf.get(INPUT_TABLE); - try { - setHTable(new HTable(new HBaseConfiguration(conf), tableName)); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - } - Scan scan = null; - try { - scan = convertStringToScan(conf.get(SCAN)); - } catch (IOException e) { - LOG.error("An error occurred.", e); - } - setScan(scan); - } - - public static String convertScanToString(Scan scan) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(out); - scan.write(dos); - return Base64.encodeBytes(out.toByteArray()); - } - - public static Scan convertStringToScan(String base64) throws IOException { - ByteArrayInputStream bis = new ByteArrayInputStream(Base64.decode(base64)); - DataInputStream dis = new DataInputStream(bis); - Scan scan = new Scan(); - scan.readFields(dis); - return scan; - } - -} Index: src/examples/org/apache/hama/examples/mapreduce/PivotMap.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/PivotMap.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/PivotMap.java (작업 사본) @@ -1,28 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.mapreduce.Mapper; - -public class PivotMap extends - Mapper { - private double max = 0; - private Pair pair = new Pair(0, 0); - private Pair dummyPair = new Pair(Integer.MAX_VALUE, Integer.MAX_VALUE); - private DoubleWritable dummyVal = new DoubleWritable(0.0); - - public void map(Pair key, DoubleWritable value, Context context) - throws IOException, InterruptedException { - if (key.getRow() != Integer.MAX_VALUE) { - if (Math.abs(value.get()) > Math.abs(max)) { - pair.set(key.getRow(), key.getColumn()); - max = value.get(); - } - } else { - context.write(pair, new DoubleWritable(max)); - context.write(dummyPair, dummyVal); - } - } - -} Index: src/examples/org/apache/hama/examples/mapreduce/RandomMatrixMapper.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/RandomMatrixMapper.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/RandomMatrixMapper.java (작업 사본) @@ -1,73 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hama.matrix.DenseVector; -import org.apache.hama.matrix.SparseVector; -import org.apache.hama.matrix.Vector; -import org.apache.hama.util.RandomVariable; -import org.apache.log4j.Logger; - -public class RandomMatrixMapper extends - Mapper implements - Configurable { - private Configuration conf = null; - static final Logger LOG = Logger.getLogger(RandomMatrixMapper.class); - protected int column; - protected double density; - protected int minNums; - protected String type; - protected Vector vector = new DenseVector(); - - public void map(IntWritable key, IntWritable value, - Context context) - throws IOException, InterruptedException { - - if (type.equals("SparseMatrix")) { - for (int i = key.get(); i <= value.get(); i++) { - ((SparseVector) vector).clear(); - for (int j = 0; j < minNums; j++) { - ((SparseVector) vector).set(RandomVariable.randInt(0, column - 1), - RandomVariable.rand()); - } - context.write(new IntWritable(i), vector.getEntries()); - } - } else { - for (int i = key.get(); i <= value.get(); i++) { - ((DenseVector) vector).clear(); - for (int j = 0; j < column; j++) { - ((DenseVector) vector).set(j, RandomVariable.rand()); - } - context.write(new IntWritable(i), vector.getEntries()); - } - } - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - column = conf.getInt("matrix.column", 0); - density = Double.parseDouble(conf.get("matrix.density")); - - double vv = (column / 100.0) * density; - minNums = Math.round((float) vv); - if (minNums == 0) - minNums = 1; - - type = conf.get("matrix.type"); - if (type.equals("SparseMatrix")) - vector = new SparseVector(); - else - vector = new DenseVector(); - } -} Index: src/examples/org/apache/hama/examples/mapreduce/RandomMatrixReducer.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/RandomMatrixReducer.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/RandomMatrixReducer.java (작업 사본) @@ -1,34 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; -import java.util.Map; - -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableReducer; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.Constants; -import org.apache.hama.util.BytesUtil; -import org.apache.log4j.Logger; - -public class RandomMatrixReducer extends - TableReducer { - static final Logger LOG = Logger.getLogger(RandomMatrixReducer.class); - - public void reduce(IntWritable key, Iterable values, - Context context) throws IOException, InterruptedException { - Put put = new Put(BytesUtil.getRowIndex(key.get())); - for (Map.Entry e : values.iterator().next().entrySet()) { - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String - .valueOf(((IntWritable) e.getKey()).get())), Bytes - .toBytes(((DoubleWritable) e.getValue()).get())); - } - - context.write(new ImmutableBytesWritable(BytesUtil.getRowIndex(key.get())), - put); - } -} Index: src/examples/org/apache/hama/examples/mapreduce/RotationInputFormat.java =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/RotationInputFormat.java (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/RotationInputFormat.java (작업 사본) @@ -1,355 +0,0 @@ -package org.apache.hama.examples.mapreduce; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.mapreduce.TableSplit; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.util.StringUtils; -import org.apache.hama.Constants; -import org.apache.hama.examples.JacobiEigen; -import org.apache.hama.util.BytesUtil; - -public class RotationInputFormat extends - InputFormat implements Configurable { - final static Log LOG = LogFactory.getLog(RotationInputFormat.class); - /** Job parameter that specifies the output table. */ - public static final String INPUT_TABLE = "hama.mapreduce.inputtable"; - /** Space delimited list of columns. */ - public static final String SCAN = "hama.mapreduce.scan"; - - /** The configuration. */ - private Configuration conf = null; - - /** Holds the details for the internal scanner. */ - private Scan scan = null; - /** The table to scan. */ - private HTable table = null; - /** The reader scanning the table, can be a custom one. */ - private RotationRecordReader rotationRecordReader; - - int pivot_row, pivot_col; - double pivot_cos, pivot_sin; - - protected static class RotationRecordReader extends - RecordReader { - private ResultScanner scanner = null; - private Scan scan = null; - private HTable htable = null; - private byte[] lastRow = null; - - private int totalRows; - private int processedRows; - int startRowId, endRowId = -1; - int size; - - int pivotrow, pivotcol; - byte[] prow, pcol; - double pivotcos, pivotsin; - - public RotationRecordReader(int pr, int pc, double psin, double pcos) { - super(); - pivotrow = pr; - pivotcol = pc; - pivotsin = psin; - pivotcos = pcos; - prow = Bytes.toBytes(pivotrow); - pcol = Bytes.toBytes(pivotcol); - LOG.info(prow); - LOG.info(pcol); - } - - public void setScan(Scan scan) { - this.scan = scan; - } - - public void setHTable(HTable htable) { - this.htable = htable; - } - - public void init() throws IOException { - restart(scan.getStartRow()); - byte[] startRow = scan.getStartRow(); - byte[] endRow = scan.getStopRow(); - - Get get = new Get(Bytes.toBytes(Constants.METADATA)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = htable.get(get).getValue(Constants.ATTRIBUTE, - Bytes.toBytes("rows")); - - size = (result != null) ? Bytes.toInt(result) : 0; - - if (endRow.length == 0) { // the last split, we don't know the end row - totalRows = 0; // so we just skip it. - if (startRow.length == 0) - startRowId = 0; - else - startRowId = BytesUtil.getRowIndex(startRow); - endRowId = -1; - } else { - if (startRow.length == 0) { // the first split, start row is 0 - totalRows = BytesUtil.getRowIndex(endRow); - startRowId = 0; - endRowId = totalRows; - } else { - startRowId = BytesUtil.getRowIndex(startRow); - endRowId = BytesUtil.getRowIndex(endRow); - totalRows = startRowId - endRowId; - } - } - processedRows = 0; - LOG.info("Split (" + startRowId + ", " + endRowId + ") -> " + totalRows); - } - - public void restart(byte[] firstRow) throws IOException { - Scan newScan = new Scan(scan); - newScan.setStartRow(firstRow); - this.scanner = this.htable.getScanner(newScan); - } - - @Override - public void close() throws IOException { - this.scanner.close(); - } - - @Override - public NullWritable getCurrentKey() throws IOException, - InterruptedException { - return NullWritable.get(); - } - - @Override - public NullWritable getCurrentValue() throws IOException, - InterruptedException { - return NullWritable.get(); - } - - @Override - public float getProgress() throws IOException, InterruptedException { - if (totalRows <= 0) { - return 0; - } else { - return Math.min(1.0f, processedRows / (float) totalRows); - } - } - - @Override - public void initialize(InputSplit split, TaskAttemptContext context) - throws IOException, InterruptedException { - } - - @Override - public boolean nextKeyValue() throws IOException, InterruptedException { - Result vv; - try { - vv = this.scanner.next(); - } catch (IOException e) { - LOG.debug("recovered from " + StringUtils.stringifyException(e)); - restart(lastRow); - scanner.next(); // skip presumed already mapped row - vv = scanner.next(); - } - - double s1, s2; - Put put; - - boolean hasMore = vv != null && vv.size() > 0; - if (hasMore) { - byte[] row = vv.getRow(); - int rowId = BytesUtil.getRowIndex(row); - if (rowId < pivotrow) { - Get get = new Get(BytesUtil.getRowIndex(rowId)); - s1 = Bytes.toDouble(htable.get(get).getValue( - Bytes.toBytes(JacobiEigen.EICOL), - Bytes.toBytes(String.valueOf(pivotrow)))); - s2 = Bytes.toDouble(htable.get(get).getValue( - Bytes.toBytes(JacobiEigen.EICOL), - Bytes.toBytes(String.valueOf(pivotcol)))); - - put = new Put(BytesUtil.getRowIndex(rowId)); - put.add(Bytes.toBytes(JacobiEigen.EICOL), Bytes.toBytes(String - .valueOf(pivotrow)), Bytes.toBytes(new Double(pivotcos * s1 - - pivotsin * s2))); - put.add(Bytes.toBytes(JacobiEigen.EICOL), Bytes.toBytes(String - .valueOf(pivotcol)), Bytes.toBytes(new Double(pivotsin * s1 - + pivotcos * s2))); - - htable.put(put); - } else if (rowId == pivotrow) { - return true; - } else if (rowId < pivotcol) { - Get get = new Get(BytesUtil.getRowIndex(pivotrow)); - s1 = Bytes.toDouble(htable.get(get).getValue( - Bytes.toBytes(JacobiEigen.EICOL), - Bytes.toBytes(String.valueOf(rowId)))); - get = new Get(BytesUtil.getRowIndex(rowId)); - - s2 = Bytes.toDouble(htable.get(get).getValue( - Bytes.toBytes(JacobiEigen.EICOL), - Bytes.toBytes(String.valueOf(pivotcol)))); - - put = new Put(BytesUtil.getRowIndex(rowId)); - put.add(Bytes.toBytes(JacobiEigen.EICOL), Bytes.toBytes(String - .valueOf(pivotcol)), Bytes.toBytes(new Double(pivotsin * s1 - + pivotcos * s2))); - htable.put(put); - - put = new Put(BytesUtil.getRowIndex(pivotrow)); - put.add(Bytes.toBytes(JacobiEigen.EICOL), Bytes.toBytes(String - .valueOf(rowId)), Bytes.toBytes(new Double(pivotcos * s1 - - pivotsin * s2))); - htable.put(put); - - } else if (rowId == pivotcol) { - for (int i = pivotcol + 1; i < size; i++) { - Get get = new Get(BytesUtil.getRowIndex(pivotrow)); - - s1 = Bytes.toDouble(htable.get(get).getValue( - Bytes.toBytes(JacobiEigen.EICOL), - Bytes.toBytes(String.valueOf(i)))); - - get = new Get(BytesUtil.getRowIndex(pivotcol)); - s2 = Bytes.toDouble(htable.get(get).getValue( - Bytes.toBytes(JacobiEigen.EICOL), - Bytes.toBytes(String.valueOf(i)))); - - double pivotC = (pivotsin * s1 + pivotcos * s2); - put = new Put(BytesUtil.getRowIndex(pivotcol)); - put.add(Bytes.toBytes(JacobiEigen.EICOL), Bytes.toBytes(String - .valueOf(i)), Bytes.toBytes(pivotC)); - htable.put(put); - - double pivotV = (pivotcos * s1 - pivotsin * s2); - put = new Put(BytesUtil.getRowIndex(pivotrow)); - put.add(Bytes.toBytes(JacobiEigen.EICOL), Bytes.toBytes(String - .valueOf(i)), Bytes.toBytes(pivotV)); - htable.put(put); - - } - } else { // rowId > pivotcol - return false; - } - - lastRow = row; - } - return hasMore; - } - - } - - @Override - public RecordReader createRecordReader( - InputSplit split, TaskAttemptContext context) throws IOException, - InterruptedException { - TableSplit tSplit = (TableSplit) split; - RotationRecordReader trr = this.rotationRecordReader; - // if no table record reader was provided use default - if (trr == null) { - trr = new RotationRecordReader(pivot_row, pivot_col, pivot_sin, pivot_cos); - } - Scan sc = new Scan(this.scan); - sc.setStartRow(tSplit.getStartRow()); - sc.setStopRow(tSplit.getEndRow()); - trr.setScan(sc); - trr.setHTable(table); - trr.init(); - return trr; - } - - @Override - public List getSplits(JobContext context) throws IOException { - if (table == null) { - throw new IOException("No table was provided."); - } - byte[][] startKeys = table.getStartKeys(); - if (startKeys == null || startKeys.length == 0) { - throw new IOException("Expecting at least one region."); - } - int realNumSplits = startKeys.length; - InputSplit[] splits = new InputSplit[realNumSplits]; - int middle = startKeys.length / realNumSplits; - int startPos = 0; - for (int i = 0; i < realNumSplits; i++) { - int lastPos = startPos + middle; - lastPos = startKeys.length % realNumSplits > i ? lastPos + 1 : lastPos; - String regionLocation = table.getRegionLocation(startKeys[startPos]) - .getServerAddress().getHostname(); - splits[i] = new TableSplit(this.table.getTableName(), - startKeys[startPos], ((i + 1) < realNumSplits) ? startKeys[lastPos] - : HConstants.EMPTY_START_ROW, regionLocation); - LOG.info("split: " + i + "->" + splits[i]); - startPos = lastPos; - } - return Arrays.asList(splits); - } - - protected HTable getHTable() { - return this.table; - } - - protected void setHTable(HTable table) { - this.table = table; - } - - public Scan getScan() { - if (this.scan == null) - this.scan = new Scan(); - return scan; - } - - public void setScan(Scan scan) { - this.scan = scan; - } - - protected void setTableRecordReader(RotationRecordReader rotationRecordReader) { - this.rotationRecordReader = rotationRecordReader; - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - pivot_row = conf.getInt(JacobiEigen.PIVOTROW, -1); - pivot_col = conf.getInt(JacobiEigen.PIVOTCOL, -1); - pivot_sin = Double.parseDouble(conf.get(JacobiEigen.PIVOTSIN)); - pivot_cos = Double.parseDouble(conf.get(JacobiEigen.PIVOTCOS)); - - this.conf = conf; - String tableName = conf.get(INPUT_TABLE); - try { - setHTable(new HTable(new HBaseConfiguration(conf), tableName)); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - } - Scan scan = null; - try { - scan = PivotInputFormat.convertStringToScan(conf.get(SCAN)); - } catch (IOException e) { - LOG.error("An error occurred.", e); - } - setScan(scan); - } - -} Index: src/examples/org/apache/hama/examples/mapreduce/package.html =================================================================== --- src/examples/org/apache/hama/examples/mapreduce/package.html (리비전 1004870) +++ src/examples/org/apache/hama/examples/mapreduce/package.html (작업 사본) @@ -1,23 +0,0 @@ - - - - - -Map/Reduce Input/OutputFormats - - Index: src/java/org/apache/hama/Constants.java =================================================================== --- src/java/org/apache/hama/Constants.java (리비전 1004870) +++ src/java/org/apache/hama/Constants.java (작업 사본) @@ -70,75 +70,6 @@ static final String CLUSTER_DISTRIBUTED = "hama.cluster.distributed"; /** Cluster is fully-distributed */ static final String CLUSTER_IS_DISTRIBUTED = "true"; - /////////////////////////////////////// - // Constants for Matrix Package - /////////////////////////////////////// - /** - * An empty instance. - */ - public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - /** - * Hbase Structure for matrices - */ - - /** Meta-columnFamily to store the matrix-info */ - public final static String METADATA = "metadata"; - /** Column index & attributes */ - public final static String CINDEX = "cIndex"; - - /** The attribute column family */ - public static byte[] ATTRIBUTE = Bytes.toBytes("attribute"); - - /** The type of the matrix */ - public final static String METADATA_TYPE = "type"; - - /** The reference of the matrix */ - /** (1) when we create a Matrix object, we set up a connection to hbase table, - * the reference of the table will be incremented. - * (2) when we close a Matrix object, we disconnect the hbase table, - * the reference of the table will be decremented. - * i) if the reference of the table is not zero: - * we should not delete the table, because some other matrix object - * connect to the table. - * ii) if the reference of the table is zero: - * we need to know if the matrix table is aliased. - * 1) if the matrix table is aliased, we should not delete the table. - * 2) if the matrix table is not aliased, we need to delete the table. - */ - public final static String METADATA_REFERENCE = "reference"; - - /** The aliase names column family */ - public final static String ALIASEFAMILY = "aliase"; - - /** Default columnFamily name */ - public static byte[] COLUMNFAMILY = Bytes.toBytes("column"); - - /** Temporary random matrices name prefix */ - public final static String RANDOM = "rand"; - - /** Admin table name */ - public final static String ADMINTABLE = "hama.admin.table"; - - /** Matrix path columnFamily */ - public static final String PATHCOLUMN = "path"; - - /** Temporary Aliase name prefix in Hama Shell */ - public static final String RANDOMALIASE = "_"; - - /** default matrix's path length (tablename length) */ - public static final int DEFAULT_PATH_LENGTH = 5; - - /** default matrix's max path length (tablename length) */ - public static final int DEFAULT_MAXPATHLEN = 10000; - - /** default try times to generate a suitable tablename */ - public static final int DEFAULT_TRY_TIMES = 10000000; - - /** block data column */ - public static final String BLOCK = "block"; - - public static final Text ROWCOUNT= new Text("row"); public static final String PEER_ID = "bsp.peer.id"; } Index: src/java/org/apache/hama/HamaAdmin.java =================================================================== --- src/java/org/apache/hama/HamaAdmin.java (리비전 1004870) +++ src/java/org/apache/hama/HamaAdmin.java (작업 사본) @@ -1,70 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import java.io.IOException; - -import org.apache.hama.matrix.Matrix; - -/** - * A administration interface to manage the matrix's namespace, and table - * allocation & garbage collection. - */ -public interface HamaAdmin { - - /** - * Saves matrix as name 'AliaseName' - * - * @param matrix - * @param aliaseName - * @return true if it saved - */ - public boolean save(Matrix matrix, String aliaseName); - - /** - * @param name - * @return return a physical path of matrix - */ - public String getPath(String name); - - /** - * @param matrixName - * @return true if matrix is exist - */ - public boolean matrixExists(String matrixName); - - /** - * Deletes matrix - * - * @param matrixName - * @throws IOException - */ - public void delete(String matrixName) throws IOException; - - /** - * Load matrix - * - * @param matrixName - * @return the matrix - * @throws IOException - */ - public Matrix getMatrix(String matrixName) throws IOException; - -} Index: src/java/org/apache/hama/HamaAdminImpl.java =================================================================== --- src/java/org/apache/hama/HamaAdminImpl.java (리비전 1004870) +++ src/java/org/apache/hama/HamaAdminImpl.java (작업 사본) @@ -1,237 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama; - -import java.io.IOException; - -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.RegionException; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.Matrix; -import org.apache.hama.matrix.SparseMatrix; -import org.apache.log4j.Logger; - -/** - * An Implementation of {@link org.apache.hama.HamaAdmin} to manage the matrix's - * namespace, and table allocation & garbage collection. - */ -public class HamaAdminImpl implements HamaAdmin { - static final Logger LOG = Logger.getLogger(HamaAdminImpl.class); - protected HamaConfiguration conf; - protected HBaseAdmin admin; - protected HTable table; - - /** - * Constructor - * - * @param conf - * @throws MasterNotRunningException - */ - public HamaAdminImpl(HamaConfiguration conf) throws MasterNotRunningException { - this.conf = conf; - this.admin = new HBaseAdmin(conf); - initialJob(); - } - - /** - * Constructor - * - * @param conf - * @param admin - */ - public HamaAdminImpl(HamaConfiguration conf, HBaseAdmin admin) { - this.conf = conf; - this.admin = admin; - initialJob(); - } - - /** - * Initializing the admin. - */ - private void initialJob() { - try { - if (!admin.tableExists(Constants.ADMINTABLE)) { - HTableDescriptor tableDesc = new HTableDescriptor(Constants.ADMINTABLE); - tableDesc.addFamily(new HColumnDescriptor(Constants.PATHCOLUMN)); - admin.createTable(tableDesc); - } - - table = new HTable(conf, Constants.ADMINTABLE); - table.setAutoFlush(true); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * @param name - * @return real table name - */ - public String getPath(String name) { - try { - Get get = new Get(Bytes.toBytes(name)); - get.addFamily(Bytes.toBytes(Constants.PATHCOLUMN)); - byte[] result = table.get(get).getValue( - Bytes.toBytes(Constants.PATHCOLUMN), null); - return Bytes.toString(result); - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - - public boolean matrixExists(String matrixName) { - try { - Get get = new Get(Bytes.toBytes(matrixName)); - get.addFamily(Bytes.toBytes(Constants.PATHCOLUMN)); - byte[] result = table.get(get).getValue( - Bytes.toBytes(Constants.PATHCOLUMN), null); - - return (result == null) ? false : true; - } catch (IOException e) { - e.printStackTrace(); - return false; - } - } - - public boolean save(Matrix mat, String aliaseName) { - boolean result = false; - - // we just store the name -> path(tablename) here. - // the matrix type is stored in its hbase table. we don't need to store - // again. - - Put put = new Put(Bytes.toBytes(aliaseName)); - put.add(Bytes.toBytes(Constants.PATHCOLUMN), null, Bytes.toBytes(mat - .getPath())); - - try { - table.put(put); - - result = true; - } catch (IOException e) { - e.printStackTrace(); - } - - return result; - } - - /** remove the entry of 'matrixName' in admin table. * */ - private void removeEntry(String matrixName) throws IOException { - Delete del = new Delete(Bytes.toBytes(matrixName)); - table.delete(del); - } - - private int getReference(String tableName) throws IOException { - HTable matrix = new HTable(conf, tableName); - - Get get = new Get(Bytes.toBytes(Constants.METADATA)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = matrix.get(get).getValue( - Constants.ATTRIBUTE, - Bytes.toBytes(Constants.METADATA_REFERENCE)); - - return (result == null) ? 0 : Bytes.toInt(result); - } - - private void clearAliaseInfo(String tableName) throws IOException { - HTable matrix = new HTable(conf, tableName); - Delete del = new Delete(Bytes.toBytes(Constants.METADATA)); - del.deleteColumns(Bytes.toBytes(Constants.ALIASEFAMILY), Bytes - .toBytes("name")); - matrix.delete(del); - } - - /** - * we remove the aliase entry store in Admin table, and clear the aliase info - * store in matrix table. And check the reference of the matrix table: - * - * 1) if the reference of the matrix table is zero: we delete the table. 2) if - * the reference of the matrix table is not zero: we let the matrix who still - * reference the table to do the garbage collection. - */ - public void delete(String matrixName) throws IOException { - if (matrixExists(matrixName)) { - String tablename = getPath(matrixName); - - // i) remove the aliase entry first. - removeEntry(matrixName); - - if (tablename == null) { // a matrixName point to a null table. we delete - // the entry. - return; - } - - if (!admin.tableExists(tablename)) { // have not specified table. - return; - } - - // ii) clear the aliase info store in matrix table. - clearAliaseInfo(tablename); - - if (getReference(tablename) <= 0) { // no reference, do gc!! - if (admin.isTableEnabled(tablename)) { - while (admin.isTableEnabled(tablename)) { - try { - admin.disableTable(tablename); - } catch (RegionException e) { - LOG.warn(e); - } - } - - admin.deleteTable(tablename); - } - } - } - } - - @Override - public Matrix getMatrix(String matrixName) throws IOException { - String path = getPath(matrixName); - if (getType(path).equals("SparseMatrix")) - return new SparseMatrix(conf, path); - else - return new DenseMatrix(conf, path); - } - - private String getType(String path) { - try { - HTable table = new HTable(conf, path); - - Get get = new Get(Bytes.toBytes(Constants.METADATA)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = table.get(get).getValue( - Constants.ATTRIBUTE, Bytes.toBytes("type")); - - return Bytes.toString(result); - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } -} Index: src/java/org/apache/hama/matrix/AbstractMatrix.java =================================================================== --- src/java/org/apache/hama/matrix/AbstractMatrix.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/AbstractMatrix.java (작업 사본) @@ -1,447 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.RegionException; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hama.Constants; -import org.apache.hama.HamaAdmin; -import org.apache.hama.HamaAdminImpl; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.util.BytesUtil; -import org.apache.hama.util.RandomVariable; -import org.apache.log4j.Logger; - -/** - * Methods of the matrix classes - */ -public abstract class AbstractMatrix implements Matrix { - static int tryPathLength = Constants.DEFAULT_PATH_LENGTH; - static final Logger LOG = Logger.getLogger(AbstractMatrix.class); - - protected HamaConfiguration config; - protected HBaseAdmin admin; - // a matrix just need a table path to point to the table which stores matrix. - // let HamaAdmin manage Matrix Name space. - protected String matrixPath; - protected HTable table; - protected HTableDescriptor tableDesc; - protected HamaAdmin hamaAdmin; - - protected boolean closed = true; - - /** a matrix copy of the original copy collected in "eicol" family * */ - public static final String EICOL = "eicol"; - /** a column family collect all values and statuses used during computation * */ - public static final String EI = "eival"; - /** a matrix collect all the eigen vectors * */ - public static final String EIVEC = "eivec"; - - /** - * Sets the job configuration - * - * @param conf configuration object - * @throws MasterNotRunningException - */ - public void setConfiguration(HamaConfiguration conf) - throws MasterNotRunningException { - this.config = conf; - this.admin = new HBaseAdmin(config); - - hamaAdmin = new HamaAdminImpl(conf, admin); - } - - /** - * try to create a new matrix with a new random name. try times will be - * (Integer.MAX_VALUE - 4) * DEFAULT_TRY_TIMES; - * - * @throws IOException - */ - protected void tryToCreateTable(String table_prefix) throws IOException { - int tryTimes = Constants.DEFAULT_TRY_TIMES; - do { - matrixPath = table_prefix + "_" - + RandomVariable.randMatrixPath(tryPathLength); - - if (!admin.tableExists(matrixPath)) { // no table 'matrixPath' in hbase. - tableDesc = new HTableDescriptor(matrixPath); - create(); - return; - } - - tryTimes--; - if (tryTimes <= 0) { // this loop has exhausted DEFAULT_TRY_TIMES. - tryPathLength++; - tryTimes = Constants.DEFAULT_TRY_TIMES; - } - - } while (tryPathLength <= Constants.DEFAULT_MAXPATHLEN); - // exhaustes the try times. - // throw out an IOException to let the user know what happened. - throw new IOException("Try too many times to create a table in hbase."); - } - - /** - * Create matrix space - */ - protected void create() throws IOException { - // It should run only when table doesn't exist. - if (!admin.tableExists(matrixPath)) { - this.tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMNFAMILY)); - this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE)); - this.tableDesc.addFamily(new HColumnDescriptor(Bytes - .toBytes(Constants.ALIASEFAMILY))); - - // It's a temporary data. - this.tableDesc.addFamily(new HColumnDescriptor(Bytes - .toBytes(Constants.BLOCK))); - // the following families are used in JacobiEigenValue computation - this.tableDesc.addFamily(new HColumnDescriptor(Bytes - .toBytes(EI))); - this.tableDesc.addFamily(new HColumnDescriptor(Bytes - .toBytes(EICOL))); - this.tableDesc.addFamily(new HColumnDescriptor(Bytes - .toBytes(EIVEC))); - - LOG.info("Initializing the matrix storage."); - this.admin.createTable(this.tableDesc); - LOG.info("Create Matrix " + matrixPath); - - // connect to the table. - table = new HTable(config, matrixPath); - table.setAutoFlush(true); - - // Record the matrix type in METADATA_TYPE - Put put = new Put(Bytes.toBytes(Constants.METADATA)); - put.add(Constants.ATTRIBUTE, Bytes.toBytes(Constants.METADATA_TYPE), - Bytes.toBytes(this.getClass().getSimpleName())); - table.put(put); - - // the new matrix's reference is 1. - setReference(1); - } - } - - public HTable getHTable() { - return this.table; - } - - /** {@inheritDoc} */ - public int getRows() throws IOException { - Get get = new Get(Bytes.toBytes(Constants.METADATA)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = table.get(get).getValue(Constants.ATTRIBUTE, - Bytes.toBytes("rows")); - - return (result != null) ? Bytes.toInt(result) : 0; - } - - /** {@inheritDoc} */ - public int getColumns() throws IOException { - Get get = new Get(Bytes.toBytes(Constants.METADATA)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = table.get(get).getValue(Constants.ATTRIBUTE, - Bytes.toBytes("columns")); - - return Bytes.toInt(result); - } - - /** {@inheritDoc} */ - public String getRowLabel(int row) throws IOException { - Get get = new Get(BytesUtil.getRowIndex(row)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = table.get(get).getValue(Constants.ATTRIBUTE, - Bytes.toBytes("string")); - - return (result != null) ? Bytes.toString(result) : null; - } - - /** {@inheritDoc} */ - public void setColumnLabel(int column, String name) throws IOException { - Put put = new Put(Bytes.toBytes(Constants.CINDEX)); - put.add(Constants.ATTRIBUTE, Bytes.toBytes(String.valueOf(column)), Bytes - .toBytes(name)); - table.put(put); - } - - /** {@inheritDoc} */ - public String getColumnLabel(int column) throws IOException { - Get get = new Get(Bytes.toBytes(Constants.CINDEX)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = table.get(get).getValue(Constants.ATTRIBUTE, - Bytes.toBytes(String.valueOf(column))); - - return (result != null) ? Bytes.toString(result) : null; - } - - /** {@inheritDoc} */ - public void setRowLabel(int row, String name) throws IOException { - Put put = new Put(BytesUtil.getRowIndex(row)); - put.add(Constants.ATTRIBUTE, Bytes.toBytes("string"), Bytes - .toBytes(name)); - table.put(put); - } - - /** {@inheritDoc} */ - public void setDimension(int rows, int columns) throws IOException { - Put put = new Put(Bytes.toBytes(Constants.METADATA)); - put.add(Constants.ATTRIBUTE, Bytes.toBytes("rows"), Bytes.toBytes(rows)); - put.add(Constants.ATTRIBUTE, Bytes.toBytes("columns"), Bytes - .toBytes(columns)); - table.put(put); - } - - /** {@inheritDoc} */ - public void add(int i, int j, double value) throws IOException { - Put put = new Put(BytesUtil.getRowIndex(i)); - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(j)), - Bytes.toBytes(value + this.get(i, j))); - table.put(put); - - } - - public static class ScanMapper extends - TableMapper implements Configurable { - private static Double alpha = null; - private Configuration conf = null; - - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - Put put = new Put(key.get()); - - NavigableMap> map = value - .getNoVersionMap(); - for (Map.Entry> a : map.entrySet()) { - byte[] family = a.getKey(); - for (Map.Entry b : a.getValue().entrySet()) { - byte[] qualifier = b.getKey(); - byte[] val = b.getValue(); - if (alpha.equals(new Double(1))) { - put.add(family, qualifier, val); - } else { - if (Bytes.toString(family).equals( - Bytes.toString(Constants.COLUMNFAMILY))) { - double currVal = Bytes.toDouble(val); - put.add(family, qualifier, Bytes.toBytes(currVal * alpha)); - } else { - put.add(family, qualifier, val); - } - } - } - } - - context.write(key, put); - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - Float f = conf.getFloat("set.alpha", 1); - alpha = f.doubleValue(); - } - } - - /** {@inheritDoc} */ - public Matrix set(Matrix B) throws IOException { - Job job = new Job(config, "set MR job : " + this.getPath()); - - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - scan.addFamily(Constants.ATTRIBUTE); - scan.addFamily(Bytes.toBytes(Constants.ALIASEFAMILY)); - scan.addFamily(Bytes.toBytes(Constants.BLOCK)); - scan.addFamily(Bytes.toBytes(EI)); - scan.addFamily(Bytes.toBytes(EICOL)); - scan.addFamily(Bytes.toBytes(EIVEC)); - - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableMapperJob(B - .getPath(), scan, ScanMapper.class, ImmutableBytesWritable.class, - Put.class, job); - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableReducerJob( - this.getPath(), - org.apache.hadoop.hbase.mapreduce.IdentityTableReducer.class, job); - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - return this; - } - - /** {@inheritDoc} */ - public Matrix set(double alpha, Matrix B) throws IOException { - Job job = new Job(config, "set MR job : " + this.getPath()); - - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - scan.addFamily(Constants.ATTRIBUTE); - scan.addFamily(Bytes.toBytes(Constants.ALIASEFAMILY)); - scan.addFamily(Bytes.toBytes(Constants.BLOCK)); - scan.addFamily(Bytes.toBytes(EI)); - scan.addFamily(Bytes.toBytes(EICOL)); - scan.addFamily(Bytes.toBytes(EIVEC)); - Float f = new Float(alpha); - job.getConfiguration().setFloat("set.alpha", f); - - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableMapperJob(B - .getPath(), scan, ScanMapper.class, ImmutableBytesWritable.class, - Put.class, job); - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableReducerJob( - this.getPath(), - org.apache.hadoop.hbase.mapreduce.IdentityTableReducer.class, job); - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - return this; - } - - /** {@inheritDoc} */ - public String getPath() { - return matrixPath; - } - - protected void setReference(int reference) throws IOException { - Put put = new Put(Bytes.toBytes(Constants.METADATA)); - put.add(Constants.ATTRIBUTE, Bytes.toBytes(Constants.METADATA_REFERENCE), - Bytes.toBytes(reference)); - table.put(put); - } - - protected int incrementAndGetRef() throws IOException { - int reference = 1; - - Get get = new Get(Bytes.toBytes(Constants.METADATA)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = table.get(get).getValue(Constants.ATTRIBUTE, - Bytes.toBytes(Constants.METADATA_REFERENCE)); - - if (result != null) { - reference = Bytes.toInt(result); - reference++; - } - setReference(reference); - return reference; - } - - protected int decrementAndGetRef() throws IOException { - int reference = 0; - - Get get = new Get(Bytes.toBytes(Constants.METADATA)); - get.addFamily(Constants.ATTRIBUTE); - byte[] result = table.get(get).getValue(Constants.ATTRIBUTE, - Bytes.toBytes(Constants.METADATA_REFERENCE)); - - if (result != null) { - reference = Bytes.toInt(result); - if (reference > 0) // reference==0, we need not to decrement it. - reference--; - } - setReference(reference); - return reference; - } - - protected boolean hasAliaseName() throws IOException { - Get get = new Get(Bytes.toBytes(Constants.METADATA)); - get.addFamily(Bytes.toBytes(Constants.ALIASEFAMILY)); - byte[] result = table.get(get).getValue( - Bytes.toBytes(Constants.ALIASEFAMILY), Bytes.toBytes("name")); - - return (result != null) ? true : false; - } - - public void close() throws IOException { - if (closed) // have been closed - return; - int reference = decrementAndGetRef(); - if (reference <= 0) { // no reference again. - if (!hasAliaseName()) { // the table has not been aliased, we delete the - // table. - if (admin.isTableEnabled(matrixPath)) { - while (admin.isTableEnabled(matrixPath)) { - try { - admin.disableTable(matrixPath); - } catch (RegionException e) { - LOG.warn(e); - } - } - - admin.deleteTable(matrixPath); - } - } - } - closed = true; - } - - public boolean checkAllJobs(List jobId) throws IOException { - Iterator it = jobId.iterator(); - boolean allTrue = true; - while (it.hasNext()) { - if (!it.next().isComplete()) { - allTrue = false; - } - } - - return allTrue; - } - - public boolean save(String aliasename) throws IOException { - // mark & update the aliase name in "alise:name" meta column. - // ! one matrix has only one aliasename now. - Put put = new Put(Bytes.toBytes(Constants.METADATA)); - put.add(Bytes.toBytes(Constants.ALIASEFAMILY), Bytes.toBytes("name"), Bytes - .toBytes(aliasename)); - put.add(Constants.ATTRIBUTE, Bytes.toBytes("type"), Bytes.toBytes(this - .getType())); - table.put(put); - - return hamaAdmin.save(this, aliasename); - } -} Index: src/java/org/apache/hama/matrix/AbstractVector.java =================================================================== --- src/java/org/apache/hama/matrix/AbstractVector.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/AbstractVector.java (작업 사본) @@ -1,103 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.util.Iterator; -import java.util.Map; -import java.util.NavigableMap; - -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hama.Constants; -import org.apache.hama.util.BytesUtil; -import org.apache.log4j.Logger; - -/** - * Methods of the vector classes - */ -public abstract class AbstractVector { - static final Logger LOG = Logger.getLogger(AbstractVector.class); - protected MapWritable entries; - - public void initMap(Result rs) { - this.entries = new MapWritable(); - NavigableMap map = rs.getFamilyMap(Constants.COLUMNFAMILY); - for (Map.Entry e : map.entrySet()) { - if(e != null) { - this.entries.put(new IntWritable(BytesUtil.bytesToInt(e.getKey())), - new DoubleWritable(Bytes.toDouble(e.getValue()))); - } - } - } - - /** - * Returns an Iterator. - * - * @return iterator - */ - public Iterator iterator() { - return this.entries.values().iterator(); - } - - /** - * Returns a size of vector. If vector is sparse, returns the number of only - * non-zero elements. - * - * @return a size of vector - */ - public int size() { - int x = 0; - if (this.entries != null && this.entries.containsKey(new Text("row"))) - x = 1; - - return (this.entries != null) ? this.entries.size() - x : 0; - } - - /** - * Returns the {@link org.apache.hadoop.io.MapWritable} - * - * @return the entries of vector - */ - public MapWritable getEntries() { - return this.entries; - } - - /** - * Checks for conformant sizes - */ - protected void checkComformantSize(Vector v2) { - if (this.size() != v2.size()) { - throw new IndexOutOfBoundsException("v1.size != v2.size (" + this.size() - + " != " + v2.size() + ")"); - } - } - - /** - * Clears the entries. - */ - public void clear() { - this.entries = null; - } -} Index: src/java/org/apache/hama/matrix/DenseMatrix.java =================================================================== --- src/java/org/apache/hama/matrix/DenseMatrix.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/DenseMatrix.java (작업 사본) @@ -1,390 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Map; - -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.Constants; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.util.BytesUtil; -import org.apache.hama.util.RandomVariable; - -/** - * This class represents a dense matrix. - */ -public class DenseMatrix extends AbstractMatrix implements Matrix { - static private final String TABLE_PREFIX = DenseMatrix.class.getSimpleName(); - - /** - * Construct a raw matrix. Just create a table in HBase. - * - * @param conf configuration object - * @param m the number of rows. - * @param n the number of columns. - * @throws IOException throw the exception to let the user know what happend, - * if we didn't create the matrix successfully. - */ - public DenseMatrix(HamaConfiguration conf, int m, int n) throws IOException { - setConfiguration(conf); - - tryToCreateTable(TABLE_PREFIX); - closed = false; - this.setDimension(m, n); - } - - /** - * Create/load a matrix aliased as 'matrixName'. - * - * @param conf configuration object - * @param matrixName the name of the matrix - * @param force if force is true, a new matrix will be created no matter - * 'matrixName' has aliased to an existed matrix; otherwise, just try - * to load an existed matrix alised 'matrixName'. - * @throws IOException - */ - public DenseMatrix(HamaConfiguration conf, String matrixName, boolean force) - throws IOException { - setConfiguration(conf); - // if force is set to true: - // 1) if this matrixName has aliase to other matrix, we will remove - // the old aliase, create a new matrix table, and aliase to it. - - // 2) if this matrixName has no aliase to other matrix, we will create - // a new matrix table, and alise to it. - // - // if force is set to false, we just try to load an existed matrix alised - // as 'matrixname'. - - boolean existed = hamaAdmin.matrixExists(matrixName); - - if (force) { - if (existed) { - // remove the old aliase - hamaAdmin.delete(matrixName); - } - // create a new matrix table. - tryToCreateTable(TABLE_PREFIX); - // save the new aliase relationship - save(matrixName); - } else { - if (existed) { - // try to get the actual path of the table - matrixPath = hamaAdmin.getPath(matrixName); - // load the matrix - table = new HTable(conf, matrixPath); - // increment the reference - incrementAndGetRef(); - } else { - throw new IOException("Try to load non-existed matrix alised as " - + matrixName); - } - } - - closed = false; - } - - /** - * Load a matrix from an existed matrix table whose tablename is 'matrixpath' - * !! It is an internal used for map/reduce. - * - * @param conf configuration object - * @param matrixpath - * @throws IOException - * @throws IOException - */ - public DenseMatrix(HamaConfiguration conf, String matrixpath) - throws IOException { - setConfiguration(conf); - matrixPath = matrixpath; - // load the matrix - table = new HTable(conf, matrixPath); - // TODO: now we don't increment the reference of the table - // for it's an internal use for map/reduce. - // if we want to increment the reference of the table, - // we don't know where to call Matrix.close in Add & Mul map/reduce - // process to decrement the reference. It seems difficulty. - } - - /** - * Create an m-by-n constant matrix. - * - * @param conf configuration object - * @param m the number of rows. - * @param n the number of columns. - * @param s fill the matrix with this scalar value. - * @throws IOException throw the exception to let the user know what happend, - * if we didn't create the matrix successfully. - */ - public DenseMatrix(HamaConfiguration conf, int m, int n, double s) - throws IOException { - setConfiguration(conf); - - tryToCreateTable(TABLE_PREFIX); - - closed = false; - - for (int i = 0; i < m; i++) { - for (int j = 0; j < n; j++) { - set(i, j, s); - } - } - - setDimension(m, n); - } - - /** - * Generate matrix with random elements - * - * @param conf configuration object - * @param m the number of rows. - * @param n the number of columns. - * @return an m-by-n matrix with uniformly distributed random elements. - * @throws IOException - */ - public static DenseMatrix random(HamaConfiguration conf, int m, int n) - throws IOException { - DenseMatrix rand = new DenseMatrix(conf, m, n); - DenseVector vector = new DenseVector(); - LOG.info("Create the " + m + " * " + n + " random matrix : " - + rand.getPath()); - - for (int i = 0; i < m; i++) { - vector.clear(); - for (int j = 0; j < n; j++) { - vector.set(j, RandomVariable.rand()); - } - rand.setRow(i, vector); - } - - return rand; - } - - /** - * Generate identity matrix - * - * @param conf configuration object - * @param m the number of rows. - * @param n the number of columns. - * @return an m-by-n matrix with ones on the diagonal and zeros elsewhere. - * @throws IOException - */ - public static DenseMatrix identity(HamaConfiguration conf, int m, int n) - throws IOException { - DenseMatrix identity = new DenseMatrix(conf, m, n); - LOG.info("Create the " + m + " * " + n + " identity matrix : " - + identity.getPath()); - - for (int i = 0; i < m; i++) { - DenseVector vector = new DenseVector(); - for (int j = 0; j < n; j++) { - vector.set(j, (i == j ? 1.0 : 0.0)); - } - identity.setRow(i, vector); - } - - return identity; - } - - /** - * Gets the double value of (i, j) - * - * @param i ith row of the matrix - * @param j jth column of the matrix - * @return the value of entry, or zero If entry is null - * @throws IOException - */ - public double get(int i, int j) throws IOException { - if (this.getRows() < i || this.getColumns() < j) - throw new ArrayIndexOutOfBoundsException(i + ", " + j); - - Get get = new Get(BytesUtil.getRowIndex(i)); - get.addColumn(Constants.COLUMNFAMILY); - byte[] result = table.get(get).getValue(Constants.COLUMNFAMILY, - Bytes.toBytes(String.valueOf(j))); - - if (result == null) - throw new NullPointerException("Unexpected null"); - - return Bytes.toDouble(result); - } - - /** - * Gets the vector of row - * - * @param i the row index of the matrix - * @return the vector of row - * @throws IOException - */ - public DenseVector getRow(int i) throws IOException { - Get get = new Get(BytesUtil.getRowIndex(i)); - get.addFamily(Constants.COLUMNFAMILY); - Result r = table.get(get); - return new DenseVector(r); - } - - /** - * Gets the vector of column - * - * @param j the column index of the matrix - * @return the vector of column - * @throws IOException - */ - public DenseVector getColumn(int j) throws IOException { - Scan scan = new Scan(); - scan.addColumn(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(j))); - ResultScanner s = table.getScanner(scan); - Result r = null; - - MapWritable trunk = new MapWritable(); - while ((r = s.next()) != null) { - byte[] value = r.getValue(Constants.COLUMNFAMILY, Bytes.toBytes(String - .valueOf(j))); - LOG.info(Bytes.toString(r.getRow())); - trunk.put(new IntWritable(BytesUtil.getRowIndex(r.getRow())), - new DoubleWritable(Bytes.toDouble(value))); - } - - return new DenseVector(trunk); - } - - /** {@inheritDoc} */ - public void set(int i, int j, double value) throws IOException { - if (this.getRows() < i || this.getColumns() < j) - throw new ArrayIndexOutOfBoundsException(this.getRows() + ", " - + this.getColumns() + ": " + i + ", " + j); - Put put = new Put(BytesUtil.getRowIndex(i)); - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(j)), Bytes - .toBytes(value)); - table.put(put); - } - - /** - * Set the row of a matrix to a given vector - * - * @param row - * @param vector - * @throws IOException - */ - public void setRow(int row, Vector vector) throws IOException { - if (this.getRows() < row || this.getColumns() < vector.size()) - throw new ArrayIndexOutOfBoundsException(row); - Put put = new Put(BytesUtil.getRowIndex(row)); - for (Map.Entry e : vector.getEntries().entrySet()) { - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String - .valueOf(((IntWritable) e.getKey()).get())), Bytes - .toBytes(((DoubleWritable) e.getValue()).get())); - } - table.put(put); - } - - /** - * Set the column of a matrix to a given vector - * - * @param column - * @param vector - * @throws IOException - */ - public void setColumn(int column, Vector vector) throws IOException { - if (this.getColumns() < column || this.getRows() < vector.size()) - throw new ArrayIndexOutOfBoundsException(column); - - for (Map.Entry e : vector.getEntries().entrySet()) { - int key = ((IntWritable) e.getKey()).get(); - double value = ((DoubleWritable) e.getValue()).get(); - Put put = new Put(BytesUtil.getRowIndex(key)); - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(column)), - Bytes.toBytes(value)); - table.put(put); - } - } - - /** - * C = alpha*A*B + C - * - * @param alpha - * @param B - * @param C - * @return C - * @throws IOException - */ - public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException { - // TODO Auto-generated method stub - return null; - } - - /** - * Returns type of matrix - */ - public String getType() { - return this.getClass().getSimpleName(); - } - - /** - * Returns the sub matrix formed by selecting certain rows and columns from a - * bigger matrix. The sub matrix is a in-memory operation only. - * - * @param i0 the start index of row - * @param i1 the end index of row - * @param j0 the start index of column - * @param j1 the end index of column - * @return the sub matrix of matrix - * @throws IOException - */ - public SubMatrix subMatrix(int i0, int i1, int j0, int j1) throws IOException { - int columnSize = (j1 - j0) + 1; - SubMatrix result = new SubMatrix((i1 - i0) + 1, columnSize); - - Scan scan = new Scan(); - for (int j = j0, jj = 0; j <= j1; j++, jj++) { - scan.addColumn(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(j))); - } - scan.setStartRow(BytesUtil.getRowIndex(i0)); - scan.setStopRow(BytesUtil.getRowIndex(i1 + 1)); - ResultScanner s = table.getScanner(scan); - Iterator it = s.iterator(); - - int i = 0; - Result rs = null; - while (it.hasNext()) { - rs = it.next(); - for (int j = j0, jj = 0; j <= j1; j++, jj++) { - byte[] vv = rs.getValue(Constants.COLUMNFAMILY, Bytes.toBytes(String - .valueOf(j))); - result.set(i, jj, vv); - } - i++; - } - - return result; - } -} Index: src/java/org/apache/hama/matrix/DenseVector.java =================================================================== --- src/java/org/apache/hama/matrix/DenseVector.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/DenseVector.java (작업 사본) @@ -1,323 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.Constants; -import org.apache.log4j.Logger; - -/** - * This class represents a dense vector. - */ -public class DenseVector extends AbstractVector implements Vector { - static final Logger LOG = Logger.getLogger(DenseVector.class); - - public DenseVector() { - this(new MapWritable()); - } - - public DenseVector(MapWritable m) { - this.entries = m; - } - - public DenseVector(Result row) { - this.initMap(row); - } - - public DenseVector(int row, Result m) { - this.initMap(m); - this.entries.put(Constants.ROWCOUNT, new IntWritable(row)); - } - - public DenseVector(int row, MapWritable m) { - this.entries = m; - this.entries.put(Constants.ROWCOUNT, new IntWritable(row)); - } - - /** - * Sets the value of index - * - * @param index - * @param value - */ - public void set(int index, double value) { - // If entries are null, create new object - if (this.entries == null) { - this.entries = new MapWritable(); - } - - this.entries.put(new IntWritable(index), new DoubleWritable(value)); - } - - /** - * Sets the vector - * - * @param v - * @return x = v - */ - public DenseVector set(Vector v) { - this.set(1, v); - return this; - } - - public Vector set(double alpha, Vector v) { - checkComformantSize(v); - boolean zeroFill = false; - if (alpha == 0) - zeroFill = true; - - for (Map.Entry e : v.getEntries().entrySet()) { - int key = ((IntWritable) e.getKey()).get(); - if (zeroFill) - this.set(key, 0); - else - this.set(key, alpha * ((DoubleWritable) e.getValue()).get()); - } - - return this; - } - - public void setRow(int row) { - this.entries.put(Constants.ROWCOUNT, new IntWritable(row)); - } - - /** - * Gets the value of index - * - * @param index - * @return the value of v(index) - * @throws IOException - */ - public double get(int index) { - double value; - try { - value = ((DoubleWritable) this.entries.get(new IntWritable(index))).get(); - } catch (NullPointerException e) { - throw new NullPointerException("Unexpected null value : " + e.toString()); - } - - return value; - } - - public int getRow() { - return ((IntWritable) this.entries.get(Constants.ROWCOUNT)).get(); - } - - /** - * Adds the value to v(index) - * - * @param index - * @param value - */ - public void add(int index, double value) { - set(index, get(index) + value); - } - - /** - * x = alpha*v + x - * - * @param alpha - * @param v - * @return x = alpha*v + x - */ - public DenseVector add(double alpha, Vector v) { - checkComformantSize(v); - if (alpha == 0) - return this; - - for (Map.Entry e : this.getEntries().entrySet()) { - int key = ((IntWritable) e.getKey()).get(); - this.add(key, alpha * v.get(key)); - } - - return this; - } - - /** - * x = v + x - * - * @param v2 - * @return x = v + x - */ - public DenseVector add(Vector v2) { - checkComformantSize(v2); - - for (Map.Entry e : this.getEntries().entrySet()) { - int key = ((IntWritable) e.getKey()).get(); - this.add(key, v2.get(key)); - } - - return this; - } - - /** - * x dot v - * - * @param v - * @return x dot v - */ - public double dot(Vector v) { - checkComformantSize(v); - - double cosine = 0.0; - double q_i, d_i; - for (int i = 0; i < Math.min(this.size(), v.size()); i++) { - q_i = v.get(i); - d_i = this.get(i); - cosine += q_i * d_i; - } - return cosine / (this.getNorm2() * ((DenseVector) v).getNorm2()); - } - - /** - * v = alpha*v - * - * @param alpha - * @return v = alpha*v - */ - public DenseVector scale(double alpha) { - for (Map.Entry e : this.entries.entrySet()) { - this.entries.put(e.getKey(), new DoubleWritable(((DoubleWritable) e - .getValue()).get() - * alpha)); - } - return this; - } - - /** - * Computes the given norm of the vector - * - * @param type - * @return norm of the vector - */ - public double norm(Norm type) { - if (type == Norm.One) - return getNorm1(); - else if (type == Norm.Two) - return getNorm2(); - else if (type == Norm.TwoRobust) - return getNorm2Robust(); - else - return getNormInf(); - } - - protected double getNorm1() { - double sum = 0.0; - - Set keySet = this.entries.keySet(); - Iterator it = keySet.iterator(); - - while (it.hasNext()) { - sum += get(((IntWritable) it.next()).get()); - } - - return sum; - } - - protected double getNorm2() { - double square_sum = 0.0; - - Set keySet = entries.keySet(); - Iterator it = keySet.iterator(); - - while (it.hasNext()) { - double value = get(((IntWritable) it.next()).get()); - square_sum += value * value; - } - - return Math.sqrt(square_sum); - } - - /** - * Returns the robust norm of the vector - * - * @return the robust norm of the vector - */ - protected double getNorm2Robust() { - double scale = 0, ssq = 1; - for (int i = 0; i < this.size(); i++) { - double val = get(i); - if (val != 0) { - double absxi = Math.abs(val); - if (scale < absxi) { - ssq = 1 + ssq * Math.pow(scale / absxi, 2); - scale = absxi; - } else - ssq = ssq + Math.pow(absxi / scale, 2); - } - } - return scale * Math.sqrt(ssq); - } - - /** - * Returns the infinity norm of the vector - * - * @return the infinity norm of the vector - */ - protected double getNormInf() { - double max = 0.0; - for (int i = 0; i < this.size(); i++) { - max = Math.max(max, Math.abs(get(i))); - } - return max; - } - - /** - * Returns a sub-vector. - * - * @param i0 the index of the first element - * @param i1 the index of the last element - * @return v[i0:i1] - */ - public DenseVector subVector(int i0, int i1) { - DenseVector res = new DenseVector(); - if (this.entries.containsKey(Constants.ROWCOUNT)) - res.setRow(this.getRow()); - - for (int i = i0; i <= i1; i++) { - res.set(i, get(i)); - } - - return res; - } - - public void zeroFill(int size) { - for (int i = 0; i < size; i++) { - this.set(i, 0); - } - } - - public String toString() { - StringBuilder result = new StringBuilder(); - for (int i = 0; i < this.size(); i++) { - result.append(this.get(i) + " "); - } - return result.toString(); - } -} Index: src/java/org/apache/hama/matrix/Matrix.java =================================================================== --- src/java/org/apache/hama/matrix/Matrix.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/Matrix.java (작업 사본) @@ -1,242 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; - -/** - * Basic matrix interface. - */ -public interface Matrix { - - /** - * Gets the double value of (i, j) - * - * @param i ith row of the matrix - * @param j jth column of the matrix - * @return the value of entry - * @throws IOException - */ - public double get(int i, int j) throws IOException; - - /** - * Gets the vector of row - * - * @param i the row index of the matrix - * @return the vector of row - * @throws IOException - */ - public Vector getRow(int i) throws IOException; - - /** - * Gets the vector of column - * - * @param j the column index of the matrix - * @return the vector of column - * @throws IOException - */ - public Vector getColumn(int j) throws IOException; - - /** - * Get the number of row of the matrix from the meta-data column - * - * @return a number of rows of the matrix - * @throws IOException - */ - public int getRows() throws IOException; - - /** - * Get the number of column of the matrix from the meta-data column - * - * @return a number of columns of the matrix - * @throws IOException - */ - public int getColumns() throws IOException; - - /** - * Gets the label of the row - * - * @throws IOException - */ - public String getRowLabel(int i) throws IOException; - - /** - * Gets the label of the column - * - * @throws IOException - */ - public String getColumnLabel(int j) throws IOException; - - /** - * Return the matrix path. - * (in hbase, path is the tablename. in filesystem, path may be a file path.) - * - * @return the name of the matrix - */ - public String getPath(); - - /** - * Sets the label of the row - * - * @param i - * @param name - * @throws IOException - */ - public void setRowLabel(int i, String name) throws IOException; - - /** - * Sets the label of the column - * - * @param j - * @param name - * @throws IOException - */ - public void setColumnLabel(int j, String name) throws IOException; - - /** - * Sets the double value of (i, j) - * - * @param i ith row of the matrix - * @param j jth column of the matrix - * @param value the value of entry - * @throws IOException - */ - public void set(int i, int j, double value) throws IOException; - - /** - * A=alpha*B - * - * @param alpha - * @param B - * @return A - * @throws IOException - */ - public Matrix set(double alpha, Matrix B) throws IOException; - - /** - * A=B - * - * @param B - * @return A - * @throws IOException - */ - public Matrix set(Matrix B) throws IOException; - - /** - * Set the row of a matrix to a given vector - * - * @param row - * @param vector - * @throws IOException - */ - public void setRow(int row, Vector vector) throws IOException; - - /** - * Set the column of a matrix to a given vector - * - * @param column - * @param vector - * @throws IOException - */ - public void setColumn(int column, Vector vector) throws IOException; - - /** - * Sets the dimension of matrix - * - * @param rows the number of rows - * @param columns the number of columns - * @throws IOException - */ - public void setDimension(int rows, int columns) throws IOException; - - /** - * A(i, j) += value - * - * @param i - * @param j - * @param value - * @throws IOException - */ - public void add(int i, int j, double value) throws IOException; - - /** - * C = alpha*A*B + C - * - * @param alpha - * @param B - * @param C - * @return C - * @throws IOException - */ - public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException; - - /** - * Supported matrix-norms. - */ - enum Norm { - /** Maximum absolute column sum */ - One, - - /** The root of sum of the sum of squares */ - Frobenius, - - /** The maximum absolute row sum */ - Infinity, - - /** Largest entry in absolute value. */ - Maxvalue - } - - /** - * Save to a table or file - * - * @param path - * @return true if saved - * @throws IOException - */ - public boolean save(String path) throws IOException; - - /** - * Returns the matrix type - * - * @return the matrix type - */ - public String getType(); - - /** - * Returns the sub matrix formed by selecting certain rows and - * columns from a bigger matrix. The sub matrix is a in-memory operation only. - * - * @param i0 the start index of row - * @param i1 the end index of row - * @param j0 the start index of column - * @param j1 the end index of column - * @return the sub matrix of matrix - * @throws IOException - */ - public SubMatrix subMatrix(int i0, int i1, int j0, int j1) throws IOException; - - /** - * Close current matrix. - * - * @throws Exception - */ - public void close() throws IOException; -} Index: src/java/org/apache/hama/matrix/SparseMatrix.java =================================================================== --- src/java/org/apache/hama/matrix/SparseMatrix.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/SparseMatrix.java (작업 사본) @@ -1,188 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.Constants; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.util.BytesUtil; -import org.apache.hama.util.RandomVariable; - -public class SparseMatrix extends AbstractMatrix implements Matrix { - static private final String TABLE_PREFIX = SparseMatrix.class.getSimpleName(); - - public SparseMatrix(HamaConfiguration conf, int m, int n) throws IOException { - setConfiguration(conf); - - tryToCreateTable(TABLE_PREFIX); - closed = false; - this.setDimension(m, n); - } - - /** - * Load a matrix from an existed matrix table whose tablename is 'matrixpath' !! - * It is an internal used for map/reduce. - * - * @param conf configuration object - * @param matrixpath - * @throws IOException - * @throws IOException - */ - public SparseMatrix(HamaConfiguration conf, String matrixpath) - throws IOException { - setConfiguration(conf); - matrixPath = matrixpath; - // load the matrix - table = new HTable(conf, matrixPath); - // TODO: now we don't increment the reference of the table - // for it's an internal use for map/reduce. - // if we want to increment the reference of the table, - // we don't know where to call Matrix.close in Add & Mul map/reduce - // process to decrement the reference. It seems difficulty. - } - - /** - * Generate matrix with random elements - * - * @param conf configuration object - * @param m the number of rows. - * @param n the number of columns. - * @return an m-by-n matrix with uniformly distributed random elements. - * @throws IOException - */ - public static SparseMatrix random(HamaConfiguration conf, int m, int n) - throws IOException { - SparseMatrix rand = new SparseMatrix(conf, m, n); - SparseVector vector = new SparseVector(); - LOG.info("Create the " + m + " * " + n + " random matrix : " - + rand.getPath()); - - for (int i = 0; i < m; i++) { - vector.clear(); - for (int j = 0; j < n; j++) { - Random r = new Random(); - if (r.nextInt(2) != 0) - vector.set(j, RandomVariable.rand()); - } - rand.setRow(i, vector); - } - - return rand; - } - - @Override - public double get(int i, int j) throws IOException { - if (this.getRows() < i || this.getColumns() < j) - throw new ArrayIndexOutOfBoundsException(i + ", " + j); - - Get get = new Get(BytesUtil.getRowIndex(i)); - get.addColumn(Constants.COLUMNFAMILY); - byte[] result = table.get(get).getValue(Constants.COLUMNFAMILY, - Bytes.toBytes(String.valueOf(j))); - return (result != null) ? Bytes.toDouble(result) : 0.0; - } - - @Override - public Vector getColumn(int j) throws IOException { - // TODO Auto-generated method stub - return null; - } - - /** - * Gets the vector of row - * - * @param i the row index of the matrix - * @return the vector of row - * @throws IOException - */ - public SparseVector getRow(int i) throws IOException { - Get get = new Get(BytesUtil.getRowIndex(i)); - get.addFamily(Constants.COLUMNFAMILY); - Result r = table.get(get); - // return new SparseVector(r.getRowResult()); - return new SparseVector(r); - } - - /** {@inheritDoc} */ - public void set(int i, int j, double value) throws IOException { - if (value != 0) { - Put put = new Put(BytesUtil.getRowIndex(i)); - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(j)), - Bytes.toBytes(value)); - table.put(put); - } - } - - /** - * Returns type of matrix - */ - public String getType() { - return this.getClass().getSimpleName(); - } - - @Override - public Matrix multAdd(double alpha, Matrix B, Matrix C) throws IOException { - // TODO Auto-generated method stub - return null; - } - - @Override - public void setColumn(int column, Vector vector) throws IOException { - // TODO Auto-generated method stub - - } - - @Override - public void setRow(int row, Vector vector) throws IOException { - if (this.getRows() < row) - throw new ArrayIndexOutOfBoundsException(row); - - if (vector.size() > 0) { // stores if size > 0 - Put put = new Put(BytesUtil.getRowIndex(row)); - for (Map.Entry e : ((SparseVector) vector).getEntries().entrySet()) { - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String.valueOf(((IntWritable) e.getKey()).get())), - Bytes.toBytes(((DoubleWritable) e.getValue()).get())); - } - table.put(put); - } - } - - @Override - public SubMatrix subMatrix(int i0, int i1, int j0, int j1) throws IOException { - // TODO Auto-generated method stub - return null; - } - -} Index: src/java/org/apache/hama/matrix/SparseVector.java =================================================================== --- src/java/org/apache/hama/matrix/SparseVector.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/SparseVector.java (작업 사본) @@ -1,184 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.util.Map; - -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Writable; -import org.apache.log4j.Logger; - -/** - * This class represents a sparse vector. - */ -public class SparseVector extends AbstractVector implements Vector { - static final Logger LOG = Logger.getLogger(SparseVector.class); - - public SparseVector() { - this(new MapWritable()); - } - - public SparseVector(MapWritable m) { - this.entries = m; - } - - public SparseVector(Result row) { - this.initMap(row); - } - - @Override - public Vector add(double alpha, Vector v) { - if (alpha == 0) - return this; - - for (Map.Entry e : v.getEntries().entrySet()) { - if (this.entries.containsKey(e.getKey())) { - // add - double value = alpha * ((DoubleWritable) e.getValue()).get() - + this.get(((IntWritable) e.getKey()).get()); - this.entries.put(e.getKey(), new DoubleWritable(value)); - } else { - // put - double value = alpha * ((DoubleWritable) e.getValue()).get(); - this.entries.put(e.getKey(), new DoubleWritable(value)); - } - } - - return this; - } - - /** - * x = v + x - * - * @param v2 - * @return x = v + x - */ - public SparseVector add(Vector v2) { - - for (Map.Entry e : v2.getEntries().entrySet()) { - int key = ((IntWritable) e.getKey()).get(); - if (this.entries.containsKey(e.getKey())) { - this.add(key, ((DoubleWritable) e.getValue()).get()); - } else { - this.set(key, ((DoubleWritable) e.getValue()).get()); - } - } - - return this; - } - - @Override - public double dot(Vector v) { - // TODO Auto-generated method stub - return 0; - } - - @Override - public double norm(Norm type) { - // TODO Auto-generated method stub - return 0; - } - - /** - * v = alpha*v - * - * @param alpha - * @return v = alpha*v - */ - public SparseVector scale(double alpha) { - for (Map.Entry e : this.entries.entrySet()) { - this.entries.put(e.getKey(), new DoubleWritable(((DoubleWritable) e - .getValue()).get() - * alpha)); - } - return this; - } - - /** - * Gets the value of index - * - * @param index - * @return the value of v(index) - * @throws IOException - */ - public double get(int index) { - double value; - try { - value = ((DoubleWritable) this.entries.get(new IntWritable(index))).get(); - } catch (NullPointerException e) { // returns zero if there is no value - return 0; - } - - return value; - } - - /** - * Sets the value of index - * - * @param index - * @param value - */ - public void set(int index, double value) { - // If entries are null, create new object - if (this.entries == null) { - this.entries = new MapWritable(); - } - - if (value != 0) // only stores non-zero element - this.entries.put(new IntWritable(index), new DoubleWritable(value)); - } - - /** - * Adds the value to v(index) - * - * @param index - * @param value - */ - public void add(int index, double value) { - set(index, get(index) + value); - } - - /** - * Sets the vector - * - * @param v - * @return x = v - */ - public SparseVector set(Vector v) { - return new SparseVector(v.getEntries()); - } - - @Override - public Vector subVector(int i0, int i1) { - // TODO Auto-generated method stub - return null; - } - - @Override - public Vector set(double alpha, Vector v) { - // TODO Auto-generated method stub - return null; - } - -} Index: src/java/org/apache/hama/matrix/SubMatrix.java =================================================================== --- src/java/org/apache/hama/matrix/SubMatrix.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/SubMatrix.java (작업 사본) @@ -1,222 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.log4j.Logger; - -/** - * A sub matrix is a matrix formed by selecting certain rows and columns from a - * bigger matrix. This is a in-memory operation only. - */ -public class SubMatrix { - static final Logger LOG = Logger.getLogger(SubMatrix.class); - private double[][] matrix; - - /** - * Constructor - * - * @param i the size of rows - * @param j the size of columns - */ - public SubMatrix(int i, int j) { - this.matrix = new double[i][j]; - } - - /** - * Constructor - * - * @param c a two dimensional double array - */ - public SubMatrix(double[][] c) { - double[][] matrix = c; - this.matrix = matrix; - } - - public SubMatrix(byte[] matrix) throws IOException { - ByteArrayInputStream bos = new ByteArrayInputStream(matrix); - DataInputStream dis = new DataInputStream(bos); - - int rows = dis.readInt(); - int columns = dis.readInt(); - this.matrix = new double[rows][columns]; - - for(int i = 0; i < rows; i++) { - for(int j = 0; j < columns; j++) { - this.matrix[i][j] = dis.readDouble(); - } - } - - dis.close(); - bos.close(); - } - - /** - * Sets the value - * - * @param row - * @param column - * @param value - */ - public void set(int row, int column, double value) { - matrix[row][column] = value; - } - - /** - * Sets the value - * - * @param row - * @param column - * @param value - */ - public void set(int row, int column, byte[] value) { - matrix[row][column] = Bytes.toDouble(value); - } - - /** - * Gets the value - * - * @param i - * @param j - * @return the value of submatrix(i, j) - */ - public double get(int i, int j) { - return matrix[i][j]; - } - - public void add(int row, int column, double value) { - matrix[row][column] = matrix[row][column] + value; - } - - /** - * c = a+b - * - * @param b - * @return c - */ - public SubMatrix add(SubMatrix b) { - SubMatrix c = new SubMatrix(this.getRows(), this.getColumns()); - - for (int i = 0; i < this.getRows(); i++) { - for (int j = 0; j < this.getColumns(); j++) { - c.set(i, j, (this.get(i, j) + b.get(i, j))); - } - } - - return c; - } - - /** - * c = a*b - * - * @param b - * @return c - */ - public SubMatrix mult(SubMatrix b) { - SubMatrix c = new SubMatrix(this.getRows(), b.getColumns()); - - for (int i = 0; i < this.getRows(); i++) { - for (int j = 0; j < b.getColumns(); j++) { - for (int k = 0; k < this.getColumns(); k++) { - c.add(i, j, this.get(i, k) * b.get(k, j)); - } - } - } - - return c; - } - - /** - * Gets the number of rows - * - * @return the number of rows - */ - public int getRows() { - return this.matrix.length; - } - - /** - * Gets the number of columns - * - * @return the number of columns - */ - public int getColumns() { - return this.matrix[0].length; - } - - /** - * Close - */ - public void close() { - matrix = null; - } - - /** - * @return the 2d double array - */ - public double[][] getDoubleArray() { - double[][] result = matrix; - return result; - } - - /** - * Gets the bytes of the sub matrix - * - * @return the bytes of the sub matrix - * @throws IOException - */ - public byte[] getBytes() throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(bos); - - dos.writeInt(this.getRows()); - dos.writeInt(this.getColumns()); - - for(int i = 0; i < this.getRows(); i++) { - for(int j = 0; j < this.getColumns(); j++) { - dos.writeDouble(this.get(i, j)); - } - } - - byte[] data = bos.toByteArray(); - dos.close(); - bos.close(); - return data; - } - - public String toString() { - StringBuilder result = new StringBuilder(); - for (int i = 0; i < this.getRows(); i++) { - for (int j = 0; j < this.getColumns(); j++) { - result.append(this.get(i, j)); - result.append('\t'); - } - result.append('\n'); - } - return result.toString(); - } -} - Index: src/java/org/apache/hama/matrix/Vector.java =================================================================== --- src/java/org/apache/hama/matrix/Vector.java (리비전 1004870) +++ src/java/org/apache/hama/matrix/Vector.java (작업 사본) @@ -1,161 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.util.Iterator; - -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Writable; - -/** - * Basic vector interface. - */ -public interface Vector { - - /** - * Size of the vector - * - * @return size of the vector - */ - public int size(); - - /** - * Gets the value of index - * - * @param index - * @return v(index) - */ - public double get(int index); - - /** - * Sets the value of index - * - * @param index - * @param value - */ - public void set(int index, double value); - - /** - * Sets the vector - * - * @param v - * @return x = v - */ - public Vector set(Vector v); - - /** - * x = alpha * v - * - * @param alpha - * @param v - * @return x = alpha * v - */ - public Vector set(double alpha, Vector v); - - /** - * Adds the value to v(index) - * - * @param index - * @param value - */ - public void add(int index, double value); - - /** - * x = alpha*v + x - * - * @param alpha - * @param v - * @return x = alpha*v + x - */ - public Vector add(double alpha, Vector v); - - /** - * x = v + x - * - * @param v - * @return x = v + x - */ - public Vector add(Vector v); - - /** - * x dot v - * - * @param v - * @return x dot v - */ - public double dot(Vector v); - - /** - * v = alpha*v - * - * @param alpha - * @return v = alpha*v - */ - public Vector scale(double alpha); - - /** - * Returns a sub-vector. - * - * @param i0 the index of the first element - * @param i1 the index of the last element - * @return v[i0:i1] - */ - public Vector subVector( int i0, int i1 ); - - /** - * Computes the given norm of the vector - * - * @param type - * @return norm of the vector - */ - public double norm(Norm type); - - /** - * Supported vector-norms. - */ - enum Norm { - - /** Sum of the absolute values of the entries */ - One, - - /** The root of sum of squares */ - Two, - - /** The robust norm of the vector */ - TwoRobust, - - /** Largest entry in absolute value */ - Infinity - } - - /** - * Returns an iterator - * - * @return iterator - */ - public Iterator iterator(); - - /** - * Returns the {@link org.apache.hadoop.io.MapWritable} - * - * @return the entries of vector - */ - public MapWritable getEntries(); -} Index: src/java/org/apache/hama/matrix/package.html =================================================================== --- src/java/org/apache/hama/matrix/package.html (리비전 1004870) +++ src/java/org/apache/hama/matrix/package.html (작업 사본) @@ -1,23 +0,0 @@ - - - - - -Dense and structured sparse matrices - - Index: src/java/org/apache/hama/util/BytesUtil.java =================================================================== --- src/java/org/apache/hama/util/BytesUtil.java (리비전 1004870) +++ src/java/org/apache/hama/util/BytesUtil.java (작업 사본) @@ -1,87 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.util; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.log4j.Logger; - -/** - * Provides a bytes utility - */ -public class BytesUtil { - static final Logger LOG = Logger.getLogger(BytesUtil.class); - public static final int SIZEOF_DOUBLE = Double.SIZE/Byte.SIZE; - public static final int PAD_SIZE = 15; - - /** - * Bytes to integer conversion - * - * @param bytes - * @return the converted value - */ - public static int bytesToInt(byte[] bytes) { - return Integer.parseInt(Bytes.toString(bytes)); - } - - /** - * Gets the row index - * - * @param bytes - * @return the converted value - */ - public static int getRowIndex(byte[] bytes) { - String rKey = Bytes.toString(bytes); //new String(bytes); - // return zero If all zero - if(rKey.equals("000000000000000")) { - return 0; - } - - if(rKey.substring(0, 8).equals("00000000")){ - int i = 8; - while (rKey.charAt(i) == '0') { - i++; - } - return Integer.parseInt(rKey.substring(i, rKey.length())); - } else { - int i = 0; - while (rKey.charAt(i) == '0') { - i++; - } - return Integer.parseInt(rKey.substring(i, rKey.length())); - } - } - - /** - * Gets the row index - * - * @param integer - * @return the converted value - */ - public static byte[] getRowIndex(int integer) { - String index = String.valueOf(integer); - int zeros = PAD_SIZE - index.length(); - StringBuffer buf = new StringBuffer(); - for (int i = 0; i < zeros; ++i) { - buf.append("0"); - } - return Bytes.toBytes(buf.toString() + index); - } - -} Index: src/java/org/apache/hama/util/RandomVariable.java =================================================================== --- src/java/org/apache/hama/util/RandomVariable.java (리비전 1004870) +++ src/java/org/apache/hama/util/RandomVariable.java (작업 사본) @@ -19,8 +19,6 @@ */ package org.apache.hama.util; -import org.apache.hama.Constants; - /** * The RandomVaraibale Class provides static methods for generating random * numbers. @@ -49,34 +47,8 @@ int i = i0 + (int) Math.floor((i1 - i0 + 1) * x); return i; } - - /** - * Generate a random matrix Path(tablename) with a default length. - * @return random matrix name - */ - public static String randMatrixPath() { - return randString(Constants.RANDOM, 5); - } /** - * Generate a random matrix Path(tablename) with a given length. - * @param length the matrix path's length - * @return random matrix name - */ - public static String randMatrixPath(int length) { - return randString(Constants.RANDOM, length); - } - - /** - * Generate a random aliase name. - * - * @return random aliase name - */ - public static String randAliaseName() { - return randString(Constants.RANDOMALIASE, 5); - } - - /** * Generate a random string using the specified prefix and a fixed length. * @param prefix * the specified string prefix. Index: src/test/org/apache/hama/Benchmarks.java =================================================================== --- src/test/org/apache/hama/Benchmarks.java (리비전 1004870) +++ src/test/org/apache/hama/Benchmarks.java (작업 사본) @@ -1,25 +0,0 @@ -package org.apache.hama; - -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.examples.*; - -public class Benchmarks { - - public static void main(String[] args) throws Exception { - if (args.length < 2) { - System.out.println("Usage: "); - System.exit(-1); - } - - HamaConfiguration conf = new HamaConfiguration(); - System.out.println("Creating random matrix"); - DenseMatrix rand = RandomMatrix.random_mapred(conf, Integer - .parseInt(args[0]), Integer.parseInt(args[0])); - - double start = System.currentTimeMillis(); - JacobiEigen.jacobiEigenValue(rand, Integer.parseInt(args[1])); - double end = System.currentTimeMillis(); - System.out.println("Runtime: " + ((end - start)) * 1000 + " sec"); - } - -} Index: src/test/org/apache/hama/TestHamaAdmin.java =================================================================== --- src/test/org/apache/hama/TestHamaAdmin.java (리비전 1004870) +++ src/test/org/apache/hama/TestHamaAdmin.java (작업 사본) @@ -1,123 +0,0 @@ -package org.apache.hama; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.Matrix; -import org.apache.log4j.Logger; - -public class TestHamaAdmin extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestHamaAdmin.class); - private int SIZE = 10; - private Matrix m1; - private Matrix m2; - private final String aliase1 = "matrix_aliase_A"; - private final String aliase2 = "matrix_aliase_B"; - private HamaConfiguration conf; - private HBaseAdmin admin; - private HamaAdmin hamaAdmin; - - /** - * @throws UnsupportedEncodingException - */ - public TestHamaAdmin() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - conf = getConf(); - admin = new HBaseAdmin(conf); - hamaAdmin = new HamaAdminImpl(conf, admin); - - m1 = DenseMatrix.random(conf, SIZE, SIZE); - m2 = DenseMatrix.random(conf, SIZE, SIZE); - } - - public void testLoadSave() throws IOException { - String path1 = m1.getPath(); - // save m1 to aliase1 - m1.save(aliase1); - // load matrix m1 using aliase1 - DenseMatrix loadTest = new DenseMatrix(conf, aliase1, false); - - for (int i = 0; i < SIZE; i++) { - for (int j = 0; j < SIZE; j++) { - assertEquals(m1.get(i, j), loadTest.get(i, j)); - } - } - - assertEquals(path1, loadTest.getPath()); - // close loadTest, it just disconnect to the table but didn't delete it. - loadTest.close(); - - // try to close m1 & load matrix m1 using aliase1 again. - m1.close(); - DenseMatrix loadTest2 = new DenseMatrix(conf, aliase1, false); - assertEquals(path1, loadTest2.getPath()); - // remove aliase1 - // because loadTest2 connect the aliase1, so we just remove aliase entry - // but didn't delete the table. - hamaAdmin.delete(aliase1); - assertEquals(true, admin.tableExists(path1)); - // close loadTest2, because it is the last one who reference table 'path1' - // it will do the gc! - loadTest2.close(); - assertEquals(false, admin.tableExists(path1)); - - // if we try to load non-existed matrix using aliase name, it should fail. - DenseMatrix loadTest3 = null; - try { - loadTest3 = new DenseMatrix(conf, aliase1, false); - fail("Try to load a non-existed matrix should fail!"); - } catch (IOException e) { - - } finally { - if (loadTest3 != null) - loadTest3.close(); - } - - forceCreate(); - } - - public void forceCreate() throws IOException { - String path2 = m2.getPath(); - // save m2 to aliase2 - m2.save(aliase2); - // load matrix m2 using aliase2 - DenseMatrix loadTest = new DenseMatrix(conf, aliase2, false); - - for (int i = 0; i < loadTest.getRows(); i++) { - for (int j = 0; j < loadTest.getColumns(); j++) { - assertEquals(m2.get(i, j), loadTest.get(i, j)); - } - } - - assertEquals(path2, loadTest.getPath()); - - Matrix test = hamaAdmin.getMatrix(aliase2); - assertEquals(test.getType(), "DenseMatrix"); - - // force to create matrix loadTest2 using aliasename 'aliase2' - DenseMatrix loadTest2 = new DenseMatrix(conf, aliase2, true); - String loadPath2 = loadTest2.getPath(); - assertFalse(path2.equals(loadPath2)); - assertEquals(loadPath2, hamaAdmin.getPath(aliase2)); - assertFalse(path2.equals(hamaAdmin.getPath(aliase2))); - - // try to close m2 & loadTest, it table will be deleted finally - m2.close(); - assertEquals(true, admin.tableExists(path2)); - loadTest.close(); - assertEquals(false, admin.tableExists(path2)); - - // remove 'aliase2' & close loadTest2 - loadTest2.close(); - assertEquals(true, admin.tableExists(loadPath2)); - hamaAdmin.delete(aliase2); - assertEquals(false, admin.tableExists(loadPath2)); - } -} Index: src/test/org/apache/hama/TestHbaseClient.java =================================================================== --- src/test/org/apache/hama/TestHbaseClient.java (리비전 1004870) +++ src/test/org/apache/hama/TestHbaseClient.java (작업 사본) @@ -1,69 +0,0 @@ -package org.apache.hama; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.hadoop.hbase.HBaseClusterTestCase; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; - -public class TestHbaseClient extends HBaseClusterTestCase { - private static final byte[] FAMILY = Bytes.toBytes("family"); - private static final byte[] ROW = Bytes.toBytes("row"); - private static final byte[] QUALIFIER = Bytes.toBytes("qualifier"); - private static final byte[] VALUE = Bytes.toBytes("value"); - private static final byte[] MISSING_ROW = Bytes.toBytes("missingrow"); - - private HTableDescriptor desc = null; - private HTable table = null; - - /** - * @throws UnsupportedEncodingException - */ - public TestHbaseClient() throws UnsupportedEncodingException { - super(); - } - - @Override - public void setUp() throws Exception { - super.setUp(); - this.desc = new HTableDescriptor("testGet"); - desc.addFamily(new HColumnDescriptor(FAMILY)); - HBaseAdmin admin = new HBaseAdmin(conf); - admin.createTable(desc); - table = new HTable(conf, desc.getName()); - } - - public void testGet_EmptyTable() throws IOException { - Get get = new Get(ROW); - get.addFamily(FAMILY); - Result r = table.get(get); - assertTrue(r.isEmpty()); - } - - public void testGet_NonExistentRow() throws IOException { - Put put = new Put(ROW); - put.add(FAMILY, QUALIFIER, VALUE); - table.put(put); - System.out.println("Row put"); - - Get get = new Get(ROW); - get.addFamily(FAMILY); - Result r = table.get(get); - assertFalse(r.isEmpty()); - assertEquals(Bytes.toString(VALUE), Bytes.toString(r.getValue(FAMILY, QUALIFIER))); - System.out.println("Row retrieved successfully"); - - get = new Get(MISSING_ROW); - get.addFamily(FAMILY); - r = table.get(get); - assertTrue(r.isEmpty()); - System.out.println("Row missing as it should be"); - } -} Index: src/test/org/apache/hama/Utils.java =================================================================== --- src/test/org/apache/hama/Utils.java (리비전 1004870) +++ src/test/org/apache/hama/Utils.java (작업 사본) @@ -1,52 +0,0 @@ -package org.apache.hama; - -import java.io.IOException; - -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; - -public class Utils { - private static HBaseConfiguration conf = new HBaseConfiguration(); - private static HBaseAdmin admin; - - public static void main(String[] args) throws Exception { - if (args.length < 1) { - System.out.println("Usage: org.apache.hama.Utils [list|delete-all]"); - System.out.println(" - list : listing the tables, related with Hama"); - System.out.println(" - delete-all : deleting the tables, related with Hama"); - System.exit(-1); - } - admin = new HBaseAdmin(conf); - - HTableDescriptor[] tables = admin.listTables(); - for (int i = 0; i < tables.length; i++) { - if (isHamaTables(tables[i].getNameAsString())) { - if (args[0].equals("list")) { - System.out.println(tables[i].getNameAsString()); - } else if (args[0].equals("delete-all")) { - deleteTable(tables[i]); - } - } - } - } - - private static void deleteTable(HTableDescriptor tableDescriptor) - throws IOException { - byte[] tableName = tableDescriptor.getName(); - while (admin.isTableEnabled(tableName)) { - admin.disableTable(tableName); - } - - admin.deleteTable(tableName); - } - - private static boolean isHamaTables(String name) { - if (name.equals(Constants.ADMINTABLE) - || name.startsWith("DenseMatrix_rand") - || name.startsWith("SparseMatrix_rand")) - return true; - - return false; - } -} Index: src/test/org/apache/hama/examples/TestBlockID.java =================================================================== --- src/test/org/apache/hama/examples/TestBlockID.java (리비전 1004870) +++ src/test/org/apache/hama/examples/TestBlockID.java (작업 사본) @@ -1,69 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; - -import junit.framework.TestCase; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hama.examples.mapreduce.BlockID; - -public class TestBlockID extends TestCase { - final static Log LOG = LogFactory.getLog(TestBlockID.class.getName()); - - /** - * BlockID object compare - */ - public void testCompare() { - BlockID a = new BlockID(1, 3); - BlockID b = new BlockID(1, 1); - assertEquals(a.compareTo(b), 1); - - BlockID c = new BlockID(3, 1); - BlockID d = new BlockID(1, 1); - assertEquals(a.compareTo(c), -1); - - assertEquals(b.compareTo(d), 0); - } - - /** - * BlockID object IO - * @throws IOException - */ - public void testIO() throws IOException { - DataOutputBuffer outBuf = new DataOutputBuffer(); - DataInputBuffer inBuf = new DataInputBuffer(); - - BlockID a = new BlockID(1, 3); - outBuf.reset(); - a.write(outBuf); - - inBuf.reset(outBuf.getData(), outBuf.getLength()); - BlockID b = new BlockID(); - b.readFields(inBuf); - - assertEquals(0, a.compareTo(b)); - } - -} Index: src/test/org/apache/hama/examples/TestBlockMatrixMapReduce.java =================================================================== --- src/test/org/apache/hama/examples/TestBlockMatrixMapReduce.java (리비전 1004870) +++ src/test/org/apache/hama/examples/TestBlockMatrixMapReduce.java (작업 사본) @@ -1,61 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; - -import org.apache.hama.HamaCluster; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.log4j.Logger; -import org.apache.hama.examples.MatrixMultiplication; - -public class TestBlockMatrixMapReduce extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestBlockMatrixMapReduce.class); - static final int SIZE = 32; - - /** constructor */ - public TestBlockMatrixMapReduce() { - super(); - } - - public void testBlockMatrixMapReduce() throws IOException, - ClassNotFoundException { - DenseMatrix m1 = DenseMatrix.random(conf, SIZE, SIZE); - DenseMatrix m2 = DenseMatrix.random(conf, SIZE, SIZE); - - DenseMatrix c = MatrixMultiplication.mult(m1, m2, 16); - - double[][] mem = new double[SIZE][SIZE]; - for (int i = 0; i < SIZE; i++) { - for (int j = 0; j < SIZE; j++) { - for (int k = 0; k < SIZE; k++) { - mem[i][k] += m1.get(i, j) * m2.get(j, k); - } - } - } - - for (int i = 0; i < SIZE; i++) { - for (int j = 0; j < SIZE; j++) { - double gap = (mem[i][j] - c.get(i, j)); - assertTrue(gap < 0.000001 || gap < -0.000001); - } - } - } -} Index: src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java =================================================================== --- src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java (리비전 1004870) +++ src/test/org/apache/hama/examples/TestCosineSimilarityMatrix.java (작업 사본) @@ -1,114 +0,0 @@ -package org.apache.hama.examples; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.IdentityTableReducer; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hama.Constants; -import org.apache.hama.HamaCluster; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.DenseVector; -import org.apache.hama.matrix.Matrix; -import org.apache.hama.matrix.Vector; -import org.apache.hama.util.BytesUtil; -import org.apache.log4j.Logger; - -public class TestCosineSimilarityMatrix extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestCosineSimilarityMatrix.class); - private int SIZE = 10; - private Matrix m1; - private Matrix symmetricMatrix; - private HamaConfiguration conf; - - /** - * @throws UnsupportedEncodingException - */ - public TestCosineSimilarityMatrix() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - conf = getConf(); - - m1 = DenseMatrix.random(conf, SIZE, SIZE); - symmetricMatrix = new DenseMatrix(conf, SIZE, SIZE); - } - - public void testCosineSimilarity() throws IOException { - Job job = new Job(conf, "set MR job test"); - job.getConfiguration().set("input.matrix", m1.getPath()); - - Scan scan = new Scan(); - scan.addFamily(Constants.COLUMNFAMILY); - - TableMapReduceUtil.initTableMapperJob(m1.getPath(), scan, - ComputeSimilarity.class, ImmutableBytesWritable.class, Put.class, job); - TableMapReduceUtil.initTableReducerJob(symmetricMatrix.getPath(), - IdentityTableReducer.class, job); - job.setNumReduceTasks(0); - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - Vector v1 = m1.getRow(0); - Vector v2 = m1.getRow(2); - assertEquals(v1.dot(v2), symmetricMatrix.get(0, 2)); - } - - public static class ComputeSimilarity extends - TableMapper implements Configurable { - private Configuration conf = null; - private DenseMatrix matrix; - - public void map(ImmutableBytesWritable key, Result value, Context context) - throws IOException, InterruptedException { - DenseVector v = new DenseVector(value); - - Put put = new Put(key.get()); - for (int i = 0; i < matrix.getRows(); i++) { - double dotProduct = matrix.getRow(i).dot(v); - if (BytesUtil.getRowIndex(key.get()) == i) { - dotProduct = 0; - } - put.add(Constants.COLUMNFAMILY, Bytes.toBytes(String - .valueOf(i)), Bytes.toBytes(dotProduct)); - } - - context.write(key, put); - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - try { - matrix = new DenseMatrix(new HamaConfiguration(conf), conf - .get("input.matrix")); - } catch (IOException e) { - e.printStackTrace(); - } - - } - } -} Index: src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java =================================================================== --- src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java (리비전 1004870) +++ src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java (작업 사본) @@ -1,226 +0,0 @@ -package org.apache.hama.examples; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hama.Constants; -import org.apache.hama.HamaCluster; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.examples.mapreduce.BlockID; -import org.apache.hama.examples.mapreduce.BlockMultMap; -import org.apache.hama.examples.mapreduce.BlockMultReduce; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.DenseVector; -import org.apache.hama.matrix.Matrix; -import org.apache.hama.util.RandomVariable; - -public class TestFileMatrixBlockMult extends HamaCluster { - final static Log LOG = LogFactory.getLog(TestFileMatrixBlockMult.class - .getName()); - private HamaConfiguration conf; - private Path[] path = new Path[2]; - private String collectionTable; - - /** - * @throws UnsupportedEncodingException - */ - public TestFileMatrixBlockMult() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - conf = getConf(); - HBaseAdmin admin = new HBaseAdmin(conf); - collectionTable = "collect_" + RandomVariable.randMatrixPath(); - HTableDescriptor desc = new HTableDescriptor(collectionTable); - desc.addFamily(new HColumnDescriptor(Bytes.toBytes(Constants.BLOCK))); - admin.createTable(desc); - } - - public void createFiles() throws IOException { - Configuration conf = new Configuration(); - LocalFileSystem fs = new LocalFileSystem(); - fs.setConf(conf); - fs.getRawFileSystem().setConf(conf); - - for (int i = 0; i < 2; i++) { - path[i] = new Path(System.getProperty("test.build.data", ".") + "/tmp" - + i + ".seq"); - SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path[i], - IntWritable.class, MapWritable.class, CompressionType.BLOCK); - - MapWritable value = new MapWritable(); - value.put(new IntWritable(0), new DoubleWritable(0.5)); - value.put(new IntWritable(1), new DoubleWritable(0.1)); - value.put(new IntWritable(2), new DoubleWritable(0.5)); - value.put(new IntWritable(3), new DoubleWritable(0.1)); - - writer.append(new IntWritable(0), value); - writer.append(new IntWritable(1), value); - writer.append(new IntWritable(2), value); - writer.append(new IntWritable(3), value); - - writer.close(); - } - - FileSystem xfs = path[0].getFileSystem(conf); - SequenceFile.Reader reader1 = new SequenceFile.Reader(xfs, path[0], conf); - // read first value from reader1 - IntWritable key = new IntWritable(); - MapWritable val = new MapWritable(); - reader1.next(key, val); - - assertEquals(key.get(), 0); - } - - public void testFileMatrixMult() throws IOException { - createFiles(); - collectBlocksFromFile(path[0], true, collectionTable, conf); - collectBlocksFromFile(path[1], false, collectionTable, conf); - - Matrix result = new DenseMatrix(conf, 4, 4); - Job job = new Job(conf, "multiplication MR job : " + result.getPath()); - - Scan scan = new Scan(); - scan.addFamily(Bytes.toBytes(Constants.BLOCK)); - - TableMapReduceUtil.initTableMapperJob(collectionTable, scan, - BlockMultMap.class, BlockID.class, BytesWritable.class, job); - TableMapReduceUtil.initTableReducerJob(result.getPath(), - BlockMultReduce.class, job); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - verifyMultResult(result); - } - - private void verifyMultResult(Matrix result) throws IOException { - double[][] a = new double[][] { { 0.5, 0.1, 0.5, 0.1 }, - { 0.5, 0.1, 0.5, 0.1 }, { 0.5, 0.1, 0.5, 0.1 }, { 0.5, 0.1, 0.5, 0.1 } }; - double[][] b = new double[][] { { 0.5, 0.1, 0.5, 0.1 }, - { 0.5, 0.1, 0.5, 0.1 }, { 0.5, 0.1, 0.5, 0.1 }, { 0.5, 0.1, 0.5, 0.1 } }; - double[][] c = new double[4][4]; - - for (int i = 0; i < 4; i++) { - for (int j = 0; j < 4; j++) { - for (int k = 0; k < 4; k++) { - c[i][k] += a[i][j] * b[j][k]; - } - } - } - - for (int i = 0; i < result.getRows(); i++) { - for (int j = 0; j < result.getColumns(); j++) { - double gap = (c[i][j] - result.get(i, j)); - assertTrue(gap < 0.000001 || gap < -0.000001); - } - } - } - - private static void collectBlocksFromFile(Path path, boolean b, - String collectionTable, HamaConfiguration conf) throws IOException { - Job job = new Job(conf, "Blocking MR job" + path); - job.setMapperClass(MyMapper.class); - job.setMapOutputKeyClass(BlockID.class); - job.setMapOutputValueClass(MapWritable.class); - job.setInputFormatClass(SequenceFileInputFormat.class); - SequenceFileInputFormat.addInputPath(job, path); - - job.getConfiguration().set(MyMapper.BLOCK_SIZE, String.valueOf(2)); - job.getConfiguration().set(MyMapper.ROWS, String.valueOf(4)); - job.getConfiguration().set(MyMapper.COLUMNS, String.valueOf(4)); - job.getConfiguration().setBoolean(MyMapper.MATRIX_POS, b); - - TableMapReduceUtil.initTableReducerJob(collectionTable, - org.apache.hama.examples.mapreduce.CollectBlocksReducer.class, job); - - try { - job.waitForCompletion(true); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } - - public static class MyMapper extends - Mapper implements - Configurable { - private Configuration conf = null; - /** Parameter of the path of the matrix to be blocked * */ - public static final String BLOCK_SIZE = "hama.blocking.size"; - public static final String ROWS = "hama.blocking.rows"; - public static final String COLUMNS = "hama.blocking.columns"; - public static final String MATRIX_POS = "a.or.b"; - - private int mBlockNum; - private int mBlockRowSize; - private int mBlockColSize; - private int mRows; - private int mColumns; - - public void map(IntWritable key, MapWritable value, Context context) - throws IOException, InterruptedException { - int startColumn, endColumn, blkRow = key.get() / mBlockRowSize, i = 0; - DenseVector dv = new DenseVector(key.get(), value); - - do { - startColumn = i * mBlockColSize; - endColumn = startColumn + mBlockColSize - 1; - if (endColumn >= mColumns) // the last sub vector - endColumn = mColumns - 1; - context.write(new BlockID(blkRow, i), dv.subVector(startColumn, - endColumn).getEntries()); - - i++; - } while (endColumn < (mColumns - 1)); - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - - mBlockNum = conf.getInt(BLOCK_SIZE, 2); - mRows = conf.getInt(ROWS, 4); - mColumns = conf.getInt(COLUMNS, 4); - - mBlockRowSize = mRows / mBlockNum; - mBlockColSize = mColumns / mBlockNum; - } - } -} Index: src/test/org/apache/hama/examples/TestJacobiEigenValue.java =================================================================== --- src/test/org/apache/hama/examples/TestJacobiEigenValue.java (리비전 1004870) +++ src/test/org/apache/hama/examples/TestJacobiEigenValue.java (작업 사본) @@ -1,182 +0,0 @@ -package org.apache.hama.examples; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.hama.HamaCluster; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.examples.JacobiEigen; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.Matrix; -import org.apache.hama.matrix.TestDenseMatrix; -import org.apache.log4j.Logger; - -public class TestJacobiEigenValue extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestDenseMatrix.class); - private int SIZE = 6; - private Matrix m5; - private HamaConfiguration conf; - /** - * The correct EigenValues are {11.099019513612875, 0.9009804864500339, - * 0.585786437634226 , 3.4142135624028565, 1.0, 1.0 } - */ - private double[][] A = new double[][] { { 4, 3, 2, 1, 0, 0 }, - { 3, 4, 3, 2, 0, 0 }, { 2, 3, 4, 3, 0, 0 }, { 1, 2, 3, 4, 0, 0 }, - { 0, 0, 0, 0, 1, 0 }, { 0, 0, 0, 0, 0, 1 } }; - - /** - * @throws UnsupportedEncodingException - */ - public TestJacobiEigenValue() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - conf = getConf(); - m5 = new DenseMatrix(conf, SIZE, SIZE); - - for (int i = 0; i < SIZE; i++) { - for (int j = 0; j < SIZE; j++) { - m5.set(i, j, A[i][j]); - } - } - } - - public void testJacobiEigenValue() throws IOException { - // copy Matrix m5 to the array - double[][] S = A; - - for (int i = 0; i < SIZE; i++) { - for (int j = 0; j < SIZE; j++) { - S[i][j] = m5.get(i, j); - } - } - - // do jacobi egien value over S array - int i, j, k, l, m, state; - double s, c, t, p, y; - double e1, e2; - // index array - int[] ind = new int[SIZE]; - boolean[] changed = new boolean[SIZE]; - - // output - double[] e = new double[SIZE]; - double[][] E = new double[SIZE][SIZE]; - - // init e & E; ind & changed - for (i = 0; i < SIZE; i++) { - for (j = 0; j < SIZE; j++) { - E[i][j] = 0; - } - E[i][i] = 1; - } - - state = SIZE; - - for (i = 0; i < SIZE; i++) { - ind[i] = maxind(S, i, SIZE); - e[i] = S[i][i]; - changed[i] = true; - } - - int loops = 100; - int icount = 0; - // next rotation - while (state != 0 && icount < loops) { - icount = icount + 1; - // find index(k, l) for pivot p - m = 0; - - for (k = 1; k <= SIZE - 2; k++) { - if (Math.abs(S[m][ind[m]]) < Math.abs(S[k][ind[k]])) { - m = k; - } - } - - k = m; - l = ind[m]; - p = S[k][l]; - - // calculate c = cos, s = sin - y = (e[l] - e[k]) / 2; - t = Math.abs(y) + Math.sqrt(p * p + y * y); - s = Math.sqrt(p * p + t * t); - c = t / s; - s = p / s; - t = (p * p) / t; - if (y < 0) { - s = -s; - t = -t; - } - - S[k][l] = 0.0; - state = update(e, changed, k, -t, state); - state = update(e, changed, l, t, state); - - for (i = 0; i <= k - 1; i++) - rotate(S, i, k, i, l, c, s); - for (i = l + 1; i < SIZE; i++) - rotate(S, k, i, l, i, c, s); - for (i = k + 1; i <= l - 1; i++) - rotate(S, k, i, i, l, c, s); - - // rotate eigenvectors - for (i = 0; i < SIZE; i++) { - e1 = E[k][i]; - e2 = E[l][i]; - - E[k][i] = c * e1 - s * e2; - E[l][i] = s * e1 + c * e2; - } - - ind[k] = maxind(S, k, SIZE); - ind[l] = maxind(S, l, SIZE); - - loops--; - } - - for (int x = 0; x < SIZE; x++) { - System.out.println(e[x]); - } - - // do m/r jacobi eigen value computation - DenseMatrix dm = (DenseMatrix) m5; - JacobiEigen.jacobiEigenValue(dm, 100); - - // verify the results - assertTrue(JacobiEigen.verifyEigenValue(e, E)); - } - - // index of largest off-diagonal element in row k - int maxind(double[][] S, int row, int size) { - int m = row + 1; - for (int i = row + 2; i < size; i++) { - if (Math.abs(S[row][i]) > Math.abs(S[row][m])) - m = i; - } - return m; - } - - int update(double[] e, boolean[] changed, int row, double value, int state) { - double y = e[row]; - e[row] += value; - - if (changed[row] && y == e[row]) { - changed[row] = false; - return state - 1; - } else if (!changed[row] && y != e[row]) { - changed[row] = true; - return state + 1; - } else - return state; - } - - void rotate(double[][] S, int k, int l, int i, int j, double c, double s) { - double s1 = S[k][l], s2 = S[i][j]; - S[k][l] = c * s1 - s * s2; - S[i][j] = s * s1 + c * s2; - } -} Index: src/test/org/apache/hama/examples/TestMatrixMult.java =================================================================== --- src/test/org/apache/hama/examples/TestMatrixMult.java (리비전 1004870) +++ src/test/org/apache/hama/examples/TestMatrixMult.java (작업 사본) @@ -1,75 +0,0 @@ -package org.apache.hama.examples; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.hama.HamaCluster; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.matrix.Matrix; - -public class TestMatrixMult extends HamaCluster { - private int SIZE = 8; - private Matrix m1; - private Matrix m2; - private HamaConfiguration conf; - - /** - * @throws UnsupportedEncodingException - */ - public TestMatrixMult() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - conf = getConf(); - - m1 = RandomMatrix.random_mapred(conf, SIZE, SIZE); - m2 = RandomMatrix.random_mapred(conf, SIZE, SIZE); - } - - /** - * Test matrices multiplication - * - * @throws IOException - */ - public void testMult() throws IOException { - Matrix result = MatrixMultiplication.mult(m1, m2); - - assertEquals(result.getRows(), SIZE); - assertEquals(result.getColumns(), SIZE); - - Matrix result2 = MatrixMultiplication.mult(m1, m2, 4); - - verifyMultResult(m1, m2, result); - verifyMultResult(m1, m2, result2); - } - - /** - * Verifying multiplication result - * - * @param m1 - * @param m2 - * @param result - * @throws IOException - */ - private void verifyMultResult(Matrix m1, Matrix m2, Matrix result) - throws IOException { - double[][] c = new double[SIZE][SIZE]; - - for (int i = 0; i < SIZE; i++) { - for (int j = 0; j < SIZE; j++) { - for (int k = 0; k < SIZE; k++) { - c[i][k] += m1.get(i, j) * m2.get(j, k); - } - } - } - - for (int i = 0; i < SIZE; i++) { - for (int j = 0; j < SIZE; j++) { - assertTrue((Math.abs(c[i][j] - result.get(i, j)) < .0000001)); - } - } - } -} Index: src/test/org/apache/hama/examples/TestRandomMatrixMapReduce.java =================================================================== --- src/test/org/apache/hama/examples/TestRandomMatrixMapReduce.java (리비전 1004870) +++ src/test/org/apache/hama/examples/TestRandomMatrixMapReduce.java (작업 사본) @@ -1,59 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.examples; - -import java.io.IOException; - -import org.apache.hama.HamaCluster; -import org.apache.hama.examples.RandomMatrix; -import org.apache.hama.matrix.DenseMatrix; -import org.apache.hama.matrix.SparseMatrix; -import org.apache.log4j.Logger; - -public class TestRandomMatrixMapReduce extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestRandomMatrixMapReduce.class); - - public void testRandomMatrixMapReduce() throws IOException { - DenseMatrix rand = RandomMatrix.random_mapred(conf, 20, 20); - assertEquals(20, rand.getRows()); - assertEquals(20, rand.getColumns()); - - for(int i = 0; i < 20; i++) { - for(int j = 0; j < 20; j++) { - assertTrue(rand.get(i, j) > 0); - } - } - - rand.close(); - - SparseMatrix rand2 = RandomMatrix.random_mapred(conf, 20, 20, 30); - assertEquals(20, rand2.getRows()); - assertEquals(20, rand2.getColumns()); - boolean zeroAppear = false; - for(int i = 0; i < 20; i++) { - for(int j = 0; j < 20; j++) { - if(rand2.get(i, j) == 0.0) - zeroAppear = true; - } - } - assertTrue(zeroAppear); - rand2.close(); - } -} Index: src/test/org/apache/hama/matrix/MatrixTestCommon.java =================================================================== --- src/test/org/apache/hama/matrix/MatrixTestCommon.java (리비전 1004870) +++ src/test/org/apache/hama/matrix/MatrixTestCommon.java (작업 사본) @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; - -class MatrixTestCommon { - - static double verifyNorm1(Matrix m1) throws IOException { - double[] colSum = new double[m1.getColumns()]; - for (int j = 0; j < m1.getColumns(); j++) { - for (int i = 0; i < m1.getRows(); i++) { - colSum[j] += Math.abs(m1.get(i, j)); - } - } - - double max = 0; - for (int i = 0; i < colSum.length; i++) { - max = Math.max(colSum[i], max); - } - return max; - } - - static double verifyNormInfinity(Matrix m1) throws IOException { - double[] rowSum = new double[m1.getRows()]; - for (int i = 0; i < m1.getRows(); i++) { - for (int j = 0; j < m1.getColumns(); j++) { - rowSum[i] += Math.abs(m1.get(i, j)); - } - } - - double max = 0; - for (int i = 0; i < rowSum.length; ++i) - max = Math.max(rowSum[i], max); - return max; - } - - static double verifyNormMaxValue(Matrix m1) throws IOException { - double max = 0; - for (int i = 0; i < m1.getRows(); i++) { - for (int j = 0; j < m1.getColumns(); j++) { - max = Math.max(Math.abs(m1.get(i, j)), max); - } - } - - return max; - } - - static double verifyNormFrobenius(Matrix m1) throws IOException { - double sqrtSum = 0; - for (int i = 0; i < m1.getRows(); i++) { - for (int j = 0; j < m1.getColumns(); j++) { - double cellValue = m1.get(i, j); - sqrtSum += (cellValue * cellValue); - } - } - return Math.sqrt(sqrtSum); - } - -} Index: src/test/org/apache/hama/matrix/TestAbstractMatrix.java =================================================================== --- src/test/org/apache/hama/matrix/TestAbstractMatrix.java (리비전 1004870) +++ src/test/org/apache/hama/matrix/TestAbstractMatrix.java (작업 사본) @@ -1,61 +0,0 @@ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.hama.HamaCluster; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.examples.MatrixNorm; -import org.apache.hama.matrix.Matrix.Norm; -import org.apache.log4j.Logger; - -public class TestAbstractMatrix extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestAbstractMatrix.class); - private int SIZE = 10; - private Matrix m1; - private Matrix m2; - private HamaConfiguration conf; - private double gap = 0.000001; - - /** - * @throws UnsupportedEncodingException - */ - public TestAbstractMatrix() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - conf = getConf(); - m1 = DenseMatrix.random(conf, SIZE, SIZE); - m2 = SparseMatrix.random(conf, SIZE, SIZE); - } - - public void testTransposeAndNorm() throws IOException { - normTest(m1); - normTest(m2); - } - - public void normTest(Matrix matrix) throws IOException { - double norm1 = MatrixNorm.norm(matrix, Norm.One); - double verify_norm1 = MatrixTestCommon.verifyNorm1(matrix); - gap = norm1 - verify_norm1; - assertTrue(gap < 0.000001 && gap > -0.000001); - - double normInfinity = MatrixNorm.norm(matrix, Norm.Infinity); - double verify_normInf = MatrixTestCommon.verifyNormInfinity(matrix); - gap = normInfinity - verify_normInf; - assertTrue(gap < 0.000001 && gap > -0.000001); - - double normFrobenius = MatrixNorm.norm(matrix, Norm.Frobenius); - double verify_normFrobenius = MatrixTestCommon.verifyNormFrobenius(matrix); - gap = normFrobenius - verify_normFrobenius; - assertTrue(gap < 0.000001 && gap > -0.000001); - - double normMaxValue = MatrixNorm.norm(matrix, Norm.Maxvalue); - double verify_normMV = MatrixTestCommon.verifyNormMaxValue(matrix); - gap = normMaxValue - verify_normMV; - assertTrue(gap < 0.000001 && gap > -0.000001); - } -} Index: src/test/org/apache/hama/matrix/TestDenseMatrix.java =================================================================== --- src/test/org/apache/hama/matrix/TestDenseMatrix.java (리비전 1004870) +++ src/test/org/apache/hama/matrix/TestDenseMatrix.java (작업 사본) @@ -1,145 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.Iterator; - -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.HamaCluster; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.util.RandomVariable; -import org.apache.log4j.Logger; - -/** - * Matrix test - */ -public class TestDenseMatrix extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestDenseMatrix.class); - private int SIZE = 10; - private Matrix m1; - private HamaConfiguration conf; - - /** - * @throws UnsupportedEncodingException - */ - public TestDenseMatrix() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - conf = getConf(); - - m1 = DenseMatrix.random(conf, SIZE, SIZE); - } - - public void testAddMult() throws IOException { - double origin = m1.get(1, 1); - m1.add(1, 1, 0.5); - assertEquals(m1.get(1, 1), origin + 0.5); - - getRowColumnVector(); - setRowColumnVector(); - - setMatrix(m1); - setAlphaMatrix(m1); - } - - public void setMatrix(Matrix m1) throws IOException { - Matrix a = new DenseMatrix(conf, m1.getRows(), m1.getColumns()); - a.set(m1); - - for (int i = 0; i < 5; i++) { - // between 0 ~ SIZE -1 - int x = RandomVariable.randInt(0, SIZE - 1); - int y = RandomVariable.randInt(0, SIZE - 1); - assertEquals(a.get(x, y), m1.get(x, y)); - } - } - - public void setAlphaMatrix(Matrix m1) throws IOException { - Matrix a = new DenseMatrix(conf, m1.getRows(), m1.getColumns()); - a.set(0.5, m1); - - for (int i = 0; i < 5; i++) { - int x = RandomVariable.randInt(0, SIZE - 1); - int y = RandomVariable.randInt(0, SIZE - 1); - assertEquals(a.get(x, y), (m1.get(x, y) * 0.5)); - } - } - - public void getRowColumnVector() throws IOException { - boolean ex = false; - try { - m1.get(SIZE + 1, SIZE + 1); - } catch (ArrayIndexOutOfBoundsException e) { - ex = true; - } - assertTrue(ex); - assertTrue(m1.get(0, 0) > 0); - - Vector v = m1.getColumn(0); - Iterator it = v.iterator(); - int x = 0; - while (it.hasNext()) { - assertEquals(m1.get(x, 0), ((DoubleWritable) it.next()).get()); - x++; - } - - m1.setRowLabel(0, "row1"); - assertEquals(m1.getRowLabel(0), "row1"); - assertEquals(m1.getRowLabel(1), null); - - m1.setColumnLabel(0, "column1"); - assertEquals(m1.getColumnLabel(0), "column1"); - assertEquals(m1.getColumnLabel(1), null); - } - - public void setRowColumnVector() throws IOException { - Vector v = new DenseVector(); - double[] entries = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }; - - for (int i = 0; i < SIZE; i++) { - v.set(i, entries[i]); - } - - m1.setRow(SIZE, v); - Iterator it = m1.getRow(SIZE).iterator(); - - int i = 0; - while (it.hasNext()) { - assertEquals(entries[i], ((DoubleWritable) it.next()).get()); - i++; - } - - m1.setColumn(SIZE, v); - Iterator it2 = m1.getColumn(SIZE).iterator(); - - int x = 0; - while (it2.hasNext()) { - assertEquals(entries[x], ((DoubleWritable) it2.next()).get()); - x++; - } - } -} Index: src/test/org/apache/hama/matrix/TestDenseVector.java =================================================================== --- src/test/org/apache/hama/matrix/TestDenseVector.java (리비전 1004870) +++ src/test/org/apache/hama/matrix/TestDenseVector.java (작업 사본) @@ -1,204 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.Iterator; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.DoubleWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hama.HamaCluster; - -public class TestDenseVector extends HamaCluster { - final static Log LOG = LogFactory.getLog(TestDenseVector.class.getName()); - - private final double cosine = 0.6978227007909176; - private final double norm1 = 12.0; - private final double norm2 = 6.782329983125268; - private final double normInf = 5.0; - private final double norm2Robust = 6.782329983125269; - private double[][] values = { { 2, 5, 1, 4 }, { 4, 1, 3, 3 } }; - private DenseMatrix m1; - private DenseVector v1; - private DenseVector v2; - private DenseVector smallSize = new DenseVector(); - - /** - * @throws UnsupportedEncodingException - */ - public TestDenseVector() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - m1 = new DenseMatrix(getConf(), 2, 4); - - for (int i = 0; i < 2; i++) - for (int j = 0; j < 4; j++) - m1.set(i, j, values[i][j]); - - v1 = m1.getRow(0); - v2 = m1.getRow(1); - smallSize.set(0, 0.5); - } - - /** - * @throws IOException - */ - public void testDenseVector() throws IOException { - double cos = v1.dot(v2); - assertEquals(cos, cosine); - - boolean except = false; - try { - v1.dot(smallSize); - } catch (IndexOutOfBoundsException e) { - except = true; - } - - assertTrue(except); - subVector(); - - assertEquals(norm1, v1.norm(Vector.Norm.One)); - assertEquals(norm2, v1.norm(Vector.Norm.Two)); - assertEquals(normInf, v1.norm(Vector.Norm.Infinity)); - assertEquals(norm2Robust, v1.norm(Vector.Norm.TwoRobust)); - - getSetTest(); - add(); - scalingTest(); - setTest(); - clear(); - } - - /** - * Test scaling - */ - public void scalingTest() { - v2.scale(0.5); - - for (int i = 0; i < v2.size(); i++) { - assertEquals(values[1][i] * 0.5, v2.get(i)); - } - } - - /** - * Test get/set methods - * @throws IOException - */ - public void getSetTest() throws IOException { - assertEquals(v1.get(0), values[0][0]); - boolean ex = false; - try { - v1.get(5); - } catch (NullPointerException e) { - ex = true; - } - assertTrue(ex); - assertEquals(m1.getColumn(0).size(), 2); - } - - /** - * Test add() - */ - public void add() { - v1.add(v2); - int i = 0; - Iterator it = v1.iterator(); - while (it.hasNext()) { - DoubleWritable c = (DoubleWritable) it.next(); - assertEquals(c.get(), values[0][i] + values[1][i]); - i++; - } - - v1.add(0.5, v2); - int j = 0; - Iterator itt = v1.iterator(); - while (itt.hasNext()) { - DoubleWritable c = (DoubleWritable) itt.next(); - assertEquals(c.get(), (values[0][j] + values[1][j]) + (0.5 * values[1][j])); - j++; - } - - double old = v1.get(0); - v1.add(0, norm1); - assertEquals(v1.get(0), old + norm1); - - boolean except = false; - try { - v1.add(smallSize); - } catch (IndexOutOfBoundsException e) { - except = true; - } - - assertTrue(except); - - except = false; - try { - v1.add(0.6, smallSize); - } catch (IndexOutOfBoundsException e) { - except = true; - } - - assertTrue(except); - } - - public void setTest() { - v1.set(v2); - - for(int i = 0; i < v1.size(); i ++) { - assertEquals(v2.get(i), v1.get(i)); - } - - boolean except = false; - try { - v1.set(0.6, smallSize); - } catch (IndexOutOfBoundsException e) { - except = true; - } - - assertTrue(except); - } - - public void subVector() { - int start = 2; - Vector subVector = v1.subVector(start, v1.size() - 1); - Iterator it = subVector.iterator(); - - int i = start; - while (it.hasNext()) { - assertEquals(v1.get(i), ((DoubleWritable) it.next()).get()); - i++; - } - } - - /** - * Clear test - */ - public void clear() { - ((DenseVector) v1).clear(); - assertEquals(v1.size(), 0); - } -} Index: src/test/org/apache/hama/matrix/TestSparseMatrix.java =================================================================== --- src/test/org/apache/hama/matrix/TestSparseMatrix.java (리비전 1004870) +++ src/test/org/apache/hama/matrix/TestSparseMatrix.java (작업 사본) @@ -1,73 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.hama.HamaCluster; -import org.apache.log4j.Logger; - -public class TestSparseMatrix extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestSparseMatrix.class); - private int SIZE = 10; - private SparseMatrix m1; - - /** - * @throws UnsupportedEncodingException - */ - public TestSparseMatrix() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - m1 = SparseMatrix.random(getConf(), SIZE, SIZE); - } - - public void testMult() throws IOException { - assertTrue(m1.getRows() > 0); - sparsity(); - m1.set(0, 0, -8); - assertEquals(m1.get(0, 0), -8.0); - - SparseVector vector = new SparseVector(); - vector.set(0, 3); - vector.set(1, -8); - m1.setRow(0, vector); - assertEquals(m1.get(0, 0), 3.0); - assertEquals(m1.get(0, 1), -8.0); - SparseVector vector2 = m1.getRow(0); - assertEquals(vector2.get(0), 3.0); - assertEquals(vector2.get(1), -8.0); - } - - public void sparsity() throws IOException { - boolean appeared = false; - for (int i = 0; i < m1.getRows(); i++) { - for (int j = 0; j < m1.getColumns(); j++) { - if (m1.get(i, j) == 0) - appeared = true; - } - } - - assertTrue(appeared); - } -} Index: src/test/org/apache/hama/matrix/TestSparseVector.java =================================================================== --- src/test/org/apache/hama/matrix/TestSparseVector.java (리비전 1004870) +++ src/test/org/apache/hama/matrix/TestSparseVector.java (작업 사본) @@ -1,95 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hama.Constants; -import org.apache.hama.HamaCluster; -import org.apache.hama.util.BytesUtil; - -public class TestSparseVector extends HamaCluster { - final static Log LOG = LogFactory.getLog(TestSparseVector.class.getName()); - private SparseMatrix m1; - private SparseVector v1; - private SparseVector v2; - private double[][] values = { { 2, 0, 0, 4 }, { 0, 0, 3, 3 } }; - - /** - * @throws UnsupportedEncodingException - */ - public TestSparseVector() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - m1 = new SparseMatrix(getConf(), 2, 4); - - for (int i = 0; i < 2; i++) - for (int j = 0; j < 4; j++) - m1.set(i, j, values[i][j]); - - v1 = m1.getRow(0); - v2 = m1.getRow(1); - } - - /** - * Test get/set methods - * - * @throws IOException - */ - public void testGetSet() throws IOException { - assertEquals(v1.get(1), 0.0); - assertEquals(v2.get(1), 0.0); - - HTable table = m1.getHTable(); - Get get = new Get(BytesUtil.getRowIndex(0)); - get.addColumn(Constants.COLUMNFAMILY, Bytes.toBytes(1)); - Result r = table.get(get); - assertTrue(r.getCellValue() == null); - - addTest(); - } - - /** - * Test add() - */ - public void addTest() { - v1.add(v2); - - for (int i = 0; i < values[0].length; i++) { - assertEquals(v1.get(i), values[0][i] + values[1][i]); - } - - v1.add(0.5, v2); - for (int i = 0; i < values[0].length; i++) { - assertEquals(v1.get(i), (values[0][i] + values[1][i]) - + (0.5 * values[1][i])); - } - } -} Index: src/test/org/apache/hama/matrix/TestSubMatirx.java =================================================================== --- src/test/org/apache/hama/matrix/TestSubMatirx.java (리비전 1004870) +++ src/test/org/apache/hama/matrix/TestSubMatirx.java (작업 사본) @@ -1,59 +0,0 @@ -package org.apache.hama.matrix; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; - -import org.apache.hama.HamaCluster; -import org.apache.hama.HamaConfiguration; -import org.apache.log4j.Logger; - -public class TestSubMatirx extends HamaCluster { - static final Logger LOG = Logger.getLogger(TestSubMatirx.class); - private int SIZE = 10; - private Matrix m1; - private Matrix m2; - private HamaConfiguration conf; - - /** - * @throws UnsupportedEncodingException - */ - public TestSubMatirx() throws UnsupportedEncodingException { - super(); - } - - public void setUp() throws Exception { - super.setUp(); - - conf = getConf(); - - m1 = DenseMatrix.random(conf, SIZE, SIZE); - m2 = DenseMatrix.random(conf, SIZE, SIZE); - } - - public void testSubMatrix() throws IOException { - SubMatrix a = m1.subMatrix(2, 4, 2, 5); // A : 3 * 4 - for (int i = 0; i < a.getRows(); i++) { - for (int j = 0; j < a.getColumns(); j++) { - assertEquals(a.get(i, j), m1.get(i + 2, j + 2)); - } - } - - SubMatrix b = m2.subMatrix(0, 3, 0, 2); // B : 4 * 3 - SubMatrix c = a.mult(b); - - double[][] C = new double[3][3]; // A * B - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - for (int k = 0; k < 4; k++) { - C[i][j] += m1.get(i + 2, k + 2) * m2.get(k, j); - } - } - } - - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - assertEquals(C[i][j], c.get(i, j)); - } - } - } -}