Index: commons/src/main/java/org/apache/hama/commons/math/DoubleVector.java =================================================================== --- commons/src/main/java/org/apache/hama/commons/math/DoubleVector.java (revision 1547052) +++ commons/src/main/java/org/apache/hama/commons/math/DoubleVector.java (working copy) @@ -281,9 +281,9 @@ * INCLUSIVE. For example vec = [0, 1, 2, 3, 4, 5], vec.slice(2, 5) = [2, 3, * 4, 5]. * - * @param offset must be > 0 and smaller than the dimension of the vector - * @param length must be > 0 and smaller than the dimension of the vector. - * This must be greater than the offset. + * @param start must be >= 0 and smaller than the dimension of the vector + * @param end must be >= 0 and smaller than the dimension of the vector. + * This must be greater than or equal to the start. * @return a new vector that is only (length) long. */ public DoubleVector slice(int start, int end); @@ -292,9 +292,9 @@ * Slices this vector from start to end, both are INCLUSIVE. For example vec = * [0, 1, 2, 3, 4, 5], vec.slice(2, 5) = [2, 3, 4, 5]. * - * @param offset must be > 0 and smaller than the dimension of the vector - * @param length must be > 0 and smaller than the dimension of the vector. - * This must be greater than the offset. + * @param start must be >= 0 and smaller than the dimension of the vector + * @param end must be >= 0 and smaller than the dimension of the vector. + * This must be greater than or equal to the start. * @return a new vector that is only (length) long. */ public DoubleVector sliceUnsafe(int start, int end); Index: ml/src/main/java/org/apache/hama/ml/ann/AutoEncoder.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/ann/AutoEncoder.java (revision 1547052) +++ ml/src/main/java/org/apache/hama/ml/ann/AutoEncoder.java (working copy) @@ -27,6 +27,7 @@ import org.apache.hama.commons.math.DoubleVector; import org.apache.hama.commons.math.FunctionFactory; import org.apache.hama.ml.ann.AbstractLayeredNeuralNetwork.LearningStyle; +import org.apache.hama.ml.util.FeatureTransformer; import com.google.common.base.Preconditions; @@ -176,8 +177,21 @@ return this.transform(inputInstance, 1); } + /** + * Get the label(s) according to the given features. + * @param inputInstance + * @return + */ public DoubleVector getOutput(DoubleVector inputInstance) { return model.getOutput(inputInstance); } + + /** + * Set the feature transformer. + * @param featureTransformer + */ + public void setFeatureTransformer(FeatureTransformer featureTransformer) { + this.model.setFeatureTransformer(featureTransformer); + } } Index: ml/src/main/java/org/apache/hama/ml/ann/NeuralNetwork.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/ann/NeuralNetwork.java (revision 1547052) +++ ml/src/main/java/org/apache/hama/ml/ann/NeuralNetwork.java (working copy) @@ -20,10 +20,14 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; +import org.apache.commons.lang.SerializationUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -31,6 +35,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; +import org.apache.hama.ml.util.DefaultFeatureTransformer; +import org.apache.hama.ml.util.FeatureTransformer; import com.google.common.base.Preconditions; @@ -53,9 +59,12 @@ // the path to store the model protected String modelPath; + protected FeatureTransformer featureTransformer; + public NeuralNetwork() { this.learningRate = DEFAULT_LEARNING_RATE; this.modelType = this.getClass().getSimpleName(); + this.featureTransformer = new DefaultFeatureTransformer(); } public NeuralNetwork(String modelPath) { @@ -179,6 +188,8 @@ return this.modelPath; } + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Override public void readFields(DataInput input) throws IOException { // read model type this.modelType = WritableUtils.readString(input); @@ -186,11 +197,40 @@ this.learningRate = input.readDouble(); // read model path this.modelPath = WritableUtils.readString(input); + if (this.modelPath.equals("null")) { this.modelPath = null; } + + // read feature transformer + int bytesLen = input.readInt(); + byte[] featureTransformerBytes = new byte[bytesLen]; + for (int i = 0; i < featureTransformerBytes.length; ++i) { + featureTransformerBytes[i] = input.readByte(); + } + + Class featureTransformerCls = (Class) SerializationUtils + .deserialize(featureTransformerBytes); + + Constructor[] constructors = featureTransformerCls + .getDeclaredConstructors(); + Constructor constructor = constructors[0]; + + try { + this.featureTransformer = (FeatureTransformer) constructor + .newInstance(new Object[] {}); + } catch (InstantiationException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } } + @Override public void write(DataOutput output) throws IOException { // write model type WritableUtils.writeString(output, modelType); @@ -202,6 +242,22 @@ } else { WritableUtils.writeString(output, "null"); } + + // serialize the class + Class featureTransformerCls = this.featureTransformer + .getClass(); + byte[] featureTransformerBytes = SerializationUtils + .serialize(featureTransformerCls); + output.writeInt(featureTransformerBytes.length); + output.write(featureTransformerBytes); } + public void setFeatureTransformer(FeatureTransformer featureTransformer) { + this.featureTransformer = featureTransformer; + } + + public FeatureTransformer getFeatureTransformer() { + return this.featureTransformer; + } + } Index: ml/src/main/java/org/apache/hama/ml/ann/NeuralNetworkTrainer.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/ann/NeuralNetworkTrainer.java (revision 1547052) +++ ml/src/main/java/org/apache/hama/ml/ann/NeuralNetworkTrainer.java (working copy) @@ -29,6 +29,8 @@ import org.apache.hama.bsp.sync.SyncException; import org.apache.hama.commons.io.VectorWritable; import org.apache.hama.ml.perception.MLPMessage; +import org.apache.hama.ml.util.DefaultFeatureTransformer; +import org.apache.hama.ml.util.FeatureTransformer; /** * The trainer that is used to train the {@link SmallLayeredNeuralNetwork} with @@ -46,13 +48,15 @@ protected int maxIteration; protected int batchSize; protected String trainingMode; - + + protected FeatureTransformer featureTransformer; + @Override final public void setup( BSPPeer peer) throws IOException, SyncException, InterruptedException { conf = peer.getConfiguration(); - + featureTransformer = new DefaultFeatureTransformer(); this.extraSetup(peer); } Index: ml/src/main/java/org/apache/hama/ml/ann/SmallLayeredNeuralNetwork.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/ann/SmallLayeredNeuralNetwork.java (revision 1547052) +++ ml/src/main/java/org/apache/hama/ml/ann/SmallLayeredNeuralNetwork.java (working copy) @@ -255,18 +255,22 @@ /** * Get the output of the model according to given feature instance. */ + @Override public DoubleVector getOutput(DoubleVector instance) { Preconditions.checkArgument(this.layerSizeList.get(0) == instance .getDimension() + 1, String.format( "The dimension of input instance should be %d.", this.layerSizeList.get(0) - 1)); + // transform the features to another space + DoubleVector transformedInstance = this.featureTransformer + .transform(instance); // add bias feature DoubleVector instanceWithBias = new DenseDoubleVector( - instance.getDimension() + 1); + transformedInstance.getDimension() + 1); instanceWithBias.set(0, 0.99999); // set bias to be a little bit less than // 1.0 for (int i = 1; i < instanceWithBias.getDimension(); ++i) { - instanceWithBias.set(i, instance.get(i - 1)); + instanceWithBias.set(i, transformedInstance.get(i - 1)); } List outputCache = getOutputInternal(instanceWithBias); @@ -280,13 +284,13 @@ * Calculate output internally, the intermediate output of each layer will be * stored. * - * @param instance The instance contains the features. + * @param instanceWithBias The instance contains the features. * @return Cached output of each layer. */ - public List getOutputInternal(DoubleVector instance) { + public List getOutputInternal(DoubleVector instanceWithBias) { List outputCache = new ArrayList(); // fill with instance - DoubleVector intermediateOutput = instance; + DoubleVector intermediateOutput = instanceWithBias; outputCache.add(intermediateOutput); for (int i = 0; i < this.layerSizeList.size() - 1; ++i) { @@ -330,6 +334,11 @@ @Override public DoubleMatrix[] trainByInstance(DoubleVector trainingInstance) { + DoubleVector transformedVector = this.featureTransformer + .transform(trainingInstance.sliceUnsafe(this.layerSizeList.get(0) - 1)); + + + int inputDimension = this.layerSizeList.get(0) - 1; int outputDimension; DoubleVector inputInstance = null; @@ -347,10 +356,11 @@ inputInstance = new DenseDoubleVector(this.layerSizeList.get(0)); inputInstance.set(0, 1); // add bias + // get the features from the transformed vector for (int i = 0; i < inputDimension; ++i) { - inputInstance.set(i + 1, trainingInstance.get(i)); + inputInstance.set(i + 1, transformedVector.get(i)); } - + // get the labels from the original training instance labels = trainingInstance.sliceUnsafe(inputInstance.getDimension() - 1, trainingInstance.getDimension() - 1); } else if (this.learningStyle == LearningStyle.UNSUPERVISED) { @@ -364,10 +374,12 @@ inputInstance = new DenseDoubleVector(this.layerSizeList.get(0)); inputInstance.set(0, 1); // add bias + // get the features from the transformed vector for (int i = 0; i < inputDimension; ++i) { - inputInstance.set(i + 1, trainingInstance.get(i)); + inputInstance.set(i + 1, transformedVector.get(i)); } - labels = trainingInstance.deepCopy(); + // get the labels by copying the transformed vector + labels = transformedVector.deepCopy(); } List internalResults = this.getOutputInternal(inputInstance); Index: ml/src/main/java/org/apache/hama/ml/perception/MultiLayerPerceptron.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/perception/MultiLayerPerceptron.java (revision 1547052) +++ ml/src/main/java/org/apache/hama/ml/perception/MultiLayerPerceptron.java (working copy) @@ -26,6 +26,8 @@ import org.apache.hama.commons.math.DoubleVector; import org.apache.hama.commons.math.FunctionFactory; import org.apache.hama.ml.ann.NeuralNetworkTrainer; +import org.apache.hama.ml.util.DefaultFeatureTransformer; +import org.apache.hama.ml.util.FeatureTransformer; /** * PerceptronBase defines the common behavior of all the concrete perceptrons. @@ -50,6 +52,9 @@ protected DoubleDoubleFunction costFunction; protected DoubleFunction squashingFunction; + // transform the original features to new space + protected FeatureTransformer featureTransformer; + /** * Initialize the MLP. * @@ -91,6 +96,8 @@ .createDoubleDoubleFunction(this.costFunctionName); this.squashingFunction = FunctionFactory .createDoubleFunction(this.squashingFunctionName); + + this.featureTransformer = new DefaultFeatureTransformer(); } /** @@ -118,8 +125,12 @@ * @param featureVector The feature of an instance to feed the perceptron. * @return The results. */ - public abstract DoubleVector output(DoubleVector featureVector); + public DoubleVector output(DoubleVector featureVector) { + return this.outputWrapper(this.featureTransformer.transform(featureVector)); + } + public abstract DoubleVector outputWrapper(DoubleVector featureVector); + /** * Use the class name as the type name. */ @@ -176,4 +187,17 @@ return layerSizeArray; } + /** + * Set the feature transformer. + * + * @param featureTransformer + */ + public void setFeatureTransformer(FeatureTransformer featureTransformer) { + this.featureTransformer = featureTransformer; + } + + public FeatureTransformer getFeatureTransformer() { + return this.featureTransformer; + } + } Index: ml/src/main/java/org/apache/hama/ml/perception/SmallMultiLayerPerceptron.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/perception/SmallMultiLayerPerceptron.java (revision 1547052) +++ ml/src/main/java/org/apache/hama/ml/perception/SmallMultiLayerPerceptron.java (working copy) @@ -20,6 +20,8 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -28,6 +30,7 @@ import java.util.Map; import java.util.Random; +import org.apache.commons.lang.SerializationUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -46,6 +49,7 @@ import org.apache.hama.commons.math.DoubleFunction; import org.apache.hama.commons.math.DoubleVector; import org.apache.hama.commons.math.FunctionFactory; +import org.apache.hama.ml.util.FeatureTransformer; import org.mortbay.log.Log; /** @@ -152,7 +156,7 @@ * {@inheritDoc} * The model meta-data is stored in memory. */ - public DoubleVector output(DoubleVector featureVector) { + public DoubleVector outputWrapper(DoubleVector featureVector) { List outputCache = this.outputInternal(featureVector); // the output of the last layer is the output of the MLP return new DenseDoubleVector(outputCache.get(outputCache.size() - 1)); @@ -175,6 +179,10 @@ // fill with input features intermediateResults[0] = 1.0; // bias + + // transform the original features to another space + featureVector = this.featureTransformer.transform(featureVector); + for (int i = 0; i < featureVector.getDimension(); ++i) { intermediateResults[i + 1] = featureVector.get(i); } @@ -245,9 +253,10 @@ return weightUpdateMatrices; } + // transform the features (exclude the labels) to new space double[] trainingVec = trainingInstance.toArray(); - double[] trainingFeature = Arrays.copyOfRange(trainingVec, 0, - this.layerSizeArray[0]); + double[] trainingFeature = this.featureTransformer.transform( + trainingInstance.sliceUnsafe(0, this.layerSizeArray[0] - 1)).toArray(); double[] trainingLabels = Arrays.copyOfRange(trainingVec, this.layerSizeArray[0], trainingVec.length); @@ -402,6 +411,12 @@ this.numberOfLayers = input.readInt(); this.squashingFunctionName = WritableUtils.readString(input); this.costFunctionName = WritableUtils.readString(input); + + this.squashingFunction = FunctionFactory + .createDoubleFunction(this.squashingFunctionName); + this.costFunction = FunctionFactory + .createDoubleDoubleFunction(this.costFunctionName); + // read the number of neurons for each layer this.layerSizeArray = new int[this.numberOfLayers]; for (int i = 0; i < numberOfLayers; ++i) { @@ -411,10 +426,28 @@ for (int i = 0; i < numberOfLayers - 1; ++i) { this.weightMatrice[i] = (DenseDoubleMatrix) MatrixWritable.read(input); } - this.squashingFunction = FunctionFactory - .createDoubleFunction(this.squashingFunctionName); - this.costFunction = FunctionFactory - .createDoubleDoubleFunction(this.costFunctionName); + + // read feature transformer + int bytesLen = input.readInt(); + byte[] featureTransformerBytes = new byte[bytesLen]; + for (int i = 0; i < featureTransformerBytes.length; ++i) { + featureTransformerBytes[i] = input.readByte(); + } + Class featureTransformerCls = (Class) SerializationUtils + .deserialize(featureTransformerBytes); + Constructor constructor = featureTransformerCls.getConstructors()[0]; + try { + this.featureTransformer = (FeatureTransformer) constructor + .newInstance(new Object[] {}); + } catch (InstantiationException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } } @Override @@ -435,6 +468,14 @@ MatrixWritable matrixWritable = new MatrixWritable(this.weightMatrice[i]); matrixWritable.write(output); } + + // serialize the feature transformer + Class featureTransformerCls = this.featureTransformer + .getClass(); + byte[] featureTransformerBytes = SerializationUtils + .serialize(featureTransformerCls); + output.writeInt(featureTransformerBytes.length); + output.write(featureTransformerBytes); } /** Index: ml/src/main/java/org/apache/hama/ml/regression/LinearRegression.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/regression/LinearRegression.java (revision 1547052) +++ ml/src/main/java/org/apache/hama/ml/regression/LinearRegression.java (working copy) @@ -26,6 +26,7 @@ import org.apache.hama.commons.math.DoubleVector; import org.apache.hama.commons.math.FunctionFactory; import org.apache.hama.ml.ann.SmallLayeredNeuralNetwork; +import org.apache.hama.ml.util.FeatureTransformer; /** * Linear regression model. It can be used for numeric regression or prediction. @@ -175,5 +176,13 @@ public DoubleVector getWeights() { return ann.getWeightsByLayer(0).getRowVector(0); } + + /** + * Set the feature transformer. + * @param transformer + */ + public void setFeatureTransformer(FeatureTransformer featureTransformer) { + this.ann.setFeatureTransformer(featureTransformer); + } } Index: ml/src/main/java/org/apache/hama/ml/regression/LogisticRegression.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/regression/LogisticRegression.java (revision 1547052) +++ ml/src/main/java/org/apache/hama/ml/regression/LogisticRegression.java (working copy) @@ -25,6 +25,7 @@ import org.apache.hama.commons.math.DoubleVector; import org.apache.hama.commons.math.FunctionFactory; import org.apache.hama.ml.ann.SmallLayeredNeuralNetwork; +import org.apache.hama.ml.util.FeatureTransformer; /** * The logistic regression model. It can be used to conduct 2-class @@ -169,4 +170,11 @@ return ann.getWeightsByLayer(0).getRowVector(0); } + /** + * Set the feature transformer. + * @param featureTransformer + */ + public void setFeatureTransformer(FeatureTransformer featureTransformer) { + this.ann.setFeatureTransformer(featureTransformer); + } } Index: ml/src/main/java/org/apache/hama/ml/util/DefaultFeatureTransformer.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/util/DefaultFeatureTransformer.java (revision 0) +++ ml/src/main/java/org/apache/hama/ml/util/DefaultFeatureTransformer.java (working copy) @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ml.util; + +import org.apache.hama.commons.math.DoubleVector; + +/** + * The defaultFeatureTransformer actually returns the original features. + */ +public class DefaultFeatureTransformer extends FeatureTransformer { + + /** + * Directly return the original features. + */ + @Override + public DoubleVector transform(DoubleVector originalFeatures) { + return originalFeatures; + } + +} Index: ml/src/main/java/org/apache/hama/ml/util/FeatureTransformer.java =================================================================== --- ml/src/main/java/org/apache/hama/ml/util/FeatureTransformer.java (revision 0) +++ ml/src/main/java/org/apache/hama/ml/util/FeatureTransformer.java (working copy) @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hama.ml.util; + +import org.apache.hama.commons.math.DoubleVector; + +/** + * FeatureTransformer defines the interface to transform the original features + * to new space. + * + * NOTE: the user defined feature transformer must have a constructor with no parameters. + * + */ +public abstract class FeatureTransformer { + + public FeatureTransformer() { + } + + /** + * Transform the original features to transformed space. + * @param originalFeatureVector + * @return + */ + public abstract DoubleVector transform(DoubleVector originalFeatures); + +} Index: ml/src/test/java/org/apache/hama/ml/ann/TestAutoEncoder.java =================================================================== --- ml/src/test/java/org/apache/hama/ml/ann/TestAutoEncoder.java (revision 1547052) +++ ml/src/test/java/org/apache/hama/ml/ann/TestAutoEncoder.java (working copy) @@ -40,7 +40,6 @@ import org.apache.hama.commons.math.DenseDoubleVector; import org.apache.hama.commons.math.DoubleVector; import org.apache.hama.ml.MLTestBase; -import org.junit.Ignore; import org.junit.Test; import org.mortbay.log.Log; @@ -77,7 +76,6 @@ } - @Ignore @Test public void testAutoEncoderSwissRollDataset() { List instanceList = new ArrayList(); Index: ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetwork.java =================================================================== --- ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetwork.java (revision 1547052) +++ ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetwork.java (working copy) @@ -47,6 +47,8 @@ import org.apache.hama.ml.MLTestBase; import org.apache.hama.ml.ann.AbstractLayeredNeuralNetwork.LearningStyle; import org.apache.hama.ml.ann.AbstractLayeredNeuralNetwork.TrainingMethod; +import org.apache.hama.ml.util.DefaultFeatureTransformer; +import org.apache.hama.ml.util.FeatureTransformer; import org.junit.Test; import org.mortbay.log.Log; @@ -79,6 +81,10 @@ matrices[1] = new DenseDoubleMatrix(1, 6, 0.8); ann.setWeightMatrices(matrices); ann.setLearningStyle(LearningStyle.UNSUPERVISED); + + FeatureTransformer defaultFeatureTransformer = new DefaultFeatureTransformer(); + ann.setFeatureTransformer(defaultFeatureTransformer); + // write to file String modelPath = "/tmp/testSmallLayeredNeuralNetworkReadWrite"; @@ -111,6 +117,9 @@ } } } + + FeatureTransformer copyTransformer = annCopy.getFeatureTransformer(); + assertEquals(defaultFeatureTransformer.getClass().getName(), copyTransformer.getClass().getName()); } @Test @@ -408,8 +417,13 @@ Log.info(String.format("Relative error: %f%%\n", errorRate * 100)); } + + @Test + public void testLogisticRegression() { + this.testLogisticRegressionDistributedVersion(); + this.testLogisticRegressionDistributedVersionWithFeatureTransformer(); + } - @Test public void testLogisticRegressionDistributedVersion() { // write data into a sequence file String tmpStrDatasetPath = "/tmp/logistic_regression_data"; @@ -514,5 +528,116 @@ (double) (end - start) / 1000)); Log.info(String.format("Relative error: %f%%\n", errorRate * 100)); } + + public void testLogisticRegressionDistributedVersionWithFeatureTransformer() { + // write data into a sequence file + String tmpStrDatasetPath = "/tmp/logistic_regression_data_feature_transformer"; + Path tmpDatasetPath = new Path(tmpStrDatasetPath); + String strDataPath = "src/test/resources/logistic_regression_data.txt"; + String modelPath = "/tmp/logistic-regression-distributed-model-feature-transformer"; + Configuration conf = new Configuration(); + List instanceList = new ArrayList(); + List trainingInstances = null; + List testInstances = null; + + try { + FileSystem fs = FileSystem.get(new URI(tmpStrDatasetPath), conf); + fs.delete(tmpDatasetPath, true); + if (fs.exists(tmpDatasetPath)) { + fs.createNewFile(tmpDatasetPath); + } + + BufferedReader br = new BufferedReader(new FileReader(strDataPath)); + String line = null; + int count = 0; + while ((line = br.readLine()) != null) { + String[] tokens = line.trim().split(","); + double[] instance = new double[tokens.length]; + for (int i = 0; i < tokens.length; ++i) { + instance[i] = Double.parseDouble(tokens[i]); + } + instanceList.add(instance); + } + br.close(); + + zeroOneNormalization(instanceList, instanceList.get(0).length - 1); + + // write training data to temporal sequence file + SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, + tmpDatasetPath, LongWritable.class, VectorWritable.class); + int testSize = 150; + + Collections.shuffle(instanceList); + testInstances = new ArrayList(); + testInstances.addAll(instanceList.subList(instanceList.size() - testSize, + instanceList.size())); + trainingInstances = instanceList.subList(0, instanceList.size() + - testSize); + + for (double[] instance : trainingInstances) { + DoubleVector vec = new DenseDoubleVector(instance); + writer.append(new LongWritable(count++), new VectorWritable(vec)); + } + writer.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + + // create model + int dimension = 8; + SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); + ann.setLearningRate(0.7); + ann.setMomemtumWeight(0.5); + ann.setRegularizationWeight(0.1); + ann.addLayer(dimension, false, + FunctionFactory.createDoubleFunction("Sigmoid")); + ann.addLayer(dimension, false, + FunctionFactory.createDoubleFunction("Sigmoid")); + ann.addLayer(dimension, false, + FunctionFactory.createDoubleFunction("Sigmoid")); + ann.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); + ann.setCostFunction(FunctionFactory + .createDoubleDoubleFunction("CrossEntropy")); + ann.setModelPath(modelPath); + + FeatureTransformer featureTransformer = new DefaultFeatureTransformer(); + + ann.setFeatureTransformer(featureTransformer); + + long start = new Date().getTime(); + Map trainingParameters = new HashMap(); + trainingParameters.put("tasks", "5"); + trainingParameters.put("training.max.iterations", "2000"); + trainingParameters.put("training.batch.size", "300"); + trainingParameters.put("convergence.check.interval", "1000"); + ann.train(tmpDatasetPath, trainingParameters); + + + long end = new Date().getTime(); + + // validate results + double errorRate = 0; + // calculate the error on test instance + for (double[] testInstance : testInstances) { + DoubleVector instance = new DenseDoubleVector(testInstance); + double expected = instance.get(instance.getDimension() - 1); + instance = instance.slice(instance.getDimension() - 1); + instance = featureTransformer.transform(instance); + double actual = ann.getOutput(instance).get(0); + if (actual < 0.5 && expected >= 0.5 || actual >= 0.5 && expected < 0.5) { + ++errorRate; + } + } + errorRate /= testInstances.size(); + + Log.info(String.format("Training time: %fs\n", + (double) (end - start) / 1000)); + Log.info(String.format("Relative error: %f%%\n", errorRate * 100)); + } + } Index: ml/src/test/java/org/apache/hama/ml/perception/TestSmallMultiLayerPerceptron.java =================================================================== --- ml/src/test/java/org/apache/hama/ml/perception/TestSmallMultiLayerPerceptron.java (revision 1547052) +++ ml/src/test/java/org/apache/hama/ml/perception/TestSmallMultiLayerPerceptron.java (working copy) @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Random; +import org.apache.commons.lang.SerializationUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -39,6 +40,8 @@ import org.apache.hama.commons.math.DenseDoubleVector; import org.apache.hama.commons.math.DoubleMatrix; import org.apache.hama.commons.math.DoubleVector; +import org.apache.hama.ml.util.DefaultFeatureTransformer; +import org.apache.hama.ml.util.FeatureTransformer; import org.junit.Test; import org.mortbay.log.Log; @@ -59,6 +62,8 @@ MultiLayerPerceptron mlp = new SmallMultiLayerPerceptron(learningRate, regularization, momentum, squashingFunctionName, costFunctionName, layerSizeArray); + FeatureTransformer transformer = new DefaultFeatureTransformer(); + mlp.setFeatureTransformer(transformer); try { mlp.writeModelToFile(modelPath); } catch (IOException e) { @@ -78,6 +83,7 @@ assertEquals(squashingFunctionName, mlp.getSquashingFunctionName()); assertEquals(costFunctionName, mlp.getCostFunctionName()); assertArrayEquals(layerSizeArray, mlp.getLayerSizeArray()); + assertEquals(transformer.getClass().getName(), mlp.getFeatureTransformer().getClass().getName()); // delete test file fs.delete(new Path(modelPath), true); } catch (IOException e) { @@ -134,6 +140,14 @@ for (DoubleMatrix mat : matrices) { MatrixWritable.write(mat, output); } + + // serialize the feature transformer + FeatureTransformer transformer = new DefaultFeatureTransformer(); + Class featureTransformerCls = transformer.getClass(); + byte[] featureTransformerBytes = SerializationUtils.serialize(featureTransformerCls); + output.writeInt(featureTransformerBytes.length); + output.write(featureTransformerBytes); + output.close(); } catch (IOException e) { @@ -352,11 +366,16 @@ e.printStackTrace(); } } + + @Test + public void testByRunningJobs() { + this.testTrainingByXOR(); + this.testFeatureTransformer(); + } /** * Test the XOR problem. */ - @Test public void testTrainingByXOR() { // write in some training instances Configuration conf = new Configuration(); @@ -429,5 +448,77 @@ } } } + + /** + * Use transformer to extract the first half features of the original features. + */ + public void testFeatureTransformer() { + // write in some training instances + Configuration conf = new Configuration(); + String strDataPath = "/tmp/xor-training-by-xor"; + Path dataPath = new Path(strDataPath); + // generate training data + DoubleVector[] trainingData = new DenseDoubleVector[] { + new DenseDoubleVector(new double[] { 0, 0, 0 }), + new DenseDoubleVector(new double[] { 0, 1, 1 }), + new DenseDoubleVector(new double[] { 1, 0, 1 }), + new DenseDoubleVector(new double[] { 1, 1, 0 }) }; + + try { + URI uri = new URI(strDataPath); + FileSystem fs = FileSystem.get(uri, conf); + fs.delete(dataPath, true); + if (!fs.exists(dataPath)) { + fs.createNewFile(dataPath); + SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, + dataPath, LongWritable.class, VectorWritable.class); + + for (int i = 0; i < 1000; ++i) { + VectorWritable vecWritable = new VectorWritable(trainingData[i % 4]); + writer.append(new LongWritable(i), vecWritable); + } + writer.close(); + } + + } catch (Exception e) { + e.printStackTrace(); + } + + // begin training + String modelPath = "/tmp/xorModel-training-by-xor.data"; + double learningRate = 0.6; + double regularization = 0.02; // no regularization + double momentum = 0.3; // no momentum + String squashingFunctionName = "Tanh"; + String costFunctionName = "SquaredError"; + int[] layerSizeArray = new int[] { 1, 5, 1 }; + SmallMultiLayerPerceptron mlp = new SmallMultiLayerPerceptron(learningRate, + regularization, momentum, squashingFunctionName, costFunctionName, + layerSizeArray); + + mlp.setFeatureTransformer(new FeatureTransformer() { + + @Override + public DoubleVector transform(DoubleVector originalFeatures) { + return originalFeatures.sliceUnsafe(originalFeatures.getDimension() / 2); + } + + }); + + Map trainingParams = new HashMap(); + trainingParams.put("training.iteration", "2000"); + trainingParams.put("training.mode", "minibatch.gradient.descent"); + trainingParams.put("training.batch.size", "100"); + trainingParams.put("tasks", "3"); + trainingParams.put("modelPath", modelPath); + + try { + mlp.train(dataPath, trainingParams); + } catch (Exception e) { + e.printStackTrace(); + } + + } + }