import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ByteWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Writer;
import org.apache.hadoop.io.SequenceFile.CompressionType;
public class Test {
public static void main(String[] args) throws IOException {
final Configuration c = new Configuration();
final FileSystem fs = FileSystem.get(c);
final int NUM = 1000*1000;
for (int i = 0; i < Integer.valueOf(args[0]); i ++) {
final int ii = i;
new Thread(new Runnable() {
@Override
public void run() {
try {
Writer f = SequenceFile.createWriter(fs, c, new Path("/test/" + ii ), ByteWritable.class, ByteWritable.class, CompressionType.NONE);
ByteWritable v = new ByteWritable();
long time = System.currentTimeMillis();
for (int i = 0; i < NUM; i ++)
f.append(v, v);
f.close();
long end = System.currentTimeMillis();
System.out.printf("%d opartions in %d msec. %f/second\n", NUM, end - time, (float)(1000 * NUM)/(end - time));
} catch (Exception e) {
e.printStackTrace();
}
}
}).start();
}
}
}
Bug
Resolved
Major
Poor IO Performance due to AtomicLong operations
