Index: conf/hadoop-default.xml
===================================================================
--- conf/hadoop-default.xml (revision 745106)
+++ conf/hadoop-default.xml (working copy)
@@ -582,6 +582,15 @@
+ dfs.support.append
+ false
+ Does HDFS allow appends to files?
+ This is currently set to false because there are bugs in the
+ "append code" and is not supported in any prodction cluster.
+
+
+
+
fs.s3.block.size
67108864
Block size to use when writing files to S3.
Index: src/test/org/apache/hadoop/hdfs/TestQuota.java
===================================================================
--- src/test/org/apache/hadoop/hdfs/TestQuota.java (revision 745106)
+++ src/test/org/apache/hadoop/hdfs/TestQuota.java (working copy)
@@ -429,6 +429,7 @@
// set a smaller block size so that we can test with smaller
// diskspace quotas
conf.set("dfs.block.size", "512");
+ conf.setBoolean("dfs.support.append", true);
final MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
final FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: "+fs.getUri(),
@@ -547,7 +548,6 @@
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
- /* NOTE: append() is not supported in 0.18.
OutputStream out = dfs.append(file2);
// appending 1 fileLen should succeed
out.write(new byte[fileLen]);
@@ -579,11 +579,7 @@
// verify space after partial append
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace);
- == end of append test == */
- // reduce quota for quotaDir1 to account for not appending
- dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 3 * fileSpace);
-
// Test set replication :
// first reduce the replication
@@ -591,7 +587,7 @@
// verify that space is reduced by file2Len
c = dfs.getContentSummary(dstPath);
- assertEquals(c.getSpaceConsumed(), 3 * fileSpace - file2Len);
+ assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
// now try to increase the replication and and expect an error.
hasException = false;
@@ -604,7 +600,7 @@
// verify space consumed remains unchanged.
c = dfs.getContentSummary(dstPath);
- assertEquals(c.getSpaceConsumed(), 3 * fileSpace - file2Len);
+ assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
// now increase the quota for quotaDir1 and quotaDir20
dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 10 * fileSpace);
@@ -614,7 +610,7 @@
dfs.setReplication(file2, (short)(replication+1));
// verify increase in space
c = dfs.getContentSummary(dstPath);
- assertEquals(c.getSpaceConsumed(), 3 * fileSpace + file2Len);
+ assertEquals(c.getSpaceConsumed(), 5 * fileSpace + file2Len);
} finally {
cluster.shutdown();
Index: src/test/org/apache/hadoop/hdfs/TestFileAppend2.java
===================================================================
--- src/test/org/apache/hadoop/hdfs/TestFileAppend2.java (revision 745106)
+++ src/test/org/apache/hadoop/hdfs/TestFileAppend2.java (working copy)
@@ -123,13 +123,13 @@
* Verify that all data exists in file.
*/
public void testSimpleAppend() throws IOException {
- /* HDFS append() is temporarily disabled in 0.19 */
- if (true) return;
Configuration conf = new Configuration();
if (simulatedStorage) {
conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
}
conf.setInt("dfs.datanode.handler.count", 50);
+ conf.setInt("dfs.datanode.handler.count", 50);
+ conf.setBoolean("dfs.support.append", true);
initBuffer(fileSize);
MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
FileSystem fs = cluster.getFileSystem();
@@ -371,8 +371,6 @@
* Test that appends to files at random offsets.
*/
public void testComplexAppend() throws IOException {
- /* HDFS append() is temporarily disabled in 0.19 */
- if (true) return;
initBuffer(fileSize);
Configuration conf = new Configuration();
conf.setInt("heartbeat.recheck.interval", 2000);
@@ -381,6 +379,7 @@
conf.setInt("dfs.socket.timeout", 30000);
conf.setInt("dfs.datanode.socket.write.timeout", 30000);
conf.setInt("dfs.datanode.handler.count", 50);
+ conf.setBoolean("dfs.support.append", true);
MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes,
true, null);
Index: src/test/org/apache/hadoop/hdfs/TestFileAppend3.java
===================================================================
--- src/test/org/apache/hadoop/hdfs/TestFileAppend3.java (revision 745106)
+++ src/test/org/apache/hadoop/hdfs/TestFileAppend3.java (working copy)
@@ -52,6 +52,7 @@
AppendTestUtil.LOG.info("setUp()");
conf = new Configuration();
conf.setInt("io.bytes.per.checksum", 512);
+ conf.setBoolean("dfs.support.append", true);
buffersize = conf.getInt("io.file.buffer.size", 4096);
cluster = new MiniDFSCluster(conf, DATANODE_NUM, true, null);
fs = (DistributedFileSystem)cluster.getFileSystem();
@@ -67,8 +68,6 @@
/** TC1: Append on block boundary. */
public void testTC1() throws Exception {
- /* HDFS append() is temporarily disabled in 0.19 */
- if (true) return;
final Path p = new Path("/TC1/foo");
System.out.println("p=" + p);
@@ -94,8 +93,6 @@
/** TC2: Append on non-block boundary. */
public void testTC2() throws Exception {
- /* HDFS append() is temporarily disabled in 0.19 */
- if (true) return;
final Path p = new Path("/TC2/foo");
System.out.println("p=" + p);
@@ -121,8 +118,6 @@
/** TC5: Only one simultaneous append. */
public void testTC5() throws Exception {
- /* HDFS append() is temporarily disabled in 0.19 */
- if (true) return;
final Path p = new Path("/TC5/foo");
System.out.println("p=" + p);
@@ -150,8 +145,6 @@
/** TC7: Corrupted replicas are present. */
public void testTC7() throws Exception {
- /* HDFS append() is temporarily disabled in 0.19 */
- if (true) return;
final short repl = 2;
final Path p = new Path("/TC7/foo");
System.out.println("p=" + p);
@@ -197,8 +190,6 @@
/** TC11: Racing rename */
public void testTC11() throws Exception {
- /* HDFS append() is temporarily disabled in 0.19 */
- if (true) return;
final Path p = new Path("/TC11/foo");
System.out.println("p=" + p);
@@ -252,8 +243,6 @@
/** TC12: Append to partial CRC chunk */
public void testTC12() throws Exception {
- /* HDFS append() is temporarily disabled in 0.19 */
- if (true) return;
final Path p = new Path("/TC12/foo");
System.out.println("p=" + p);
@@ -278,4 +267,4 @@
//c. Reopen file and read 25687+5877 bytes of data from file. Close file.
AppendTestUtil.check(fs, p, len1 + len2);
}
-}
\ No newline at end of file
+}
Index: src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
===================================================================
--- src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java (revision 745106)
+++ src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java (working copy)
@@ -175,11 +175,9 @@
/** This optional operation is not yet supported. */
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- // disable append() in 0.19.x
- throw new UnsupportedOperationException("HDFS does not support append yet");
-
- //DFSOutputStream op = (DFSOutputStream)dfs.append(getPathName(f), bufferSize, progress);
- //return new FSDataOutputStream(op, statistics, op.getInitialLen());
+
+ DFSOutputStream op = (DFSOutputStream)dfs.append(getPathName(f), bufferSize, progress);
+ return new FSDataOutputStream(op, statistics, op.getInitialLen());
}
public FSDataOutputStream create(Path f, FsPermission permission,