diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java index 3dca0a3..540d2d5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.security; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; @@ -37,6 +38,9 @@ public class SecureBulkLoadUtil { } public static Path getBaseStagingDir(Configuration conf) { - return new Path(conf.get(BULKLOAD_STAGING_DIR)); + String hbaseTmpFsDir = + conf.get(HConstants.TEMPORARY_HDFS_DIRECTORY_KEY, + HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY); + return new Path(conf.get(BULKLOAD_STAGING_DIR, hbaseTmpFsDir)); } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 6fafad3..83c20b8 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1264,6 +1264,10 @@ public final class HConstants { public static final String ZK_SERVER_KERBEROS_PRINCIPAL = "hbase.zookeeper.server.kerberos.principal"; + /** Config key for hbase temporary directory in hdfs */ + public static final String TEMPORARY_HDFS_DIRECTORY_KEY = "hbase.fs.tmp.dir"; + public static final String DEFAULT_TEMPORARY_HDFS_DIRECTORY = "/tmp/hbase-staging"; + private HConstants() { // Can't be instantiated with this ctor. } diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index d9a1994..936fe2b 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -64,7 +64,7 @@ possible configurations would overwhelm and obscure the important. hbase.fs.tmp.dir - /user/${user.name}/hbase-staging + /tmp/hbase-staging A staging directory in default file system (HDFS) for keeping temporary data. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 3b041f0..ced074e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -636,7 +636,10 @@ public class HFileOutputFormat2 Configuration conf = job.getConfiguration(); // create the partitions file FileSystem fs = FileSystem.get(conf); - Path partitionsPath = new Path(conf.get("hbase.fs.tmp.dir"), "partitions_" + UUID.randomUUID()); + String hbaseTmpFsDir = + conf.get(HConstants.TEMPORARY_HDFS_DIRECTORY_KEY, + HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY); + Path partitionsPath = new Path(hbaseTmpFsDir, "partitions_" + UUID.randomUUID()); fs.makeQualified(partitionsPath); writePartitions(conf, partitionsPath, splitPoints); fs.deleteOnExit(partitionsPath); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index 337eeac..d9508ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -346,10 +346,11 @@ public class TestHFileOutputFormat2 { assertTrue(files.length > 0); } - @Ignore("Goes zombie too frequently; needs work. See HBASE-14563") @Test + @Test public void testJobConfiguration() throws Exception { Configuration conf = new Configuration(this.util.getConfiguration()); - conf.set("hbase.fs.tmp.dir", util.getDataTestDir("testJobConfiguration").toString()); + conf.set(HConstants.TEMPORARY_HDFS_DIRECTORY_KEY, util.getDataTestDir("testJobConfiguration") + .toString()); Job job = new Job(conf); job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration")); Table table = Mockito.mock(Table.class); @@ -360,6 +361,22 @@ public class TestHFileOutputFormat2 { assertEquals(job.getNumReduceTasks(), 4); } + @Test + public void testJobConfigurationWithNoResource() throws Exception { + Configuration conf = new Configuration(); + Job job = new Job(conf); + job.setWorkingDirectory(util.getDataTestDir("testJobConfigurationWithNoResource")); + Table table = Mockito.mock(Table.class); + RegionLocator regionLocator = Mockito.mock(RegionLocator.class); + setupMockStartKeys(regionLocator); + setupMockTableName(regionLocator); + LOG.info("hbase.fs.tmp.dir setting in job conf: " + + job.getConfiguration().get(HConstants.TEMPORARY_HDFS_DIRECTORY_KEY, + HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY)); + HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); + assertEquals(job.getNumReduceTasks(), 4); + } + private byte [][] generateRandomStartKeys(int numKeys) { Random random = new Random(); byte[][] ret = new byte[numKeys][];