From 7d7279d19d534f660eaa8dd9dec63fe8b547fb89 Mon Sep 17 00:00:00 2001 From: Yi Liang Date: Mon, 3 Apr 2017 20:39:35 -0700 Subject: [PATCH 1/2] HBASE-17861: Regionserver down when checking the permission of staging dir if hbase.rootdir is on S3 Signed-off-by: tedyu (cherry picked from commit 4057a6c89c74c9f595cb51ac3bdc288396a0b257) Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java --- .../hadoop/hbase/security/access/SecureBulkLoadEndpoint.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java index fc2baaf7f9..f0169ab61f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java @@ -145,16 +145,20 @@ public class SecureBulkLoadEndpoint extends SecureBulkLoadService Set fsSet = getFileSystemSchemesWithoutPermissionSupport(conf); try { - fs = FileSystem.get(conf); - fs.mkdirs(baseStagingDir, PERM_HIDDEN); - fs.setPermission(baseStagingDir, PERM_HIDDEN); + fs = baseStagingDir.getFileSystem(conf); + if (!fs.exists(baseStagingDir)) { + fs.mkdirs(baseStagingDir, PERM_HIDDEN); + } else { + fs.setPermission(baseStagingDir, PERM_HIDDEN); + } //no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased fs.mkdirs(new Path(baseStagingDir,"DONOTERASE"), PERM_HIDDEN); FileStatus status = fs.getFileStatus(baseStagingDir); if(status == null) { throw new IllegalStateException("Failed to create staging directory"); } - if(!status.getPermission().equals(PERM_HIDDEN)) { + String scheme = fs.getScheme().toLowerCase(); + if (!fsSet.contains(scheme) && !status.getPermission().equals(PERM_HIDDEN)) { throw new IllegalStateException( "Staging directory of " + baseStagingDir + " already exists but permissions aren't set to '-rwx--x--x' "); } -- 2.16.1 From d1b0c322373e27f10ac104ae2c2fd2645444929f Mon Sep 17 00:00:00 2001 From: Pankaj Kumar Date: Mon, 14 Aug 2017 21:27:45 +0800 Subject: [PATCH 2/2] HBASE-18512, Region Server will abort with IllegalStateException if HDFS umask has limited scope Signed-off-by: tedyu (cherry picked from commit 12201249383bb7dab56ff857fba074c6ed311990) --- .../security/access/SecureBulkLoadEndpoint.java | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java index f0169ab61f..2ad8cee857 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java @@ -148,15 +148,26 @@ public class SecureBulkLoadEndpoint extends SecureBulkLoadService fs = baseStagingDir.getFileSystem(conf); if (!fs.exists(baseStagingDir)) { fs.mkdirs(baseStagingDir, PERM_HIDDEN); - } else { - fs.setPermission(baseStagingDir, PERM_HIDDEN); } - //no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased - fs.mkdirs(new Path(baseStagingDir,"DONOTERASE"), PERM_HIDDEN); FileStatus status = fs.getFileStatus(baseStagingDir); - if(status == null) { + if (status == null) { throw new IllegalStateException("Failed to create staging directory"); } + + // If HDFS UMASK value has limited scope then staging directory permission may not be 711 + // after creation, so we should set staging directory permission explicitly. + if (!status.getPermission().equals(PERM_HIDDEN)) { + fs.setPermission(baseStagingDir, PERM_HIDDEN); + status = fs.getFileStatus(baseStagingDir); + } + + // no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased + Path doNotEraseDir = new Path(baseStagingDir, "DONOTERASE"); + if (!fs.exists(doNotEraseDir)) { + fs.mkdirs(doNotEraseDir, PERM_HIDDEN); + fs.setPermission(doNotEraseDir, PERM_HIDDEN); + } + String scheme = fs.getScheme().toLowerCase(); if (!fsSet.contains(scheme) && !status.getPermission().equals(PERM_HIDDEN)) { throw new IllegalStateException( -- 2.16.1