From 0c4ac465c66eb4d3453d0ec2cfe5e7f1f0fd65f8 Mon Sep 17 00:00:00 2001
From: hzyaozhongqiang <hzyaozhongqiang@corp.netease.com>
Date: Thu, 14 Sep 2017 13:14:22 +0800
Subject: [PATCH] hbase&hive address conflict bug

---
 .../org/apache/kylin/common/KylinConfigBase.java   | 25 ++++++++++++++++++++++
 .../storage/hbase/ITHBaseResourceStoreTest.java    |  2 +-
 .../kylin/storage/hbase/HBaseResourceStore.java    | 10 ++++-----
 3 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b622825..a008f44 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -226,6 +226,31 @@ abstract public class KylinConfigBase implements Serializable {
         return cachedHdfsWorkingDirectory;
     }
 
+
+    public String getHdfsWorkingDirectory(FileSystem fs) {
+        String root = getOptional("kylin.env.hdfs-working-dir", "/kylin");
+
+        Path path = new Path(root);
+        if (!path.isAbsolute())
+            throw new IllegalArgumentException("kylin.env.hdfs-working-dir must be absolute, but got " + root);
+
+        // make sure path is qualified
+        path = fs.makeQualified(path);
+
+        // append metadata-url prefix
+        root = new Path(path, StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).toString();
+
+        if (!root.endsWith("/"))
+            root += "/";
+
+        if (root.startsWith("file:")) {
+            root = root.replace("file:", "file://");
+        } else if (root.startsWith("maprfs:")) {
+            root = root.replace("maprfs:", "maprfs://");
+        }
+        return root;
+    }
+
     /**
      * A comma separated list of host:port pairs, each corresponding to a ZooKeeper server
      */
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
index d879fac..a1eb9ce 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
@@ -76,9 +76,9 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
             StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
             assertEquals(content, t);
 
-            Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
 
             FileSystem fileSystem = FileSystem.get(hconf);
+            Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path, fileSystem);
             assertTrue(fileSystem.exists(redirectPath));
 
             FSDataInputStream in = fileSystem.open(redirectPath);
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index ccbcde8..59394e0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -246,8 +246,8 @@ public class HBaseResourceStore extends ResourceStore {
         }
         byte[] value = r.getValue(B_FAMILY, B_COLUMN);
         if (value.length == 0) {
-            Path redirectPath = bigCellHDFSPath(resPath);
             FileSystem fileSystem = FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration());
+            Path redirectPath = bigCellHDFSPath(resPath, fileSystem);
 
             try {
                 return fileSystem.open(redirectPath);
@@ -339,8 +339,8 @@ public class HBaseResourceStore extends ResourceStore {
             table.delete(del);
 
             if (hdfsResourceExist) { // remove hdfs cell value
-                Path redirectPath = bigCellHDFSPath(resPath);
                 FileSystem fileSystem = FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration());
+                Path redirectPath = bigCellHDFSPath(resPath, fileSystem);
 
                 if (fileSystem.exists(redirectPath)) {
                     fileSystem.delete(redirectPath, true);
@@ -387,8 +387,8 @@ public class HBaseResourceStore extends ResourceStore {
     }
 
     private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
-        Path redirectPath = bigCellHDFSPath(resPath);
         FileSystem fileSystem = FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration());
+        Path redirectPath = bigCellHDFSPath(resPath, fileSystem);
 
         if (fileSystem.exists(redirectPath)) {
             fileSystem.delete(redirectPath, true);
@@ -405,8 +405,8 @@ public class HBaseResourceStore extends ResourceStore {
         return redirectPath;
     }
 
-    public Path bigCellHDFSPath(String resPath) {
-        String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory();
+    public Path bigCellHDFSPath(String resPath, FileSystem fs) {
+        String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory(fs);
         Path redirectPath = new Path(hdfsWorkingDirectory, "resources" + resPath);
         return redirectPath;
     }
-- 
2.7.2.windows.1