From 98e8994c2ae2250660064c9f2227cdb870a5f636 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Fri, 16 Sep 2016 11:51:11 +0800 Subject: [PATCH] KYLIN-1565: Read KV max size from HBase config --- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ---- .../apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java | 10 +++++----- .../org/apache/kylin/storage/hbase/HBaseResourceStore.java | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 3b06ed8..3e41055 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -570,10 +570,6 @@ abstract public class KylinConfigBase implements Serializable { return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft); } - public int getHBaseKeyValueSize() { - return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760")); - } - public String getDefaultIGTStorage() { return getOptional("kylin.query.storage.default.gtstorage", "org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC"); } diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java index bc5cdf1..5a2d5dd 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java @@ -57,12 +57,12 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase { String path = "/cube/_test_large_cell.json"; String largeContent = "THIS_IS_A_LARGE_CELL"; StringEntity content = new StringEntity(largeContent); - KylinConfig config = KylinConfig.getInstanceFromEnv(); - int origSize = config.getHBaseKeyValueSize(); + Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); + int origSize = Integer.parseInt(hconf.get("hbase.client.keyvalue.maxsize", "10485760")); ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); try { - config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1)); + hconf.set("hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1)); store.deleteResource(path); @@ -72,7 +72,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase { assertEquals(content, t); Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path); - Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); + FileSystem fileSystem = FileSystem.get(hconf); assertTrue(fileSystem.exists(redirectPath)); @@ -82,7 +82,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase { store.deleteResource(path); } finally { - config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize); + hconf.set("hbase.client.keyvalue.maxsize", "" + origSize); store.deleteResource(path); } } diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index e2f3661..ee4b7e4 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -364,7 +364,7 @@ public class HBaseResourceStore extends ResourceStore { } private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException { - int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize(); + int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760")); if (content.length > kvSizeLimit) { writeLargeCellToHdfs(resPath, content, table); content = BytesUtil.EMPTY_BYTE_ARRAY; -- 2.7.4 (Apple Git-66)