From debde85dd5eae81b48917ad44340a04c4ec81973 Mon Sep 17 00:00:00 2001 From: John Zhao Date: Thu, 7 Apr 2016 14:39:52 -0700 Subject: [PATCH] KYLIN-1571 Update pom and update hbase API for CubeHTableUtil & MergeGCStep --- pom.xml | 2 +- .../kylin/storage/hbase/steps/CubeHTableUtil.java | 16 ++++++------- .../kylin/storage/hbase/steps/MergeGCStep.java | 26 ++++++++++++---------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index 2574b73..4e224f0 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ 3.4.6 0.14.0 0.14.0 - 0.98.8-hadoop2 + 1.1.3 0.8.1 diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java index 65192c2..d92b651 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java @@ -21,12 +21,12 @@ package org.apache.kylin.storage.hbase.steps; import java.io.IOException; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.regionserver.BloomType; @@ -72,11 +72,11 @@ public class CubeHTableUtil { tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString()); - Configuration conf = HBaseConfiguration.create(); - HBaseAdmin admin = new HBaseAdmin(conf); + Connection connection = ConnectionFactory.createConnection(); + Admin admin = connection.getAdmin(); try { - if (User.isHBaseSecurityEnabled(conf)) { + if (User.isHBaseSecurityEnabled(connection.getConfiguration())) { // add coprocessor for bulk load tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint"); } @@ -132,7 +132,7 @@ public class CubeHTableUtil { tableDesc.addFamily(cf); } - if (admin.tableExists(tableName)) { + if (admin.tableExists(TableName.valueOf(tableName))) { // admin.disableTable(tableName); // admin.deleteTable(tableName); throw new RuntimeException("HBase table " + tableName + " exists!"); @@ -141,7 +141,7 @@ public class CubeHTableUtil { DeployCoprocessorCLI.deployCoprocessor(tableDesc); admin.createTable(tableDesc, splitKeys); - Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons"); + Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons"); logger.info("create hbase table " + tableName + " done."); } catch (Exception e) { logger.error("Failed to create HTable", e); diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java index a4a8a35..ab15a01 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java @@ -24,12 +24,12 @@ import java.util.Collections; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Bytes; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableContext; @@ -69,19 +69,21 @@ public class MergeGCStep extends AbstractExecutable { List oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); - Configuration conf = HBaseConfiguration.create(); - HBaseAdmin admin = null; + + Admin admin = null; try { - admin = new HBaseAdmin(conf); + Connection connection = ConnectionFactory.createConnection(); + admin = connection.getAdmin(); for (String table : oldTables) { - if (admin.tableExists(table)) { - HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table)); + TableName tbName = TableName.valueOf(table); + if (admin.tableExists(tbName)) { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tbName); String host = tableDescriptor.getValue(IRealizationConstants.HTableTag); if (metadataUrlPrefix.equalsIgnoreCase(host)) { - if (admin.isTableEnabled(table)) { - admin.disableTable(table); + if (admin.isTableEnabled(tbName)) { + admin.disableTable(tbName); } - admin.deleteTable(table); + admin.deleteTable(tbName); logger.debug("Dropped htable: " + table); output.append("HBase table " + table + " is dropped. \n"); } else { -- 2.6.4 (Apple Git-63)