From 86f1ffae7ce9d4cf63cfd1cd6e4cf43d091d7483 Mon Sep 17 00:00:00 2001 From: ZhansShaoxiong Date: Sat, 2 Dec 2017 15:28:00 +0800 Subject: [PATCH] KYLIN-3052 Support Redshift as data source --- .../java/org/apache/kylin/job/constant/ExecutableConstants.java | 1 + .../main/java/org/apache/kylin/rest/service/TableService.java | 8 ++++---- .../main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java | 3 ++- .../org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java | 6 ++---- .../apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java | 9 ++------- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index 36496fead..9eae271ff 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -35,6 +35,7 @@ public final class ExecutableConstants { public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension Dictionary"; public static final String STEP_NAME_BUILD_UHC_DICTIONARY = "Build UHC Dictionary"; public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table"; + public static final String STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE = "Sqoop To Flat Hive Table"; public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = "Materialize Hive View in Lookup Tables"; public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns"; public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base Cuboid"; diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index ffb71a698..901ac46e0 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -119,7 +119,7 @@ public class TableService extends BasicService { SetMultimap db2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { String[] parts = HadoopUtil.parseHiveTableName(fullTableName); - db2tables.put(parts[0].toUpperCase(), parts[1].toUpperCase()); + db2tables.put(parts[0], parts[1]); } // load all tables first @@ -128,9 +128,9 @@ public class TableService extends BasicService { for (Map.Entry entry : db2tables.entries()) { Pair pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project); TableDesc tableDesc = pair.getFirst(); - Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey())); - Preconditions.checkState(tableDesc.getName().equals(entry.getValue())); - Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey() + "." + entry.getValue())); + Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase())); + Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase())); + Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase() + "." + entry.getValue().toUpperCase())); TableExtDesc extDesc = pair.getSecond(); Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity())); allMeta.add(pair); diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java index 15259cc18..0bfc145ec 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java @@ -74,6 +74,7 @@ public class JdbcHiveMRInput extends HiveMRInput { HiveCmdStep step = new HiveCmdStep(); step.setCmd(hiveInitStatements + dropTableHql + createTableHql); + step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE); return step; } @@ -171,7 +172,7 @@ public class JdbcHiveMRInput extends HiveMRInput { logger.debug(String.format("sqoop cmd:%s", cmd)); CmdStep step = new CmdStep(); step.setCmd(cmd); - step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE); + step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE); return step; } diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java index 6404fd6c5..54c2a033c 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java @@ -35,10 +35,8 @@ public class MySQLJdbcMetadata extends DefaultJdbcMetadata { @Override public List listDatabases() throws SQLException { List ret = new ArrayList<>(); - try (Connection con = SqlUtil.getConnection(dbconf); ResultSet res = con.getMetaData().getCatalogs()) { - while (res.next()) { - ret.add(res.getString("TABLE_CAT")); - } + try (Connection con = SqlUtil.getConnection(dbconf)) { + ret.add(con.getCatalog()); } return ret; } diff --git a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java index d0cb6c4a3..e461c1588 100644 --- a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java +++ b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java @@ -47,16 +47,11 @@ public class MySQLJdbcMetadataTest extends DefaultJdbcMetadataTest { @Test public void testListDatabases() throws SQLException { - ResultSet rs = mock(ResultSet.class); - when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false); - when(rs.getString("TABLE_CAT")).thenReturn("catalog1").thenReturn("catalog2"); - - when(connection.getMetaData()).thenReturn(dbmd); - when(dbmd.getCatalogs()).thenReturn(rs); + when(connection.getCatalog()).thenReturn("catalog1"); List dbs = jdbcMetadata.listDatabases(); - Assert.assertEquals(2, dbs.size()); + Assert.assertEquals(1, dbs.size()); Assert.assertEquals("catalog1", dbs.get(0)); } -- 2.14.1