From b44f60edb6f20225b308fb4324cf30fa1c34fc47 Mon Sep 17 00:00:00 2001 From: srinivasan Date: Thu, 18 Jun 2015 22:38:36 +0530 Subject: [PATCH 1/5] Added support for external tables --- .../org/apache/kylin/dict/lookup/HiveTable.java | 22 ++++++++++++++----- 1 files changed, 16 insertions(+), 6 deletions(-) diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java index ae224ac..b57cad7 100644 --- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java +++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java @@ -48,6 +48,8 @@ public class HiveTable implements ReadableTable { private int nColumns; private String hdfsLocation; private FileTable fileTable; + private HiveClient hiveClient; + private boolean needFilePath; public HiveTable(MetadataManager metaMgr, String table) { TableDesc tableDesc = metaMgr.getTableDesc(table); @@ -73,19 +75,19 @@ public class HiveTable implements ReadableTable { private FileTable getFileTable() throws IOException { if (fileTable == null) { - fileTable = new FileTable(getHDFSLocation(true), nColumns); + fileTable = new FileTable(getHDFSLocation(), nColumns); } return fileTable; } - public String getHDFSLocation(boolean needFilePath) throws IOException { + public String getHDFSLocation() throws IOException { if (hdfsLocation == null) { - hdfsLocation = computeHDFSLocation(needFilePath); + hdfsLocation = computeHDFSLocation(); } return hdfsLocation; } - private String computeHDFSLocation(boolean needFilePath) throws IOException { + private String computeHDFSLocation() throws IOException { String override = KylinConfig.getInstanceFromEnv().getOverrideHiveTableLocation(hiveTable); if (override != null) { @@ -95,8 +97,8 @@ public class HiveTable implements ReadableTable { String hdfsDir = null; try { - HiveClient hiveClient = new HiveClient(); - hdfsDir = hiveClient.getHiveTableLocation(database, hiveTable); + hdfsDir = getHiveClient().getHiveTableLocation(database, hiveTable); + needFilePath = getHiveClient().isManagedTable(database, hiveTable); } catch (Exception e) { e.printStackTrace(); throw new IOException(e); @@ -129,4 +131,12 @@ public class HiveTable implements ReadableTable { return "hive: database=[" + database + "], table=[" + hiveTable + "]"; } + public HiveClient getHiveClient() { + + if (hiveClient == null) { + hiveClient = new HiveClient(); + } + return hiveClient; + } + } -- 1.7.1 From 0a08266797217af04400298dd458819bcc1e3d49 Mon Sep 17 00:00:00 2001 From: Srinivasan Date: Thu, 18 Jun 2015 22:40:37 +0530 Subject: [PATCH 2/5] Added method to find hive table type --- .../org/apache/kylin/common/util/HiveClient.java | 14 ++++++++++++++ 1 files changed, 14 insertions(+), 0 deletions(-) diff --git a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java index cfeea31..d832b15 100644 --- a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java +++ b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java @@ -44,6 +44,7 @@ public class HiveClient { protected HiveConf hiveConf = null; protected Driver driver = null; protected HiveMetaStoreClient metaStoreClient = null; + protected String type; public HiveClient() { hiveConf = new HiveConf(HiveClient.class); @@ -155,4 +156,17 @@ public class HiveClient { return result; } + /** + * + * @param database + * @param tableName + * @throws Exception + */ + + public boolean isManagedTable(String database, String tableName) throws Exception{ + + return getMetaStoreClient().getTable(database, tableName).getTableType().toLowerCase().startsWith("managed"); + + } + } -- 1.7.1 From 09ee52fe3263b69d25bbc7a3ba2601d6dc50074f Mon Sep 17 00:00:00 2001 From: Srinivasan Date: Sat, 20 Jun 2015 00:25:21 +0530 Subject: [PATCH 3/5] Added method to find table is native or non native --- .../org/apache/kylin/common/util/HiveClient.java | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java index d832b15..aab6e09 100644 --- a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java +++ b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; @@ -111,6 +112,7 @@ public class HiveClient { return metaStoreClient; } + public Table getHiveTable(String database, String tableName) throws Exception { return getMetaStoreClient().getTable(database, tableName); } @@ -163,9 +165,10 @@ public class HiveClient { * @throws Exception */ - public boolean isManagedTable(String database, String tableName) throws Exception{ + public boolean isNativeTable(String database, String tableName) throws Exception{ + + return !MetaStoreUtils.isNonNativeTable(getMetaStoreClient().getTable(database, tableName)); - return getMetaStoreClient().getTable(database, tableName).getTableType().toLowerCase().startsWith("managed"); } -- 1.7.1 From af04573c92a76cb319e7c4b4013a26932de874b1 Mon Sep 17 00:00:00 2001 From: Srinivasan Date: Sat, 20 Jun 2015 00:26:35 +0530 Subject: [PATCH 4/5] Added code to check for files under HDFS location only for native hive tables --- .../org/apache/kylin/dict/lookup/HiveTable.java | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java index b57cad7..aa22814 100644 --- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java +++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java @@ -49,7 +49,7 @@ public class HiveTable implements ReadableTable { private String hdfsLocation; private FileTable fileTable; private HiveClient hiveClient; - private boolean needFilePath; + private boolean nativeTable; public HiveTable(MetadataManager metaMgr, String table) { TableDesc tableDesc = metaMgr.getTableDesc(table); @@ -74,8 +74,14 @@ public class HiveTable implements ReadableTable { } private FileTable getFileTable() throws IOException { - if (fileTable == null) { - fileTable = new FileTable(getHDFSLocation(), nColumns); + try { + if (fileTable == null) { + nativeTable = getHiveClient().isNativeTable(database, hiveTable); + fileTable = new FileTable(getHDFSLocation(), nColumns, nativeTable); + } + } catch (Exception e) { + e.printStackTrace(); + throw new IOException(e); } return fileTable; } @@ -98,13 +104,12 @@ public class HiveTable implements ReadableTable { String hdfsDir = null; try { hdfsDir = getHiveClient().getHiveTableLocation(database, hiveTable); - needFilePath = getHiveClient().isManagedTable(database, hiveTable); } catch (Exception e) { e.printStackTrace(); throw new IOException(e); } - if (needFilePath) { + if (nativeTable) { FileSystem fs = HadoopUtil.getFileSystem(hdfsDir); FileStatus file = findOnlyFile(hdfsDir, fs); return file.getPath().toString(); -- 1.7.1 From 4e87b0d13b0ffdbc1efdb1f1c1374ee164d3bb06 Mon Sep 17 00:00:00 2001 From: Srinivasan Date: Sat, 20 Jun 2015 00:28:02 +0530 Subject: [PATCH 5/5] Addeed current timestamp as last accessed time for non native tables --- .../org/apache/kylin/dict/lookup/FileTable.java | 18 +++++++++++++++--- 1 files changed, 15 insertions(+), 3 deletions(-) diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java index e6e3de9..a659ca3 100644 --- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java +++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java @@ -35,15 +35,24 @@ public class FileTable implements ReadableTable { String path; String delim; int nColumns; + boolean nativeTable; public FileTable(String path, int nColumns) { - this(path, DELIM_AUTO, nColumns); + this(path, DELIM_AUTO, nColumns, true); } - public FileTable(String path, String delim, int nColumns) { + public FileTable(String path, String delim, int nColumns, boolean nativeTable) { this.path = path; this.delim = delim; this.nColumns = nColumns; + this.nativeTable = nativeTable; + } + + public FileTable(String path, int nColumns, boolean nativeTable) { + this.path = path; + this.delim = DELIM_AUTO; + this.nColumns = nColumns; + this.nativeTable = nativeTable; } @Override @@ -60,7 +69,10 @@ public class FileTable implements ReadableTable { public TableSignature getSignature() throws IOException { FileSystem fs = HadoopUtil.getFileSystem(path); FileStatus status = fs.getFileStatus(new Path(path)); - return new TableSignature(path, status.getLen(), status.getModificationTime()); + if (nativeTable) { + return new TableSignature(path, status.getLen(), status.getModificationTime()); + } + return new TableSignature(path, status.getLen(), System.currentTimeMillis()); } @Override -- 1.7.1