From b052d6f22930c45bd34f1d7a2fd493cbf4330791 Mon Sep 17 00:00:00 2001 From: Billy Liu Date: Tue, 3 Jan 2017 11:15:06 +0800 Subject: [PATCH] KYLIN-2323 refactor Table Ext Info, display table type --- .../org/apache/kylin/metadata/MetadataManager.java | 4 +- .../apache/kylin/metadata/model/TableExtDesc.java | 52 +--------------------- .../kylin/rest/response/TableDescResponse.java | 1 + .../apache/kylin/rest/service/TableService.java | 21 ++++----- .../kylin/source/hive/HiveSourceTableLoader.java | 16 +++---- webapp/app/js/services/tables.js | 2 - webapp/app/partials/tables/table_detail.html | 16 ++++--- 7 files changed, 28 insertions(+), 84 deletions(-) diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java index 3b6eb61..49ec96e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java @@ -504,7 +504,7 @@ public class MetadataManager { List models = new ArrayList<>(); for (DataModelDesc modelDesc : getModels(projectName)) { for (TableRef tableRef : modelDesc.getAllTables()) { - if (tableRef.getTableName().equalsIgnoreCase(tableName)) { + if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) { models.add(modelDesc.getName()); } } @@ -515,7 +515,7 @@ public class MetadataManager { public boolean isTableInAnyModel(String tableName) { for (DataModelDesc modelDesc : getModels()) { for (TableRef tableRef : modelDesc.getAllTables()) { - if (tableRef.getTableName().equalsIgnoreCase(tableName)) { + if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) { return true; } } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java index 6393dbf..97dd2c1 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java @@ -47,18 +47,8 @@ public class TableExtDesc extends RootPersistentEntity { @JsonProperty("sample_rows") private List sampleRows = new ArrayList<>(); - @JsonProperty("storage_location") - private String storageLocation; - @JsonProperty("owner") - private String owner; - @JsonProperty("last_access_time") - private String lastAccessTime; @JsonProperty("last_modified_time") private String lastModifiedTime; - @JsonProperty("partition_column") - private String partitionColumn; - @JsonProperty("total_file_size") - private String totalFileSize; @JsonProperty("total_rows") private String totalRows; @JsonProperty("data_source_properties") @@ -161,22 +151,6 @@ public class TableExtDesc extends RootPersistentEntity { this.tableName = this.tableName.toUpperCase(); } - public void setStorageLocation(String storageLocation) { - this.storageLocation = storageLocation; - } - - public String getStorageLocation() { - return this.storageLocation; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getOwner() { - return this.owner; - } - public void setLastModifiedTime(String lastModifiedTime) { this.lastModifiedTime = lastModifiedTime; } @@ -185,32 +159,8 @@ public class TableExtDesc extends RootPersistentEntity { return this.lastModifiedTime; } - public void setLastAccessTime(String lastAccessTime) { - this.lastAccessTime = lastAccessTime; - } - - public String getLastAccessTime() { - return this.lastAccessTime; - } - - public void setPartitionColumn(String partitionColumn) { - this.partitionColumn = partitionColumn; - } - - public String getPartitionColumn() { - return this.partitionColumn; - } - public boolean isPartitioned() { - return this.partitionColumn == null ? false : !this.partitionColumn.isEmpty(); - } - - public void setTotalFileSize(String totalFileSize) { - this.totalFileSize = totalFileSize; - } - - public String getTotalFileSize() { - return this.totalFileSize; + return this.dataSourceProps.get("partition_column") == null ? false : !this.dataSourceProps.get("partition_column").isEmpty(); } @Override diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java index f53b951..9bc5360 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java +++ b/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java @@ -76,6 +76,7 @@ public class TableDescResponse extends TableDesc { this.setName(table.getName()); this.setSourceType(table.getSourceType()); this.setUuid(table.getUuid()); + this.setTableType(table.getTableType()); } } \ No newline at end of file diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index 3325907..e3df60a 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.UUID; import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.engine.mr.HadoopUtil; import org.apache.kylin.engine.mr.common.HadoopShellExecutable; import org.apache.kylin.engine.mr.common.MapReduceExecutable; @@ -91,7 +92,7 @@ public class TableService extends BasicService { return table; } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN) + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public String[] loadHiveTablesToProject(String[] tables, String project) throws IOException { Set loaded = HiveSourceTableLoader.loadHiveTables(tables, getConfig()); String[] result = (String[]) loaded.toArray(new String[loaded.size()]); @@ -99,18 +100,17 @@ public class TableService extends BasicService { return result; } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) private void unLoadHiveTable(String tableName) throws IOException { tableName = normalizeHiveTableName(tableName); - HiveSourceTableLoader.unLoadHiveTable(tableName); + MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + metaMgr.removeSourceTable(tableName); + metaMgr.removeTableExt(tableName); } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) private void syncTableToProject(String[] tables, String project) throws IOException { getProjectManager().addTableDescToProject(tables, project); } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) private void removeTableFromProject(String tableName, String projectName) throws IOException { tableName = normalizeHiveTableName(tableName); getProjectManager().removeTableDescFromProject(tableName, projectName); @@ -123,6 +123,7 @@ public class TableService extends BasicService { * @param project * @return */ + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public boolean unLoadHiveTable(String tableName, String project) { boolean rtn = false; int tableType = 0; @@ -178,6 +179,7 @@ public class TableService extends BasicService { * @param project * @throws IOException */ + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public void addStreamingTable(TableDesc desc, String project) throws IOException { desc.setUuid(UUID.randomUUID().toString()); getMetadataManager().saveSourceTable(desc); @@ -230,11 +232,6 @@ public class TableService extends BasicService { rtableDesc.setCardinality(cardinality); } dataSourceProp.putAll(tableExtDesc.getDataSourceProp()); - dataSourceProp.put("location", tableExtDesc.getStorageLocation()); - dataSourceProp.put("owner", tableExtDesc.getOwner()); - dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime()); - dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn()); - dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize()); rtableDesc.setDescExd(dataSourceProp); return rtableDesc; } @@ -277,8 +274,8 @@ public class TableService extends BasicService { TableDesc table = getMetadataManager().getTableDesc(tableName); final TableExtDesc tableExt = getMetadataManager().getTableExt(tableName); if (table == null) { - IllegalArgumentException e = new IllegalArgumentException("Cannot find table descirptor " + tableName); - logger.error("Cannot find table descirptor " + tableName, e); + IllegalArgumentException e = new IllegalArgumentException("Cannot find table descriptor " + tableName); + logger.error("Cannot find table descriptor " + tableName, e); throw e; } diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java index b56009a..286ffac 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java @@ -76,12 +76,6 @@ public class HiveSourceTableLoader { return loadedTables; } - public static void unLoadHiveTable(String hiveTable) throws IOException { - MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - metaMgr.removeSourceTable(hiveTable); - metaMgr.removeTableExt(hiveTable); - } - private static List extractHiveTables(String database, Set tables, IHiveClient hiveClient) throws IOException { List loadedTables = Lists.newArrayList(); @@ -132,11 +126,11 @@ public class HiveSourceTableLoader { } TableExtDesc tableExtDesc = metaMgr.getTableExt(tableDesc.getIdentity()); - tableExtDesc.setStorageLocation(hiveTableMeta.sdLocation); - tableExtDesc.setOwner(hiveTableMeta.owner); - tableExtDesc.setLastAccessTime(String.valueOf(hiveTableMeta.lastAccessTime)); - tableExtDesc.setPartitionColumn(partitionColumnString.toString()); - tableExtDesc.setTotalFileSize(String.valueOf(hiveTableMeta.fileSize)); + tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation); + tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner); + tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime)); + tableExtDesc.addDataSourceProp("partition_column", partitionColumnString.toString()); + tableExtDesc.addDataSourceProp("total_file_size", String.valueOf(hiveTableMeta.fileSize)); tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(hiveTableMeta.fileNum)); tableExtDesc.addDataSourceProp("hive_inputFormat", hiveTableMeta.sdInputFormat); tableExtDesc.addDataSourceProp("hive_outputFormat", hiveTableMeta.sdOutputFormat); diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js index 4e7a7c4..38bf8a0 100755 --- a/webapp/app/js/services/tables.js +++ b/webapp/app/js/services/tables.js @@ -20,8 +20,6 @@ KylinApp.factory('TableService', ['$resource', function ($resource, config) { return $resource(Config.service.url + 'tables/:tableName/:action/:database', {}, { list: {method: 'GET', params: {}, cache: true, isArray: true}, get: {method: 'GET', params: {}, isArray: false}, - getExd: {method: 'GET', params: {action: 'exd-map'}, isArray: false}, - reload: {method: 'PUT', params: {action: 'reload'}, isArray: false}, loadHiveTable: {method: 'POST', params: {}, isArray: false}, unLoadHiveTable: {method: 'DELETE', params: {}, isArray: false}, genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false}, diff --git a/webapp/app/partials/tables/table_detail.html b/webapp/app/partials/tables/table_detail.html index 6399422..f227826 100644 --- a/webapp/app/partials/tables/table_detail.html +++ b/webapp/app/partials/tables/table_detail.html @@ -39,7 +39,7 @@ - + @@ -47,6 +47,10 @@ + + + + @@ -56,11 +60,11 @@ - + - + @@ -70,11 +74,11 @@ - + - + @@ -82,7 +86,7 @@ - +
NAMETABLE NAME {{ tableModel.selectedSrcTable.name}}
{{tableModel.selectedSrcTable.database}}
TABLE TYPE{{tableModel.selectedSrcTable.table_type}}
SNAPSHOT TIME {{tableModel.selectedSrcTable.exd.lastUpdateTime | utcToConfigTimeZone}}
INPUT FORMAT{{tableModel.selectedSrcTable.exd.inputformat}}{{tableModel.selectedSrcTable.exd.hive_inputFormat}}
OUTPUT FORMAT{{tableModel.selectedSrcTable.exd.outputformat}}{{tableModel.selectedSrcTable.exd.hive_outputFormat}}
OWNER
TOTAL FILE NUMBER{{tableModel.selectedSrcTable.exd.totalNumberFiles}}{{tableModel.selectedSrcTable.exd.total_file_number}}
TOTAL FILE SIZE{{tableModel.selectedSrcTable.exd.totalFileSize}}{{tableModel.selectedSrcTable.exd.total_file_size}}
PARTITIONED
PARTITION COLUMNS{{tableModel.selectedSrcTable.exd.partitionColumns}}{{tableModel.selectedSrcTable.exd.partition_column}}
-- 2.10.1 (Apple Git-78)