From d8ad3eaa3859bf141f012c3c0a1ea0f259181966 Mon Sep 17 00:00:00 2001 From: nichunen Date: Mon, 16 Nov 2015 19:30:36 +0800 Subject: [PATCH] KYLIN-1074 Load hive tables with selecting mode KYLIN-1074 Load hive tables with selecting mode --- conf/kylin.properties | 2 + .../kylin/rest/controller/TableController.java | 72 +++++++++ webapp/app/js/controllers/sourceMeta.js | 174 ++++++++++++++++++++- webapp/app/js/services/kylinProperties.js | 7 + webapp/app/js/services/tables.js | 6 +- webapp/app/partials/tables/source_table_tree.html | 26 +++ webapp/bower.json | 2 +- 7 files changed, 285 insertions(+), 4 deletions(-) diff --git a/conf/kylin.properties b/conf/kylin.properties index 7f2af8a..a38d35b 100644 --- a/conf/kylin.properties +++ b/conf/kylin.properties @@ -119,6 +119,8 @@ kylin.web.hadoop= kylin.web.diagnostic= #contact mail on web page ,optional kylin.web.contact_mail= +#set the number of databases and tables of each database shown on the hive tree +kylin.web.hive.limit=20 ###########################config info for front####################### diff --git a/server/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server/src/main/java/org/apache/kylin/rest/controller/TableController.java index 5e817e3..7f86fc5 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/TableController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/TableController.java @@ -18,7 +18,9 @@ package org.apache.kylin.rest.controller; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -194,4 +196,74 @@ public class TableController extends BasicController { this.cubeMgmtService = cubeService; } + /** + * Show all databases in Hive + * + * @return Hive databases list + * @throws IOException + */ + @RequestMapping(value = "/hive", method = { RequestMethod.GET }) + @ResponseBody + private static List showHiveDatabases() throws IOException { + String sql = "show databases;"; + List command = new ArrayList(); + + command.add("hive"); + command.add("-e"); + command.add(sql); + + List results = new ArrayList(); + + try { + ProcessBuilder hiveProcessBuilder = new ProcessBuilder(command); + Process hiveProcess = hiveProcessBuilder.start(); + BufferedReader br = new BufferedReader(new InputStreamReader(hiveProcess.getInputStream())); + String data; + + while ((data = br.readLine()) != null) { + results.add(data); + } + } catch (Exception e) { + e.printStackTrace(); + throw new IOException(e); + } + return results; + } + + /** + * Show all tables in a Hive database + * + * @return Hive table list + * @throws IOException + */ + @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET }) + @ResponseBody + private static List showHiveTables(@PathVariable String database) throws IOException { + String sql = "use " + database + ";show tables;"; + List command = new ArrayList(); + + command.add("hive"); + command.add("-e"); + command.add(sql); + + List results = new ArrayList(); + + try { + ProcessBuilder hiveProcessBuilder = new ProcessBuilder(command); + Process hiveProcess = hiveProcessBuilder.start(); + BufferedReader br = new BufferedReader(new InputStreamReader(hiveProcess.getInputStream())); + String data; + + while ((data = br.readLine()) != null) { + results.add(data); + } + } catch (Exception e) { + e.printStackTrace(); + throw new IOException(e); + } + return results; + } + +} + } diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js index d05dc7f..5de7f19 100755 --- a/webapp/app/js/controllers/sourceMeta.js +++ b/webapp/app/js/controllers/sourceMeta.js @@ -101,13 +101,185 @@ KylinApp }); }; - var ModalInstanceCtrl = function ($scope, $location, $modalInstance, tableNames, MessageService, projectName, scope) { + $scope.openTreeModal = function () { + $modal.open({ + templateUrl: 'addHiveTableFromTree.html', + controller: ModalInstanceCtrl, + resolve: { + tableNames: function () { + return $scope.tableNames; + }, + projectName:function(){ + return $scope.projectModel.selectedProject; + }, + scope: function () { + return $scope; + } + } + }); + }; + + var ModalInstanceCtrl = function ($scope, $location, $modalInstance, tableNames, MessageService,projectName, kylinConfig, scope) { $scope.tableNames = ""; $scope.projectName = projectName; $scope.cancel = function () { $modalInstance.dismiss('cancel'); }; + $scope.treeOptions = {multiSelection: true}; + $scope.selectedNodes = []; + $scope.hiveLimit = kylinConfig.getHiveLimit(); + + $scope.loadHive = function () { + if($scope.hiveLoaded) + return; + TableService.showHiveDatabases({}, function (databases) { + $scope.dbNum = databases.length; + if (databases.length > 0) { + $scope.hiveMap = {}; + for (var i = 0; i < databases.length; i++) { + var dbName = databases[i]; + var hiveData = {"dbname":dbName,"tables":[],"expanded":false}; + $scope.hive.push(hiveData); + $scope.hiveMap[dbName] = i; + } + } + $scope.hiveLoaded = true; + $scope.showMoreDatabases(); + }); + } + + $scope.showMoreTables = function(hiveTables, node){ + var shownTimes = parseInt(node.children.length / $scope.hiveLimit); + var from = $scope.hiveLimit * shownTimes; + var to = 0; + var hasMore = false; + if(from + $scope.hiveLimit > hiveTables.length) { + to = hiveTables.length - 1; + } else { + to = from + $scope.hiveLimit - 1; + hasMore = true; + } + if(!angular.isUndefined(node.children[from])){ + node.children.pop(); + } + + for(var idx = from; idx <= to; idx++){ + node.children.push({"label":node.label+'.'+hiveTables[idx],"id":idx-from+1,"children":[]}); + } + + if(hasMore){ + var loading = {"label":"","id":65535,"children":[]}; + node.children.push(loading); + } + } + + $scope.showAllTables = function(hiveTables, node){ + var shownTimes = parseInt(node.children.length / $scope.hiveLimit); + var from = $scope.hiveLimit * shownTimes; + var to = hiveTables.length - 1; + if(!angular.isUndefined(node.children[from])){ + node.children.pop(); + } + for(var idx = from; idx <= to; idx++){ + node.children.push({"label":node.label+'.'+hiveTables[idx],"id":idx-from+1,"children":[]}); + } + } + + $scope.showMoreDatabases = function(){ + var shownTimes = parseInt($scope.treedata.length / $scope.hiveLimit); + var from = $scope.hiveLimit * shownTimes; + var to = 0; + var hasMore = false; + if(from + $scope.hiveLimit > $scope.hive.length) { + to = $scope.hive.length - 1; + } else { + to = from + $scope.hiveLimit - 1; + hasMore = true; + } + if(!angular.isUndefined($scope.treedata[from])){ + $scope.treedata.pop(); + } + + for(var idx = from; idx <= to; idx++){ + var children = []; + var loading = {"label":"","id":0,"children":[]}; + children.push(loading); + $scope.treedata.push({"label":$scope.hive[idx].dbname,"id":idx+1,"children":children,"expanded":false}); + } + + if(hasMore){ + var loading = {"label":"","id":65535,"children":[0]}; + $scope.treedata.push(loading); + } + } + + $scope.showAllDatabases = function(){ + var shownTimes = parseInt($scope.treedata.length / $scope.hiveLimit); + var from = $scope.hiveLimit * shownTimes; + var to = $scope.hive.length - 1; + + if(!angular.isUndefined($scope.treedata[from])){ + $scope.treedata.pop(); + } + + for(var idx = from; idx <= to; idx++){ + var children = []; + var loading = {"label":"","id":0,"children":[]}; + children.push(loading); + $scope.treedata.push({"label":$scope.hive[idx].dbname,"id":idx+1,"children":children,"expanded":false}); + } + } + + $scope.showMoreClicked = function($parentNode){ + if($parentNode == null){ + $scope.showMoreDatabases(); + } else { + $scope.showMoreTables($scope.hive[$scope.hiveMap[$parentNode.label]].tables,$parentNode); + } + } + + $scope.showAllClicked = function($parentNode){ + if($parentNode == null){ + $scope.showAllDatabases(); + } else { + $scope.showAllTables($scope.hive[$scope.hiveMap[$parentNode.label]].tables,$parentNode); + } + } + + $scope.showToggle = function(node) { + if(node.expanded == false){ + TableService.showHiveTables({"database": node.label},function (hive_tables){ + var tables = []; + for (var i = 0; i < hive_tables.length; i++) { + tables.push(hive_tables[i]); + } + $scope.hive[$scope.hiveMap[node.label]].tables = tables; + $scope.showMoreTables(tables,node); + node.expanded = true; + }); + } + } + + $scope.showSelected = function(node) { + + } + + if(angular.isUndefined($scope.hive) || angular.isUndefined($scope.hiveLoaded) || angular.isUndefined($scope.treedata) ){ + $scope.hive = []; + $scope.hiveLoaded = false; + $scope.treedata = []; + $scope.loadHive(); + } + $scope.add = function () { + if($scope.tableNames.length === 0 && $scope.selectedNodes.length > 0) { + for(var i = 0; i < $scope.selectedNodes.length; i++){ + if($scope.selectedNodes[i].label.indexOf(".") >= 0){ + $scope.tableNames += ($scope.selectedNodes[i].label) += ','; + } + } + } + if ($scope.tableNames.trim() === "") { SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info'); return; diff --git a/webapp/app/js/services/kylinProperties.js b/webapp/app/js/services/kylinProperties.js index 76b8087..bf6339e 100644 --- a/webapp/app/js/services/kylinProperties.js +++ b/webapp/app/js/services/kylinProperties.js @@ -47,6 +47,13 @@ KylinApp.service('kylinConfig', function (AdminService, $log) { return this.timezone; } + this.getHiveLimit = function () { + if (!this.hiveLimit) { + this.hiveLimit = this.getProperty("kylin.web.hive.limit"); + } + return this.hiveLimit; + } + this.getDeployEnv = function () { if (!this.deployEnv) { this.deployEnv = this.getProperty("deploy.env").trim(); diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js index 1f1f15a..a92ae8d 100755 --- a/webapp/app/js/services/tables.js +++ b/webapp/app/js/services/tables.js @@ -17,12 +17,14 @@ */ KylinApp.factory('TableService', ['$resource', function ($resource, config) { - return $resource(Config.service.url + 'tables/:tableName/:action', {}, { + return $resource(Config.service.url + 'tables/:tableName/:action/:database', {}, { list: {method: 'GET', params: {}, cache: true, isArray: true}, get: {method: 'GET', params: {}, isArray: false}, getExd: {method: 'GET', params: {action: 'exd-map'}, isArray: false}, reload: {method: 'PUT', params: {action: 'reload'}, isArray: false}, loadHiveTable: {method: 'POST', params: {}, isArray: false}, - genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false} + genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false}, + showHiveDatabases: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true}, + showHiveTables: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true} }); }]); diff --git a/webapp/app/partials/tables/source_table_tree.html b/webapp/app/partials/tables/source_table_tree.html index bdbe079..7af88a4 100755 --- a/webapp/app/partials/tables/source_table_tree.html +++ b/webapp/app/partials/tables/source_table_tree.html @@ -26,6 +26,7 @@
+
@@ -60,3 +61,28 @@ + + diff --git a/webapp/bower.json b/webapp/bower.json index f9d99fb..5f1f66e 100755 --- a/webapp/bower.json +++ b/webapp/bower.json @@ -13,7 +13,7 @@ "font-awesome": "4.3.0", "angular-ui-ace": "0.1.0", "angular-base64": "2.0.1", - "angular-tree-control": "0.1.4", + "angular-tree-control": "0.2.8", "angularLocalStorage": "~0.3.0", "messenger": "1.4.1", "moment": "2.5.1", -- 1.9.5.msysgit.1