From 806382372c2e5f3413db3c8f21268660dce72ccd Mon Sep 17 00:00:00 2001 From: "hzfengyu@corp.netease.com" Date: Tue, 8 Dec 2015 19:50:07 +0800 Subject: [PATCH 5/8] git metadata package part patch, KYLIN-1172 Signed-off-by: hzfengyu@corp.netease.com --- .../apache/kylin/metadata/project/ProjectInstance.java | 15 ++++++++++++++- .../org/apache/kylin/metadata/project/ProjectManager.java | 8 ++++---- .../apache/kylin/metadata/tool/HiveSourceTableLoader.java | 9 +++++++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java index 71740ca..976a87c 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java @@ -70,6 +70,9 @@ public class ProjectInstance extends RootPersistentEntity { @JsonProperty("realizations") private List realizationEntries; + + @JsonProperty("hiveName") + private String hiveName; public String getResourcePath() { return concatResourcePath(name); @@ -86,13 +89,15 @@ public class ProjectInstance extends RootPersistentEntity { return project.toUpperCase(); } - public static ProjectInstance create(String name, String owner, String description, List realizationEntries) { + public static ProjectInstance create(String name, String owner, String description, String hiveName, List realizationEntries) { ProjectInstance projectInstance = new ProjectInstance(); projectInstance.updateRandomUuid(); projectInstance.setName(name); projectInstance.setOwner(owner); projectInstance.setDescription(description); + if(hiveName != null) + projectInstance.setHiveName(hiveName); projectInstance.setStatus(ProjectStatusEnum.ENABLED); projectInstance.setCreateTimeUTC(System.currentTimeMillis()); if (realizationEntries != null) @@ -233,6 +238,14 @@ public class ProjectInstance extends RootPersistentEntity { public void setRealizationEntries(List entries) { this.realizationEntries = entries; } + + public String getHiveName() { + return this.hiveName; + } + + public void setHiveName(String hiveName) { + this.hiveName = hiveName; + } public void init() { if (name == null) diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index fc74731..920ad25 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -155,12 +155,12 @@ public class ProjectManager { return projectMap.get(projectName); } - public ProjectInstance createProject(String projectName, String owner, String description) throws IOException { + public ProjectInstance createProject(String projectName, String owner, String description, String hiveName) throws IOException { logger.info("Creating project '" + projectName); ProjectInstance currentProject = getProject(projectName); if (currentProject == null) { - currentProject = ProjectInstance.create(projectName, owner, description, null); + currentProject = ProjectInstance.create(projectName, owner, description, hiveName, null); } else { throw new IllegalStateException("The project named " + projectName + "already exists"); } @@ -193,7 +193,7 @@ public class ProjectManager { public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException { if (!project.getName().equals(newName)) { - ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc); + ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc, null); // FIXME table lost?? newProject.setCreateTimeUTC(project.getCreateTimeUTC()); newProject.recordUpdateTime(System.currentTimeMillis()); @@ -225,7 +225,7 @@ public class ProjectManager { String newProjectName = norm(project); ProjectInstance newProject = getProject(newProjectName); if (newProject == null) { - newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding realization " + realizationName + "(" + type + ")"); + newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding realization " + realizationName + "(" + type + ")", null); } newProject.addRealizationEntry(type, realizationName); saveResource(newProject); diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java b/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java index 81b3830..3719d70 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java @@ -30,10 +30,12 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HadoopUtil; import org.apache.kylin.common.util.HiveClient; +import org.apache.kylin.common.util.HiveManager; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.metadata.project.ProjectManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,12 +82,15 @@ public class HiveSourceTableLoader { } private static List extractHiveTables(String database, Set tables, KylinConfig config) throws IOException { - List loadedTables = Lists.newArrayList(); MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + String hiveName = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(HiveManager.getInstance().getCurrentProject()). + getHiveName(); + HiveClient hiveClient = HiveManager.getInstance().createHiveClient(hiveName); + for (String tableName : tables) { Table table = null; - HiveClient hiveClient = new HiveClient(); +// HiveClient hiveClient = new HiveClient(); List partitionFields = null; List fields = null; try { -- 1.9.4.msysgit.2