From 177965ddeeb2db4727b06d1d0817d4fc20b90c3d Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Tue, 6 Jan 2015 10:34:01 +0800 Subject: [PATCH 1/3] small update in SnapshotManagerTest.java --- dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java b/dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java index 14819e3..db69510 100644 --- a/dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java +++ b/dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java @@ -43,8 +43,10 @@ @Before public void setup() throws Exception { - ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath()); createTestMetadata(); + if (useSandbox()) { + ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath()); + } snapshotMgr = SnapshotManager.getInstance(this.getTestConfig()); } From 97d7a9c6f95d0ba294bd8a5ee39a2d1229c83b6f Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Tue, 6 Jan 2015 13:50:50 +0800 Subject: [PATCH 2/3] Code refine in HiveClient.java --- .../common/util/HBaseMetadataTestCase.java | 17 +++++- .../java/com/kylinolap/dict/lookup/HiveTable.java | 7 +-- .../com/kylinolap/dict/lookup/HiveTableReader.java | 68 +++++++++++++++------- .../com/kylinolap/dict/HiveTableReaderTest.java | 26 +++------ .../com/kylinolap/dict/SnapshotManagerTest.java | 4 -- .../test/java/com/kylinolap/job/DeployUtil.java | 40 +++++++------ .../com/kylinolap/job/SampleCubeSetupTest.java | 12 ---- .../com/kylinolap/metadata/tool/HiveClient.java | 66 ++++++++++++--------- .../metadata/tool/HiveSourceTableLoader.java | 6 +- 9 files changed, 134 insertions(+), 112 deletions(-) diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java index f11f2a8..3b39912 100644 --- a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java +++ b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java @@ -16,11 +16,23 @@ package com.kylinolap.common.util; +import java.io.File; + /** * @author ysong1 */ public class HBaseMetadataTestCase extends AbstractKylinTestCase { + static { + if (useSandbox()) { + try { + ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath()); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + @Override public void createTestMetadata() throws Exception { staticCreateTestMetadata(); @@ -38,13 +50,12 @@ public static void staticCreateTestMetadata() throws Exception { staticCreateTestMetadata(AbstractKylinTestCase.MINICLUSTER_TEST_DATA); HBaseMiniclusterMetadataTestCase.startupMinicluster(); } - + } - + public static boolean useSandbox() { String useSandbox = System.getProperty("useSandbox"); return Boolean.parseBoolean(useSandbox); - } } diff --git a/dictionary/src/main/java/com/kylinolap/dict/lookup/HiveTable.java b/dictionary/src/main/java/com/kylinolap/dict/lookup/HiveTable.java index 0baca68..6fc2564 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/lookup/HiveTable.java +++ b/dictionary/src/main/java/com/kylinolap/dict/lookup/HiveTable.java @@ -23,7 +23,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,17 +92,15 @@ private String computeHDFSLocation(boolean needFilePath) throws IOException { return override; } - Table table = null; + String hdfsDir = null; try { HiveClient hiveClient = new HiveClient(); - HiveMetaStoreClient metaDataClient = hiveClient.getMetaStoreClient(); - table = metaDataClient.getTable(database, hiveTable); + hdfsDir = hiveClient.getHiveTableLocation(database, hiveTable); } catch (Exception e) { e.printStackTrace(); throw new IOException(e); } - String hdfsDir = table.getSd().getLocation(); if (needFilePath) { FileSystem fs = HadoopUtil.getFileSystem(hdfsDir); FileStatus file = findOnlyFile(hdfsDir, fs); diff --git a/dictionary/src/main/java/com/kylinolap/dict/lookup/HiveTableReader.java b/dictionary/src/main/java/com/kylinolap/dict/lookup/HiveTableReader.java index fa485f5..1855c36 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/lookup/HiveTableReader.java +++ b/dictionary/src/main/java/com/kylinolap/dict/lookup/HiveTableReader.java @@ -18,23 +18,31 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.data.transfer.DataTransferFactory; import org.apache.hive.hcatalog.data.transfer.HCatReader; +import org.apache.hive.hcatalog.data.transfer.ReadEntity; import org.apache.hive.hcatalog.data.transfer.ReaderContext; -import com.kylinolap.metadata.tool.HiveClient; - +/** + * An implementation of TableReader with HCatalog for Hive table. + * @author shaoshi + * + */ public class HiveTableReader implements TableReader { private String dbName; private String tableName; private int currentSplit = -1; - private ReaderContext readCntxt; + private ReaderContext readCntxt = null; private Iterator currentHCatRecordItr = null; private HCatRecord currentHCatRecord; private int numberOfSplits = 0; @@ -42,9 +50,12 @@ public HiveTableReader(String dbName, String tableName) throws IOException { this.dbName = dbName; this.tableName = tableName; + initialize(); + } + + private void initialize() throws IOException { try { - HiveClient hiveClient = new HiveClient(); - this.readCntxt = hiveClient.getReaderContext(dbName, tableName); + this.readCntxt = getHiveReaderContext(dbName, tableName); } catch (Exception e) { e.printStackTrace(); throw new IOException(e); @@ -52,23 +63,17 @@ public HiveTableReader(String dbName, String tableName) throws IOException { this.numberOfSplits = readCntxt.numSplits(); } - - @Override - public void close() throws IOException { - this.readCntxt = null; - this.currentHCatRecordItr = null; - this.currentHCatRecord = null; - this.currentSplit = -1; - } - + @Override public boolean next() throws IOException { + while (currentHCatRecordItr == null || !currentHCatRecordItr.hasNext()) { currentSplit++; if (currentSplit == numberOfSplits) { return false; } - currentHCatRecordItr = loadHCatRecordItr(currentSplit); + + currentHCatRecordItr = loadHCatRecordItr(readCntxt, currentSplit); } currentHCatRecord = currentHCatRecordItr.next(); @@ -76,11 +81,6 @@ public boolean next() throws IOException { return true; } - private Iterator loadHCatRecordItr(int dataSplit) throws HCatException { - HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit); - return currentHCatReader.read(); - } - @Override public String[] getRow() { List allFields = currentHCatRecord.getAll(); @@ -97,8 +97,36 @@ public void setExpectedColumnNumber(int expectedColumnNumber) { } + @Override + public void close() throws IOException { + this.readCntxt = null; + this.currentHCatRecordItr = null; + this.currentHCatRecord = null; + this.currentSplit = -1; + } + public String toString() { return "hive table reader for: " + dbName + "." + tableName; } + private static ReaderContext getHiveReaderContext(String database, String table) throws Exception { + HiveConf hiveConf = new HiveConf(HiveTableReader.class); + Iterator> itr = hiveConf.iterator(); + Map map = new HashMap(); + while (itr.hasNext()) { + Entry kv = itr.next(); + map.put(kv.getKey(), kv.getValue()); + } + + ReadEntity entity = new ReadEntity.Builder().withDatabase(database).withTable(table).build(); + HCatReader reader = DataTransferFactory.getHCatReader(entity, map); + ReaderContext cntxt = reader.prepareRead(); + + return cntxt; + } + + private static Iterator loadHCatRecordItr(ReaderContext readCntxt, int dataSplit) throws HCatException { + HCatReader currentHCatReader = DataTransferFactory.getHCatReader(readCntxt, dataSplit); + return currentHCatReader.read(); + } } diff --git a/dictionary/src/test/java/com/kylinolap/dict/HiveTableReaderTest.java b/dictionary/src/test/java/com/kylinolap/dict/HiveTableReaderTest.java index 60fadef..b5718ef 100644 --- a/dictionary/src/test/java/com/kylinolap/dict/HiveTableReaderTest.java +++ b/dictionary/src/test/java/com/kylinolap/dict/HiveTableReaderTest.java @@ -16,34 +16,25 @@ package com.kylinolap.dict; -import java.io.File; import java.io.IOException; import org.apache.commons.lang.ArrayUtils; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; -import com.kylinolap.common.util.ClasspathUtil; +import com.kylinolap.common.util.HBaseMetadataTestCase; import com.kylinolap.dict.lookup.HiveTableReader; -public class HiveTableReaderTest { - - private static HiveTableReader reader = null; - - @BeforeClass - public static void setup() throws Exception { - ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath()); - - reader = new HiveTableReader("default", "test_kylin_fact"); - } - - public static void tearDown() throws IOException { - reader.close(); - } +/** + * This test case need the hive runtime; Please run it with sandbox; It is in the exclude list of default profile in pom.xml + * @author shaoshi + * + */ +public class HiveTableReaderTest extends HBaseMetadataTestCase { @Test public void test() throws IOException { + HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact"); int rowNumber = 0; while (reader.next()) { String[] row = reader.getRow(); @@ -53,5 +44,6 @@ public void test() throws IOException { } Assert.assertEquals(10000, rowNumber); + reader.close(); } } diff --git a/dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java b/dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java index db69510..d1c3526 100644 --- a/dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java +++ b/dictionary/src/test/java/com/kylinolap/dict/SnapshotManagerTest.java @@ -44,10 +44,6 @@ @Before public void setup() throws Exception { createTestMetadata(); - if (useSandbox()) { - ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath()); - } - snapshotMgr = SnapshotManager.getInstance(this.getTestConfig()); } diff --git a/job/src/test/java/com/kylinolap/job/DeployUtil.java b/job/src/test/java/com/kylinolap/job/DeployUtil.java index 1825436..983936f 100644 --- a/job/src/test/java/com/kylinolap/job/DeployUtil.java +++ b/job/src/test/java/com/kylinolap/job/DeployUtil.java @@ -1,9 +1,11 @@ package com.kylinolap.job; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; -import com.kylinolap.common.persistence.ResourceStore; -import com.kylinolap.job.tools.LZOSupportnessChecker; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.maven.model.Model; @@ -14,6 +16,7 @@ import org.slf4j.LoggerFactory; import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.persistence.ResourceStore; import com.kylinolap.common.persistence.ResourceTool; import com.kylinolap.common.util.AbstractKylinTestCase; import com.kylinolap.common.util.CliCommandExecutor; @@ -22,9 +25,11 @@ import com.kylinolap.job.dataGen.FactTableGenerator; import com.kylinolap.job.engine.JobEngineConfig; import com.kylinolap.job.hadoop.hive.SqlHiveDataTypeMapping; +import com.kylinolap.job.tools.LZOSupportnessChecker; import com.kylinolap.metadata.MetadataManager; import com.kylinolap.metadata.model.ColumnDesc; import com.kylinolap.metadata.model.TableDesc; +import com.kylinolap.metadata.tool.HiveClient; public class DeployUtil { @SuppressWarnings("unused") @@ -200,26 +205,23 @@ private static void deployHiveTables() throws Exception { } temp.delete(); + HiveClient hiveClient = new HiveClient(); + // create hive tables - execHiveCommand("CREATE DATABASE IF NOT EXISTS EDW;"); - execHiveCommand(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); - execHiveCommand(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase()))); - execHiveCommand(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase()))); - execHiveCommand(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM.toUpperCase()))); - execHiveCommand(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase()))); + hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW;"); + hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); + hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase()))); + hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase()))); + hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM.toUpperCase()))); + hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase()))); // load data to hive tables // LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename - execHiveCommand(generateLoadDataHql(TABLE_CAL_DT)); - execHiveCommand(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS)); - execHiveCommand(generateLoadDataHql(TABLE_KYLIN_FACT)); - execHiveCommand(generateLoadDataHql(TABLE_SELLER_TYPE_DIM)); - execHiveCommand(generateLoadDataHql(TABLE_SITES)); - } - - private static void execHiveCommand(String hql) throws IOException { - String hiveCmd = "hive -e \"" + hql + "\""; - config().getCliCommandExecutor().execute(hiveCmd); + hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES)); } private static String generateLoadDataHql(String tableName) { diff --git a/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java b/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java index 7709461..6537a08 100644 --- a/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java +++ b/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java @@ -25,13 +25,6 @@ @Before public void before() throws Exception { - try { - this.testConnectivity(); - } catch (Exception e) { - System.out.println("Failed to connect to remote CLI with given password"); - throw e; - } - String confPaths = System.getenv("KYLIN_HBASE_CONF_PATH"); System.out.println("The conf paths is " + confPaths); if (confPaths != null) { @@ -49,11 +42,6 @@ public void before() throws Exception { } } - private void testConnectivity() throws Exception { - KylinConfig cfg = KylinConfig.getInstanceFromEnv(); - cfg.getCliCommandExecutor().execute("echo hello"); - } - @Test public void testCubes() throws Exception { DeployUtil.initCliWorkDir(); diff --git a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveClient.java b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveClient.java index 398a293..c6f6855 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveClient.java +++ b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveClient.java @@ -3,13 +3,19 @@ import java.io.IOException; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; @@ -18,6 +24,7 @@ import org.apache.hive.hcatalog.data.transfer.HCatReader; import org.apache.hive.hcatalog.data.transfer.ReadEntity; import org.apache.hive.hcatalog.data.transfer.ReaderContext; +import org.apache.thrift.TException; /* * Copyright 2013-2014 eBay Software Foundation @@ -41,15 +48,8 @@ protected Driver driver = null; protected HiveMetaStoreClient metaStoreClient = null; - public HiveClient() throws MetaException { - setup(); - } - - private void setup() throws MetaException { - hiveConf = new HiveConf(HiveSourceTableLoader.class); - driver = new Driver(hiveConf); - metaStoreClient = new HiveMetaStoreClient(hiveConf); - SessionState.start(new CliSessionState(hiveConf)); + public HiveClient() { + hiveConf = new HiveConf(HiveClient.class); } public HiveConf getHiveConf() { @@ -60,39 +60,49 @@ public HiveConf getHiveConf() { * Get the hive ql driver to execute ddl or dml * @return */ - public Driver getDriver() { + private Driver getDriver() { + if (driver == null) { + driver = new Driver(hiveConf); + SessionState.start(new CliSessionState(hiveConf)); + } + return driver; } /** - * Get the Hive Meta store client; - * @return + * + * @param hql + * @throws CommandNeedRetryException + * @throws IOException */ - public HiveMetaStoreClient getMetaStoreClient() { - return metaStoreClient; + public void executeHQL(String hql) throws CommandNeedRetryException, IOException { + int retCode = getDriver().run(hql).getResponseCode(); + if (retCode != 0) { + throw new IOException("Failed to execute hql [" + hql + "], return code from hive driver : [" + retCode + "]"); + } } - public ReaderContext getReaderContext(String database, String table) throws MetaException, CommandNeedRetryException, IOException, ClassNotFoundException { - - Iterator> itr = hiveConf.iterator(); - Map map = new HashMap(); - while (itr.hasNext()) { - Entry kv = itr.next(); - map.put(kv.getKey(), kv.getValue()); + private HiveMetaStoreClient getMetaStoreClient() throws Exception { + if (metaStoreClient == null) { + metaStoreClient = new HiveMetaStoreClient(hiveConf); } + return metaStoreClient; + } - ReaderContext readCntxt = runsInMaster(map, database, table); + public Table getHiveTable(String database, String tableName) throws Exception { + return getMetaStoreClient().getTable(database, tableName); + } - return readCntxt; + public List getHiveTableFields(String database, String tableName) throws Exception { + return getMetaStoreClient().getFields(database, tableName); } - private ReaderContext runsInMaster(Map config, String database, String table) throws HCatException { - ReadEntity entity = new ReadEntity.Builder().withDatabase(database).withTable(table).build(); - HCatReader reader = DataTransferFactory.getHCatReader(entity, config); - ReaderContext cntxt = reader.prepareRead(); - return cntxt; + public String getHiveTableLocation(String database, String tableName) throws Exception { + Table t = getHiveTable(database, tableName); + return t.getSd().getLocation(); } + public HCatReader getHCatReader(ReaderContext cntxt, int slaveNum) throws HCatException { HCatReader reader = DataTransferFactory.getHCatReader(cntxt, slaveNum); diff --git a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java index 3a62844..ff5005c 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java +++ b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java @@ -26,7 +26,6 @@ import java.util.Set; import java.util.UUID; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; import org.slf4j.Logger; @@ -116,9 +115,8 @@ List fields = null; try { HiveClient hiveClient = new HiveClient(); - HiveMetaStoreClient metaDataClient = hiveClient.getMetaStoreClient(); - table = metaDataClient.getTable(database, tableName); - fields = metaDataClient.getFields(database, tableName); + table = hiveClient.getHiveTable(database, tableName); + fields = hiveClient.getHiveTableFields(database, tableName); } catch (Exception e) { e.printStackTrace(); throw new IOException(e); From bb09b932b60d8c8ea4ddd2c2beb1b7f94163c3fd Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Tue, 6 Jan 2015 17:28:48 +0800 Subject: [PATCH 3/3] Use hive client driver to execute hql --- .../test/java/com/kylinolap/job/DeployUtil.java | 29 +++++++++++----------- .../com/kylinolap/metadata/tool/HiveClient.java | 5 ++++ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/job/src/test/java/com/kylinolap/job/DeployUtil.java b/job/src/test/java/com/kylinolap/job/DeployUtil.java index 983936f..6a65376 100644 --- a/job/src/test/java/com/kylinolap/job/DeployUtil.java +++ b/job/src/test/java/com/kylinolap/job/DeployUtil.java @@ -200,15 +200,15 @@ private static void deployHiveTables() throws Exception { hbaseDataStream.close(); localFileStream.close(); - config().getCliCommandExecutor().copyFile(localBufferFile.getPath(), config().getCliWorkingDir()); - localBufferFile.delete(); + localBufferFile.deleteOnExit(); } + String tableFileDir = temp.getParent(); temp.delete(); HiveClient hiveClient = new HiveClient(); // create hive tables - hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW;"); + hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW"); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase()))); hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase()))); @@ -217,21 +217,22 @@ private static void deployHiveTables() throws Exception { // load data to hive tables // LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename - hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM)); - hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM, tableFileDir)); + hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir)); } - private static String generateLoadDataHql(String tableName) { - return "LOAD DATA LOCAL INPATH '" + config().getCliWorkingDir() + "/" + tableName.toUpperCase() + ".csv' OVERWRITE INTO TABLE " + tableName.toUpperCase(); + private static String generateLoadDataHql(String tableName, String tableFileDir) { + return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName.toUpperCase() + ".csv' OVERWRITE INTO TABLE " + tableName.toUpperCase(); } - private static String generateCreateTableHql(TableDesc tableDesc) { + private static String[] generateCreateTableHql(TableDesc tableDesc) { + + String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity(); StringBuilder ddl = new StringBuilder(); - ddl.append("DROP TABLE IF EXISTS " + tableDesc.getIdentity() + ";\n"); ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n"); ddl.append("(" + "\n"); @@ -245,9 +246,9 @@ private static String generateCreateTableHql(TableDesc tableDesc) { ddl.append(")" + "\n"); ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); - ddl.append("STORED AS TEXTFILE;"); + ddl.append("STORED AS TEXTFILE"); - return ddl.toString(); + return new String[] {dropsql, ddl.toString()}; } } diff --git a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveClient.java b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveClient.java index c6f6855..ae53cbf 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveClient.java +++ b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveClient.java @@ -81,6 +81,11 @@ public void executeHQL(String hql) throws CommandNeedRetryException, IOException throw new IOException("Failed to execute hql [" + hql + "], return code from hive driver : [" + retCode + "]"); } } + + public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException { + for(String sql: hqls) + executeHQL(sql); + } private HiveMetaStoreClient getMetaStoreClient() throws Exception { if (metaStoreClient == null) {